DYD_NE-day.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# DYD Thermostat Data \n",
    "\n",
    "## Preprocess\n",
    "\n",
    "1. Generated CSV file from queries in BigQueary\n",
    "\n",
    "2. Data separated into states\n",
    "\n",
    "3. Aggregated the data in Pandas by month\n",
    "\n",
    "4. Combine 4 years\n",
    "\n",
    "5. Group by Identifier\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Dependencies\n",
    "import pandas as pd\n",
    "import os\n",
    "import numpy as np\n",
    "from pathlib import Path\n",
    "from datetime import datetime"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "## January"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2017 January Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "jan_2017 = pd.read_csv(\"../data_large/NE-day/2017-jan-day-NE.csv\")\n",
    "\n",
    "# jan_2017"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>d78e9843002347d8166cbb8d4dd6b7907605bf29</td>\n",
       "      <td>2017-01-20 19:10:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>712</td>\n",
       "      <td>737</td>\n",
       "      <td>709</td>\n",
       "      <td>NE</td>\n",
       "      <td>Lincoln</td>\n",
       "      <td>25</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>46bf2a77cde604c97c4aa1f291d453ed4178676e</td>\n",
       "      <td>2017-01-07 19:35:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>709</td>\n",
       "      <td>762</td>\n",
       "      <td>692</td>\n",
       "      <td>NE</td>\n",
       "      <td>Lincoln</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>484f2ca2f908af5160afd910c8349433da3afc81</td>\n",
       "      <td>2017-01-08 16:30:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>648</td>\n",
       "      <td>655</td>\n",
       "      <td>652</td>\n",
       "      <td>NE</td>\n",
       "      <td>Hickman</td>\n",
       "      <td>0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>5182408f94362ec1153ac7567db8bdc5cb2f1aea</td>\n",
       "      <td>2017-01-28 16:30:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>673</td>\n",
       "      <td>673</td>\n",
       "      <td>663</td>\n",
       "      <td>NE</td>\n",
       "      <td>Kearney</td>\n",
       "      <td>35</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>4675e2b2038d8f9f67bb9217ca824d0727b629e1</td>\n",
       "      <td>2017-01-24 18:30:00 UTC</td>\n",
       "      <td>auxHeatOnly</td>\n",
       "      <td>hold</td>\n",
       "      <td>709</td>\n",
       "      <td>719</td>\n",
       "      <td>719</td>\n",
       "      <td>NE</td>\n",
       "      <td>Omaha</td>\n",
       "      <td>56</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>39093</th>\n",
       "      <td>89d1333ab2a46ce0b691b9b804c9dd61e64748e1</td>\n",
       "      <td>2017-01-27 14:50:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>701</td>\n",
       "      <td>760</td>\n",
       "      <td>700</td>\n",
       "      <td>NE</td>\n",
       "      <td>Lincoln</td>\n",
       "      <td>35</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>39094</th>\n",
       "      <td>89d1333ab2a46ce0b691b9b804c9dd61e64748e1</td>\n",
       "      <td>2017-01-21 18:50:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>714</td>\n",
       "      <td>760</td>\n",
       "      <td>700</td>\n",
       "      <td>NE</td>\n",
       "      <td>Lincoln</td>\n",
       "      <td>35</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>39095</th>\n",
       "      <td>89d1333ab2a46ce0b691b9b804c9dd61e64748e1</td>\n",
       "      <td>2017-01-27 16:50:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>697</td>\n",
       "      <td>760</td>\n",
       "      <td>700</td>\n",
       "      <td>NE</td>\n",
       "      <td>Lincoln</td>\n",
       "      <td>35</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>39096</th>\n",
       "      <td>c1c075cfc90c4712fe95f5afb28d980c6c427631</td>\n",
       "      <td>2017-01-10 17:00:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>712</td>\n",
       "      <td>760</td>\n",
       "      <td>710</td>\n",
       "      <td>NE</td>\n",
       "      <td>Lincoln</td>\n",
       "      <td>35</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>39097</th>\n",
       "      <td>c1c075cfc90c4712fe95f5afb28d980c6c427631</td>\n",
       "      <td>2017-01-10 17:05:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>710</td>\n",
       "      <td>760</td>\n",
       "      <td>710</td>\n",
       "      <td>NE</td>\n",
       "      <td>Lincoln</td>\n",
       "      <td>35</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>37096 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                     Identifier                date_time  \\\n",
       "1      d78e9843002347d8166cbb8d4dd6b7907605bf29  2017-01-20 19:10:00 UTC   \n",
       "2      46bf2a77cde604c97c4aa1f291d453ed4178676e  2017-01-07 19:35:00 UTC   \n",
       "3      484f2ca2f908af5160afd910c8349433da3afc81  2017-01-08 16:30:00 UTC   \n",
       "5      5182408f94362ec1153ac7567db8bdc5cb2f1aea  2017-01-28 16:30:00 UTC   \n",
       "6      4675e2b2038d8f9f67bb9217ca824d0727b629e1  2017-01-24 18:30:00 UTC   \n",
       "...                                         ...                      ...   \n",
       "39093  89d1333ab2a46ce0b691b9b804c9dd61e64748e1  2017-01-27 14:50:00 UTC   \n",
       "39094  89d1333ab2a46ce0b691b9b804c9dd61e64748e1  2017-01-21 18:50:00 UTC   \n",
       "39095  89d1333ab2a46ce0b691b9b804c9dd61e64748e1  2017-01-27 16:50:00 UTC   \n",
       "39096  c1c075cfc90c4712fe95f5afb28d980c6c427631  2017-01-10 17:00:00 UTC   \n",
       "39097  c1c075cfc90c4712fe95f5afb28d980c6c427631  2017-01-10 17:05:00 UTC   \n",
       "\n",
       "          HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "1             heat          hold               712                      737   \n",
       "2             auto          auto               709                      762   \n",
       "3             heat          hold               648                      655   \n",
       "5             heat          hold               673                      673   \n",
       "6      auxHeatOnly          hold               709                      719   \n",
       "...            ...           ...               ...                      ...   \n",
       "39093         auto          auto               701                      760   \n",
       "39094         auto          auto               714                      760   \n",
       "39095         auto          auto               697                      760   \n",
       "39096         heat          auto               712                      760   \n",
       "39097         heat          auto               710                      760   \n",
       "\n",
       "       TemperatureExpectedHeat ProvinceState     City  Age_of_Home__years_  \\\n",
       "1                          709            NE  Lincoln                   25   \n",
       "2                          692            NE  Lincoln                   10   \n",
       "3                          652            NE  Hickman                    0   \n",
       "5                          663            NE  Kearney                   35   \n",
       "6                          719            NE    Omaha                   56   \n",
       "...                        ...           ...      ...                  ...   \n",
       "39093                      700            NE  Lincoln                   35   \n",
       "39094                      700            NE  Lincoln                   35   \n",
       "39095                      700            NE  Lincoln                   35   \n",
       "39096                      710            NE  Lincoln                   35   \n",
       "39097                      710            NE  Lincoln                   35   \n",
       "\n",
       "       allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "1                 False         False            False   \n",
       "2                 False         False             True   \n",
       "3                  True         False             True   \n",
       "5                 False         False            False   \n",
       "6                 False         False             True   \n",
       "...                 ...           ...              ...   \n",
       "39093             False         False            False   \n",
       "39094             False         False            False   \n",
       "39095             False         False            False   \n",
       "39096             False         False            False   \n",
       "39097             False         False            False   \n",
       "\n",
       "      Auxilliary_Heat_Fuel_Type  \n",
       "1                           Gas  \n",
       "2                      Electric  \n",
       "3                      Electric  \n",
       "5                           Gas  \n",
       "6                      Electric  \n",
       "...                         ...  \n",
       "39093                       Gas  \n",
       "39094                       Gas  \n",
       "39095                       Gas  \n",
       "39096                       Gas  \n",
       "39097                       Gas  \n",
       "\n",
       "[37096 rows x 14 columns]"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "jan_2017.drop(jan_2017[jan_2017['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "jan_2017.drop(jan_2017[jan_2017['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "jan_2017.drop(jan_2017[jan_2017['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "jan_2017.drop(jan_2017[jan_2017['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "jan_2017"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "jan_2017[\"Year\"] = \"2017\"\n",
    "jan_2017[\"Month\"] = \"Jan\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "jan_2017 = jan_2017.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>Temperature_ctrl_ave</th>\n",
       "      <th>TemperatureExpectedCool_ave</th>\n",
       "      <th>TemperatureExpectedHeat_ave</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Identifier</th>\n",
       "      <th>Month</th>\n",
       "      <th>Year</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>City</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">05320d509a18b2d4022029409e7a267736e78e0b</th>\n",
       "      <th rowspan=\"2\" valign=\"top\">Jan</th>\n",
       "      <th rowspan=\"2\" valign=\"top\">2017</th>\n",
       "      <th rowspan=\"2\" valign=\"top\">heat</th>\n",
       "      <th>auto</th>\n",
       "      <th>Lincoln</th>\n",
       "      <td>702.837209</td>\n",
       "      <td>696.116279</td>\n",
       "      <td>694.720930</td>\n",
       "      <td>90.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>hold</th>\n",
       "      <th>Lincoln</th>\n",
       "      <td>681.258824</td>\n",
       "      <td>656.300000</td>\n",
       "      <td>652.817647</td>\n",
       "      <td>90.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">0d779ed5bbc93a32506824e72f1b051e9980a0cf</th>\n",
       "      <th rowspan=\"2\" valign=\"top\">Jan</th>\n",
       "      <th rowspan=\"2\" valign=\"top\">2017</th>\n",
       "      <th rowspan=\"2\" valign=\"top\">heat</th>\n",
       "      <th>auto</th>\n",
       "      <th>Blair</th>\n",
       "      <td>697.016129</td>\n",
       "      <td>708.387097</td>\n",
       "      <td>707.508065</td>\n",
       "      <td>35.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>hold</th>\n",
       "      <th>Blair</th>\n",
       "      <td>697.666667</td>\n",
       "      <td>703.833333</td>\n",
       "      <td>703.833333</td>\n",
       "      <td>35.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>139fad3ed64a1a20e1def36f6af543b56e705d97</th>\n",
       "      <th>Jan</th>\n",
       "      <th>2017</th>\n",
       "      <th>heat</th>\n",
       "      <th>hold</th>\n",
       "      <th>Papillion</th>\n",
       "      <td>659.891892</td>\n",
       "      <td>660.312500</td>\n",
       "      <td>659.981419</td>\n",
       "      <td>5.0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <th>...</th>\n",
       "      <th>...</th>\n",
       "      <th>...</th>\n",
       "      <th>...</th>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">fa41ecba2896263b0616f05220abba8af9cd378d</th>\n",
       "      <th rowspan=\"2\" valign=\"top\">Jan</th>\n",
       "      <th rowspan=\"2\" valign=\"top\">2017</th>\n",
       "      <th rowspan=\"2\" valign=\"top\">heat</th>\n",
       "      <th>auto</th>\n",
       "      <th>Geneva</th>\n",
       "      <td>717.517241</td>\n",
       "      <td>720.000000</td>\n",
       "      <td>720.000000</td>\n",
       "      <td>100.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>hold</th>\n",
       "      <th>Geneva</th>\n",
       "      <td>709.848315</td>\n",
       "      <td>712.758427</td>\n",
       "      <td>711.342697</td>\n",
       "      <td>100.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>faa85f28edb3a9a38ac6130d63b1c3f892466036</th>\n",
       "      <th>Jan</th>\n",
       "      <th>2017</th>\n",
       "      <th>heat</th>\n",
       "      <th>hold</th>\n",
       "      <th>Omaha</th>\n",
       "      <td>697.808824</td>\n",
       "      <td>700.000000</td>\n",
       "      <td>690.000000</td>\n",
       "      <td>55.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">fc9679ea396d94ad5b502ffedbca140b30ec008d</th>\n",
       "      <th rowspan=\"2\" valign=\"top\">Jan</th>\n",
       "      <th rowspan=\"2\" valign=\"top\">2017</th>\n",
       "      <th rowspan=\"2\" valign=\"top\">heat</th>\n",
       "      <th>auto</th>\n",
       "      <th>Bennington</th>\n",
       "      <td>715.714286</td>\n",
       "      <td>719.857143</td>\n",
       "      <td>719.357143</td>\n",
       "      <td>10.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>hold</th>\n",
       "      <th>Bennington</th>\n",
       "      <td>701.875000</td>\n",
       "      <td>705.208333</td>\n",
       "      <td>704.791667</td>\n",
       "      <td>10.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>122 rows × 7 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                                                       Temperature_ctrl_ave  \\\n",
       "Identifier                               Month Year HvacMode CalendarEvent City                               \n",
       "05320d509a18b2d4022029409e7a267736e78e0b Jan   2017 heat     auto          Lincoln               702.837209   \n",
       "                                                             hold          Lincoln               681.258824   \n",
       "0d779ed5bbc93a32506824e72f1b051e9980a0cf Jan   2017 heat     auto          Blair                 697.016129   \n",
       "                                                             hold          Blair                 697.666667   \n",
       "139fad3ed64a1a20e1def36f6af543b56e705d97 Jan   2017 heat     hold          Papillion             659.891892   \n",
       "...                                                                                                     ...   \n",
       "fa41ecba2896263b0616f05220abba8af9cd378d Jan   2017 heat     auto          Geneva                717.517241   \n",
       "                                                             hold          Geneva                709.848315   \n",
       "faa85f28edb3a9a38ac6130d63b1c3f892466036 Jan   2017 heat     hold          Omaha                 697.808824   \n",
       "fc9679ea396d94ad5b502ffedbca140b30ec008d Jan   2017 heat     auto          Bennington            715.714286   \n",
       "                                                             hold          Bennington            701.875000   \n",
       "\n",
       "                                                                                       TemperatureExpectedCool_ave  \\\n",
       "Identifier                               Month Year HvacMode CalendarEvent City                                      \n",
       "05320d509a18b2d4022029409e7a267736e78e0b Jan   2017 heat     auto          Lincoln                      696.116279   \n",
       "                                                             hold          Lincoln                      656.300000   \n",
       "0d779ed5bbc93a32506824e72f1b051e9980a0cf Jan   2017 heat     auto          Blair                        708.387097   \n",
       "                                                             hold          Blair                        703.833333   \n",
       "139fad3ed64a1a20e1def36f6af543b56e705d97 Jan   2017 heat     hold          Papillion                    660.312500   \n",
       "...                                                                                                            ...   \n",
       "fa41ecba2896263b0616f05220abba8af9cd378d Jan   2017 heat     auto          Geneva                       720.000000   \n",
       "                                                             hold          Geneva                       712.758427   \n",
       "faa85f28edb3a9a38ac6130d63b1c3f892466036 Jan   2017 heat     hold          Omaha                        700.000000   \n",
       "fc9679ea396d94ad5b502ffedbca140b30ec008d Jan   2017 heat     auto          Bennington                   719.857143   \n",
       "                                                             hold          Bennington                   705.208333   \n",
       "\n",
       "                                                                                       TemperatureExpectedHeat_ave  \\\n",
       "Identifier                               Month Year HvacMode CalendarEvent City                                      \n",
       "05320d509a18b2d4022029409e7a267736e78e0b Jan   2017 heat     auto          Lincoln                      694.720930   \n",
       "                                                             hold          Lincoln                      652.817647   \n",
       "0d779ed5bbc93a32506824e72f1b051e9980a0cf Jan   2017 heat     auto          Blair                        707.508065   \n",
       "                                                             hold          Blair                        703.833333   \n",
       "139fad3ed64a1a20e1def36f6af543b56e705d97 Jan   2017 heat     hold          Papillion                    659.981419   \n",
       "...                                                                                                            ...   \n",
       "fa41ecba2896263b0616f05220abba8af9cd378d Jan   2017 heat     auto          Geneva                       720.000000   \n",
       "                                                             hold          Geneva                       711.342697   \n",
       "faa85f28edb3a9a38ac6130d63b1c3f892466036 Jan   2017 heat     hold          Omaha                        690.000000   \n",
       "fc9679ea396d94ad5b502ffedbca140b30ec008d Jan   2017 heat     auto          Bennington                   719.357143   \n",
       "                                                             hold          Bennington                   704.791667   \n",
       "\n",
       "                                                                                       Age_of_Home__years_  \\\n",
       "Identifier                               Month Year HvacMode CalendarEvent City                              \n",
       "05320d509a18b2d4022029409e7a267736e78e0b Jan   2017 heat     auto          Lincoln                    90.0   \n",
       "                                                             hold          Lincoln                    90.0   \n",
       "0d779ed5bbc93a32506824e72f1b051e9980a0cf Jan   2017 heat     auto          Blair                      35.0   \n",
       "                                                             hold          Blair                      35.0   \n",
       "139fad3ed64a1a20e1def36f6af543b56e705d97 Jan   2017 heat     hold          Papillion                   5.0   \n",
       "...                                                                                                    ...   \n",
       "fa41ecba2896263b0616f05220abba8af9cd378d Jan   2017 heat     auto          Geneva                    100.0   \n",
       "                                                             hold          Geneva                    100.0   \n",
       "faa85f28edb3a9a38ac6130d63b1c3f892466036 Jan   2017 heat     hold          Omaha                      55.0   \n",
       "fc9679ea396d94ad5b502ffedbca140b30ec008d Jan   2017 heat     auto          Bennington                 10.0   \n",
       "                                                             hold          Bennington                 10.0   \n",
       "\n",
       "                                                                                       allowCompWithAux  \\\n",
       "Identifier                               Month Year HvacMode CalendarEvent City                           \n",
       "05320d509a18b2d4022029409e7a267736e78e0b Jan   2017 heat     auto          Lincoln                False   \n",
       "                                                             hold          Lincoln                False   \n",
       "0d779ed5bbc93a32506824e72f1b051e9980a0cf Jan   2017 heat     auto          Blair                  False   \n",
       "                                                             hold          Blair                  False   \n",
       "139fad3ed64a1a20e1def36f6af543b56e705d97 Jan   2017 heat     hold          Papillion               True   \n",
       "...                                                                                                 ...   \n",
       "fa41ecba2896263b0616f05220abba8af9cd378d Jan   2017 heat     auto          Geneva                 False   \n",
       "                                                             hold          Geneva                 False   \n",
       "faa85f28edb3a9a38ac6130d63b1c3f892466036 Jan   2017 heat     hold          Omaha                  False   \n",
       "fc9679ea396d94ad5b502ffedbca140b30ec008d Jan   2017 heat     auto          Bennington             False   \n",
       "                                                             hold          Bennington             False   \n",
       "\n",
       "                                                                                       Has_Electric  \\\n",
       "Identifier                               Month Year HvacMode CalendarEvent City                       \n",
       "05320d509a18b2d4022029409e7a267736e78e0b Jan   2017 heat     auto          Lincoln            False   \n",
       "                                                             hold          Lincoln            False   \n",
       "0d779ed5bbc93a32506824e72f1b051e9980a0cf Jan   2017 heat     auto          Blair              False   \n",
       "                                                             hold          Blair              False   \n",
       "139fad3ed64a1a20e1def36f6af543b56e705d97 Jan   2017 heat     hold          Papillion          False   \n",
       "...                                                                                             ...   \n",
       "fa41ecba2896263b0616f05220abba8af9cd378d Jan   2017 heat     auto          Geneva             False   \n",
       "                                                             hold          Geneva             False   \n",
       "faa85f28edb3a9a38ac6130d63b1c3f892466036 Jan   2017 heat     hold          Omaha              False   \n",
       "fc9679ea396d94ad5b502ffedbca140b30ec008d Jan   2017 heat     auto          Bennington         False   \n",
       "                                                             hold          Bennington         False   \n",
       "\n",
       "                                                                                       Has_a_Heat_Pump  \n",
       "Identifier                               Month Year HvacMode CalendarEvent City                         \n",
       "05320d509a18b2d4022029409e7a267736e78e0b Jan   2017 heat     auto          Lincoln               False  \n",
       "                                                             hold          Lincoln               False  \n",
       "0d779ed5bbc93a32506824e72f1b051e9980a0cf Jan   2017 heat     auto          Blair                 False  \n",
       "                                                             hold          Blair                 False  \n",
       "139fad3ed64a1a20e1def36f6af543b56e705d97 Jan   2017 heat     hold          Papillion             False  \n",
       "...                                                                                                ...  \n",
       "fa41ecba2896263b0616f05220abba8af9cd378d Jan   2017 heat     auto          Geneva                False  \n",
       "                                                             hold          Geneva                False  \n",
       "faa85f28edb3a9a38ac6130d63b1c3f892466036 Jan   2017 heat     hold          Omaha                 False  \n",
       "fc9679ea396d94ad5b502ffedbca140b30ec008d Jan   2017 heat     auto          Bennington            False  \n",
       "                                                             hold          Bennington            False  \n",
       "\n",
       "[122 rows x 7 columns]"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "jan_2017_ave = jan_2017.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "jan_2017_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "jan_2017_ave.to_csv(\"data/day/NE/jan/jan_2017_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2018 January Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "jan_2018 = pd.read_csv(\"../data_large/NE-day/2018-jan-day-NE.csv\")\n",
    "\n",
    "# jan_2018"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0f5333f7a040e9e6da0179fda8302f89aefcbcbd</td>\n",
       "      <td>2018-01-20 15:05:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>677</td>\n",
       "      <td>787</td>\n",
       "      <td>642</td>\n",
       "      <td>NE</td>\n",
       "      <td>Lincoln</td>\n",
       "      <td>50</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1985472cdc3f8d3bf064a68f444737c05c724ea8</td>\n",
       "      <td>2018-01-05 19:10:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>710</td>\n",
       "      <td>785</td>\n",
       "      <td>715</td>\n",
       "      <td>NE</td>\n",
       "      <td>Omaha</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1985472cdc3f8d3bf064a68f444737c05c724ea8</td>\n",
       "      <td>2018-01-11 17:35:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>700</td>\n",
       "      <td>785</td>\n",
       "      <td>705</td>\n",
       "      <td>NE</td>\n",
       "      <td>Omaha</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>6c71018df5d7d3ec67c41d961ba0a0bbdd3a9e19</td>\n",
       "      <td>2018-01-06 14:20:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>664</td>\n",
       "      <td>683</td>\n",
       "      <td>683</td>\n",
       "      <td>NE</td>\n",
       "      <td>Lincoln</td>\n",
       "      <td>25</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>1985472cdc3f8d3bf064a68f444737c05c724ea8</td>\n",
       "      <td>2018-01-09 16:50:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>722</td>\n",
       "      <td>775</td>\n",
       "      <td>725</td>\n",
       "      <td>NE</td>\n",
       "      <td>Omaha</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>123067</th>\n",
       "      <td>ff8d6679e9ce5c1228cf640311c0110b3aafb5bd</td>\n",
       "      <td>2018-01-20 12:45:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>718</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>NE</td>\n",
       "      <td>Omaha</td>\n",
       "      <td>0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>123068</th>\n",
       "      <td>ff8d6679e9ce5c1228cf640311c0110b3aafb5bd</td>\n",
       "      <td>2018-01-10 16:50:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>755</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>NE</td>\n",
       "      <td>Omaha</td>\n",
       "      <td>0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>123069</th>\n",
       "      <td>ff8d6679e9ce5c1228cf640311c0110b3aafb5bd</td>\n",
       "      <td>2018-01-10 19:00:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>758</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>NE</td>\n",
       "      <td>Omaha</td>\n",
       "      <td>0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>123070</th>\n",
       "      <td>ff8d6679e9ce5c1228cf640311c0110b3aafb5bd</td>\n",
       "      <td>2018-01-10 16:35:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>755</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>NE</td>\n",
       "      <td>Omaha</td>\n",
       "      <td>0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>123071</th>\n",
       "      <td>ff8d6679e9ce5c1228cf640311c0110b3aafb5bd</td>\n",
       "      <td>2018-01-20 16:30:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>766</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>NE</td>\n",
       "      <td>Omaha</td>\n",
       "      <td>0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>120023 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                      Identifier                date_time  \\\n",
       "0       0f5333f7a040e9e6da0179fda8302f89aefcbcbd  2018-01-20 15:05:00 UTC   \n",
       "1       1985472cdc3f8d3bf064a68f444737c05c724ea8  2018-01-05 19:10:00 UTC   \n",
       "3       1985472cdc3f8d3bf064a68f444737c05c724ea8  2018-01-11 17:35:00 UTC   \n",
       "4       6c71018df5d7d3ec67c41d961ba0a0bbdd3a9e19  2018-01-06 14:20:00 UTC   \n",
       "5       1985472cdc3f8d3bf064a68f444737c05c724ea8  2018-01-09 16:50:00 UTC   \n",
       "...                                          ...                      ...   \n",
       "123067  ff8d6679e9ce5c1228cf640311c0110b3aafb5bd  2018-01-20 12:45:00 UTC   \n",
       "123068  ff8d6679e9ce5c1228cf640311c0110b3aafb5bd  2018-01-10 16:50:00 UTC   \n",
       "123069  ff8d6679e9ce5c1228cf640311c0110b3aafb5bd  2018-01-10 19:00:00 UTC   \n",
       "123070  ff8d6679e9ce5c1228cf640311c0110b3aafb5bd  2018-01-10 16:35:00 UTC   \n",
       "123071  ff8d6679e9ce5c1228cf640311c0110b3aafb5bd  2018-01-20 16:30:00 UTC   \n",
       "\n",
       "       HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0          heat          auto               677                      787   \n",
       "1          auto          hold               710                      785   \n",
       "3          auto          hold               700                      785   \n",
       "4          heat          hold               664                      683   \n",
       "5          auto          hold               722                      775   \n",
       "...         ...           ...               ...                      ...   \n",
       "123067     heat          hold               718                      760   \n",
       "123068     heat          hold               755                      760   \n",
       "123069     heat          hold               758                      760   \n",
       "123070     heat          hold               755                      760   \n",
       "123071     heat          hold               766                      760   \n",
       "\n",
       "        TemperatureExpectedHeat ProvinceState     City  Age_of_Home__years_  \\\n",
       "0                           642            NE  Lincoln                   50   \n",
       "1                           715            NE    Omaha                   10   \n",
       "3                           705            NE    Omaha                   10   \n",
       "4                           683            NE  Lincoln                   25   \n",
       "5                           725            NE    Omaha                   10   \n",
       "...                         ...           ...      ...                  ...   \n",
       "123067                      760            NE    Omaha                    0   \n",
       "123068                      760            NE    Omaha                    0   \n",
       "123069                      760            NE    Omaha                    0   \n",
       "123070                      760            NE    Omaha                    0   \n",
       "123071                      760            NE    Omaha                    0   \n",
       "\n",
       "        allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                  False         False            False   \n",
       "1                  False         False            False   \n",
       "3                  False         False            False   \n",
       "4                  False         False             True   \n",
       "5                  False         False            False   \n",
       "...                  ...           ...              ...   \n",
       "123067              True         False             True   \n",
       "123068              True         False             True   \n",
       "123069              True         False             True   \n",
       "123070              True         False             True   \n",
       "123071              True         False             True   \n",
       "\n",
       "       Auxilliary_Heat_Fuel_Type  \n",
       "0                            Gas  \n",
       "1                            Gas  \n",
       "3                            Gas  \n",
       "4                       Electric  \n",
       "5                            Gas  \n",
       "...                          ...  \n",
       "123067                  Electric  \n",
       "123068                  Electric  \n",
       "123069                  Electric  \n",
       "123070                  Electric  \n",
       "123071                  Electric  \n",
       "\n",
       "[120023 rows x 14 columns]"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "jan_2018.drop(jan_2018[jan_2018['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "jan_2018.drop(jan_2018[jan_2018['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "jan_2018.drop(jan_2018[jan_2018['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "jan_2018.drop(jan_2018[jan_2018['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "jan_2018"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "jan_2018[\"Year\"] = \"2018\"\n",
    "jan_2018[\"Month\"] = \"Jan\"\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "jan_2018 = jan_2018.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "jan_2018_ave = jan_2018.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# jan_2018_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "jan_2018_ave.to_csv(\"data/day/NE/jan/jan_2018_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2019 January Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "jan_2019 = pd.read_csv(\"../data_large/NE-day/2019-jan-day-NE.csv\")\n",
    "\n",
    "# jan_2019"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>02674609e156f6807ee78e1c9ac2851c16854804</td>\n",
       "      <td>2019-01-10 19:20:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>675</td>\n",
       "      <td>685</td>\n",
       "      <td>685</td>\n",
       "      <td>NE</td>\n",
       "      <td>Omaha</td>\n",
       "      <td>50</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>d78e9843002347d8166cbb8d4dd6b7907605bf29</td>\n",
       "      <td>2019-01-30 19:40:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>719</td>\n",
       "      <td>721</td>\n",
       "      <td>721</td>\n",
       "      <td>NE</td>\n",
       "      <td>Lincoln</td>\n",
       "      <td>25</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>02674609e156f6807ee78e1c9ac2851c16854804</td>\n",
       "      <td>2019-01-09 17:25:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>684</td>\n",
       "      <td>685</td>\n",
       "      <td>685</td>\n",
       "      <td>NE</td>\n",
       "      <td>Omaha</td>\n",
       "      <td>50</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>c039f7f15fbd5d98b80efba2631acb48bef7d066</td>\n",
       "      <td>2019-01-05 14:45:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>712</td>\n",
       "      <td>775</td>\n",
       "      <td>715</td>\n",
       "      <td>NE</td>\n",
       "      <td>Bennington</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>3b46790ec167583ec99f9fcc4d1aa248ecd4fa77</td>\n",
       "      <td>2019-01-17 14:20:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>716</td>\n",
       "      <td>727</td>\n",
       "      <td>720</td>\n",
       "      <td>NE</td>\n",
       "      <td>Gretna</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>205298</th>\n",
       "      <td>f15f623c199dc4f547c63da94db6260301a8cc81</td>\n",
       "      <td>2019-01-03 19:45:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>686</td>\n",
       "      <td>760</td>\n",
       "      <td>690</td>\n",
       "      <td>NE</td>\n",
       "      <td>Springfield</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>205299</th>\n",
       "      <td>f15f623c199dc4f547c63da94db6260301a8cc81</td>\n",
       "      <td>2019-01-03 19:55:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>684</td>\n",
       "      <td>760</td>\n",
       "      <td>690</td>\n",
       "      <td>NE</td>\n",
       "      <td>Springfield</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>205300</th>\n",
       "      <td>f15f623c199dc4f547c63da94db6260301a8cc81</td>\n",
       "      <td>2019-01-06 16:50:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>688</td>\n",
       "      <td>760</td>\n",
       "      <td>690</td>\n",
       "      <td>NE</td>\n",
       "      <td>Springfield</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>205301</th>\n",
       "      <td>f15f623c199dc4f547c63da94db6260301a8cc81</td>\n",
       "      <td>2019-01-04 19:45:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>686</td>\n",
       "      <td>760</td>\n",
       "      <td>690</td>\n",
       "      <td>NE</td>\n",
       "      <td>Springfield</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>205302</th>\n",
       "      <td>f15f623c199dc4f547c63da94db6260301a8cc81</td>\n",
       "      <td>2019-01-03 17:15:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>687</td>\n",
       "      <td>760</td>\n",
       "      <td>690</td>\n",
       "      <td>NE</td>\n",
       "      <td>Springfield</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>202560 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                      Identifier                date_time  \\\n",
       "0       02674609e156f6807ee78e1c9ac2851c16854804  2019-01-10 19:20:00 UTC   \n",
       "1       d78e9843002347d8166cbb8d4dd6b7907605bf29  2019-01-30 19:40:00 UTC   \n",
       "2       02674609e156f6807ee78e1c9ac2851c16854804  2019-01-09 17:25:00 UTC   \n",
       "3       c039f7f15fbd5d98b80efba2631acb48bef7d066  2019-01-05 14:45:00 UTC   \n",
       "4       3b46790ec167583ec99f9fcc4d1aa248ecd4fa77  2019-01-17 14:20:00 UTC   \n",
       "...                                          ...                      ...   \n",
       "205298  f15f623c199dc4f547c63da94db6260301a8cc81  2019-01-03 19:45:00 UTC   \n",
       "205299  f15f623c199dc4f547c63da94db6260301a8cc81  2019-01-03 19:55:00 UTC   \n",
       "205300  f15f623c199dc4f547c63da94db6260301a8cc81  2019-01-06 16:50:00 UTC   \n",
       "205301  f15f623c199dc4f547c63da94db6260301a8cc81  2019-01-04 19:45:00 UTC   \n",
       "205302  f15f623c199dc4f547c63da94db6260301a8cc81  2019-01-03 17:15:00 UTC   \n",
       "\n",
       "       HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0          heat          hold               675                      685   \n",
       "1          heat          hold               719                      721   \n",
       "2          heat          hold               684                      685   \n",
       "3          auto          hold               712                      775   \n",
       "4          heat          auto               716                      727   \n",
       "...         ...           ...               ...                      ...   \n",
       "205298     heat          auto               686                      760   \n",
       "205299     heat          auto               684                      760   \n",
       "205300     heat          auto               688                      760   \n",
       "205301     heat          auto               686                      760   \n",
       "205302     heat          auto               687                      760   \n",
       "\n",
       "        TemperatureExpectedHeat ProvinceState         City  \\\n",
       "0                           685            NE        Omaha   \n",
       "1                           721            NE      Lincoln   \n",
       "2                           685            NE        Omaha   \n",
       "3                           715            NE   Bennington   \n",
       "4                           720            NE       Gretna   \n",
       "...                         ...           ...          ...   \n",
       "205298                      690            NE  Springfield   \n",
       "205299                      690            NE  Springfield   \n",
       "205300                      690            NE  Springfield   \n",
       "205301                      690            NE  Springfield   \n",
       "205302                      690            NE  Springfield   \n",
       "\n",
       "        Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                        50             False         False             True   \n",
       "1                        25             False         False            False   \n",
       "2                        50             False         False             True   \n",
       "3                         0             False         False            False   \n",
       "4                         0             False         False            False   \n",
       "...                     ...               ...           ...              ...   \n",
       "205298                    0             False         False             True   \n",
       "205299                    0             False         False             True   \n",
       "205300                    0             False         False             True   \n",
       "205301                    0             False         False             True   \n",
       "205302                    0             False         False             True   \n",
       "\n",
       "       Auxilliary_Heat_Fuel_Type  \n",
       "0                       Electric  \n",
       "1                            Gas  \n",
       "2                       Electric  \n",
       "3                            Gas  \n",
       "4                            Gas  \n",
       "...                          ...  \n",
       "205298                  Electric  \n",
       "205299                  Electric  \n",
       "205300                  Electric  \n",
       "205301                  Electric  \n",
       "205302                  Electric  \n",
       "\n",
       "[202560 rows x 14 columns]"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "jan_2019.drop(jan_2019[jan_2019['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "jan_2019.drop(jan_2019[jan_2019['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "jan_2019.drop(jan_2019[jan_2019['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "jan_2019.drop(jan_2019[jan_2019['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "jan_2019"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "jan_2019[\"Year\"] = \"2019\"\n",
    "jan_2019[\"Month\"] = \"Jan\"\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "jan_2019 = jan_2019.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "jan_2019_ave = jan_2019.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# jan_2019_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "jan_2019_ave.to_csv(\"data/day/NE/jan/jan_2019_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2020 January Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "jan_2020 = pd.read_csv(\"../data_large/NE-day/2020-jan-day-NE.csv\")\n",
    "\n",
    "# jan_2020"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>c039f7f15fbd5d98b80efba2631acb48bef7d066</td>\n",
       "      <td>2020-01-08 13:35:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>699</td>\n",
       "      <td>765</td>\n",
       "      <td>705</td>\n",
       "      <td>NE</td>\n",
       "      <td>Bennington</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>197f6257e48f1d14dffab296917ddf2c3090264b</td>\n",
       "      <td>2020-01-13 16:20:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>702</td>\n",
       "      <td>765</td>\n",
       "      <td>715</td>\n",
       "      <td>NE</td>\n",
       "      <td>Omaha</td>\n",
       "      <td>15</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>c039f7f15fbd5d98b80efba2631acb48bef7d066</td>\n",
       "      <td>2020-01-02 18:25:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>704</td>\n",
       "      <td>765</td>\n",
       "      <td>705</td>\n",
       "      <td>NE</td>\n",
       "      <td>Bennington</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>670cb331c3d05d1b746e7f968c11986ee3a15a93</td>\n",
       "      <td>2020-01-18 10:35:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>717</td>\n",
       "      <td>719</td>\n",
       "      <td>719</td>\n",
       "      <td>NE</td>\n",
       "      <td>Omaha</td>\n",
       "      <td>40</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>124aefa02526f136375928b0c92df70f356d26b9</td>\n",
       "      <td>2020-01-18 13:35:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>663</td>\n",
       "      <td>675</td>\n",
       "      <td>675</td>\n",
       "      <td>NE</td>\n",
       "      <td>Omaha</td>\n",
       "      <td>50</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>217318</th>\n",
       "      <td>e50f2e1393d5251e0586c8b339bca1f26e3430e6</td>\n",
       "      <td>2020-01-01 17:45:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>698</td>\n",
       "      <td>760</td>\n",
       "      <td>700</td>\n",
       "      <td>NE</td>\n",
       "      <td>Lincoln</td>\n",
       "      <td>27</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>217319</th>\n",
       "      <td>e50f2e1393d5251e0586c8b339bca1f26e3430e6</td>\n",
       "      <td>2020-01-01 13:55:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>698</td>\n",
       "      <td>760</td>\n",
       "      <td>700</td>\n",
       "      <td>NE</td>\n",
       "      <td>Lincoln</td>\n",
       "      <td>27</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>217320</th>\n",
       "      <td>e50f2e1393d5251e0586c8b339bca1f26e3430e6</td>\n",
       "      <td>2020-01-02 15:10:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>700</td>\n",
       "      <td>760</td>\n",
       "      <td>700</td>\n",
       "      <td>NE</td>\n",
       "      <td>Lincoln</td>\n",
       "      <td>27</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>217321</th>\n",
       "      <td>e50f2e1393d5251e0586c8b339bca1f26e3430e6</td>\n",
       "      <td>2020-01-07 16:05:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>694</td>\n",
       "      <td>760</td>\n",
       "      <td>700</td>\n",
       "      <td>NE</td>\n",
       "      <td>Lincoln</td>\n",
       "      <td>27</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>217322</th>\n",
       "      <td>e50f2e1393d5251e0586c8b339bca1f26e3430e6</td>\n",
       "      <td>2020-01-03 15:35:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>693</td>\n",
       "      <td>760</td>\n",
       "      <td>700</td>\n",
       "      <td>NE</td>\n",
       "      <td>Lincoln</td>\n",
       "      <td>27</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>210050 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                      Identifier                date_time  \\\n",
       "0       c039f7f15fbd5d98b80efba2631acb48bef7d066  2020-01-08 13:35:00 UTC   \n",
       "1       197f6257e48f1d14dffab296917ddf2c3090264b  2020-01-13 16:20:00 UTC   \n",
       "2       c039f7f15fbd5d98b80efba2631acb48bef7d066  2020-01-02 18:25:00 UTC   \n",
       "3       670cb331c3d05d1b746e7f968c11986ee3a15a93  2020-01-18 10:35:00 UTC   \n",
       "4       124aefa02526f136375928b0c92df70f356d26b9  2020-01-18 13:35:00 UTC   \n",
       "...                                          ...                      ...   \n",
       "217318  e50f2e1393d5251e0586c8b339bca1f26e3430e6  2020-01-01 17:45:00 UTC   \n",
       "217319  e50f2e1393d5251e0586c8b339bca1f26e3430e6  2020-01-01 13:55:00 UTC   \n",
       "217320  e50f2e1393d5251e0586c8b339bca1f26e3430e6  2020-01-02 15:10:00 UTC   \n",
       "217321  e50f2e1393d5251e0586c8b339bca1f26e3430e6  2020-01-07 16:05:00 UTC   \n",
       "217322  e50f2e1393d5251e0586c8b339bca1f26e3430e6  2020-01-03 15:35:00 UTC   \n",
       "\n",
       "       HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0          auto          hold               699                      765   \n",
       "1          auto          hold               702                      765   \n",
       "2          auto          hold               704                      765   \n",
       "3          heat          hold               717                      719   \n",
       "4          heat          hold               663                      675   \n",
       "...         ...           ...               ...                      ...   \n",
       "217318     auto          hold               698                      760   \n",
       "217319     auto          hold               698                      760   \n",
       "217320     auto          hold               700                      760   \n",
       "217321     auto          hold               694                      760   \n",
       "217322     auto          hold               693                      760   \n",
       "\n",
       "        TemperatureExpectedHeat ProvinceState        City  \\\n",
       "0                           705            NE  Bennington   \n",
       "1                           715            NE       Omaha   \n",
       "2                           705            NE  Bennington   \n",
       "3                           719            NE       Omaha   \n",
       "4                           675            NE       Omaha   \n",
       "...                         ...           ...         ...   \n",
       "217318                      700            NE     Lincoln   \n",
       "217319                      700            NE     Lincoln   \n",
       "217320                      700            NE     Lincoln   \n",
       "217321                      700            NE     Lincoln   \n",
       "217322                      700            NE     Lincoln   \n",
       "\n",
       "        Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                         0             False         False            False   \n",
       "1                        15             False         False            False   \n",
       "2                         0             False         False            False   \n",
       "3                        40              True         False            False   \n",
       "4                        50              True         False             True   \n",
       "...                     ...               ...           ...              ...   \n",
       "217318                   27             False         False            False   \n",
       "217319                   27             False         False            False   \n",
       "217320                   27             False         False            False   \n",
       "217321                   27             False         False            False   \n",
       "217322                   27             False         False            False   \n",
       "\n",
       "       Auxilliary_Heat_Fuel_Type  \n",
       "0                            Gas  \n",
       "1                            Gas  \n",
       "2                            Gas  \n",
       "3                            Gas  \n",
       "4                       Electric  \n",
       "...                          ...  \n",
       "217318                       Gas  \n",
       "217319                       Gas  \n",
       "217320                       Gas  \n",
       "217321                       Gas  \n",
       "217322                       Gas  \n",
       "\n",
       "[210050 rows x 14 columns]"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "jan_2020.drop(jan_2020[jan_2020['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "jan_2020.drop(jan_2020[jan_2020['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "jan_2020.drop(jan_2020[jan_2020['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "jan_2020.drop(jan_2020[jan_2020['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "jan_2020"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "jan_2020[\"Year\"] = \"2020\"\n",
    "jan_2020[\"Month\"] = \"Jan\"\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "jan_2020 = jan_2020.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "jan_2020_ave = jan_2020.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# jan_2020_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "jan_2020_ave.to_csv(\"data/day/NE/jan/jan_2020_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2021 January Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "jan_2021 = pd.read_csv(\"../data_large/NE-day/2021-jan-day-NE.csv\")\n",
    "\n",
    "# jan_2021"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>4b62cface5b5ca77d4a85a0ce65e7a0af360e9d1</td>\n",
       "      <td>2021-01-04 19:25:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>711</td>\n",
       "      <td>722</td>\n",
       "      <td>722</td>\n",
       "      <td>NE</td>\n",
       "      <td>Omaha</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>a6f5c07aac86f2cc5e78dc0165a33a5edec68476</td>\n",
       "      <td>2021-01-11 17:15:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>705</td>\n",
       "      <td>743</td>\n",
       "      <td>700</td>\n",
       "      <td>NE</td>\n",
       "      <td>Omaha</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>ae207699741703a86828265b6556382c300adb71</td>\n",
       "      <td>2021-01-27 16:15:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>718</td>\n",
       "      <td>773</td>\n",
       "      <td>723</td>\n",
       "      <td>NE</td>\n",
       "      <td>Lincoln</td>\n",
       "      <td>95</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2504df671d8f2abfb4e209289ba629963f2db33b</td>\n",
       "      <td>2021-01-17 18:35:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>753</td>\n",
       "      <td>752</td>\n",
       "      <td>752</td>\n",
       "      <td>NE</td>\n",
       "      <td>Lincoln</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>6a4878bb967916a664ed4d25a33d2d8c92342a59</td>\n",
       "      <td>2021-01-30 13:30:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>689</td>\n",
       "      <td>696</td>\n",
       "      <td>696</td>\n",
       "      <td>NE</td>\n",
       "      <td>HAMPTON</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>136171</th>\n",
       "      <td>b89130febe4af29347ca1be608faf2350540a600</td>\n",
       "      <td>2021-01-16 15:20:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>757</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>NE</td>\n",
       "      <td>North Platte</td>\n",
       "      <td>15</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>136172</th>\n",
       "      <td>b89130febe4af29347ca1be608faf2350540a600</td>\n",
       "      <td>2021-01-16 19:10:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>761</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>NE</td>\n",
       "      <td>North Platte</td>\n",
       "      <td>15</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>136173</th>\n",
       "      <td>b89130febe4af29347ca1be608faf2350540a600</td>\n",
       "      <td>2021-01-16 19:40:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>NE</td>\n",
       "      <td>North Platte</td>\n",
       "      <td>15</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>136174</th>\n",
       "      <td>b89130febe4af29347ca1be608faf2350540a600</td>\n",
       "      <td>2021-01-16 17:15:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>757</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>NE</td>\n",
       "      <td>North Platte</td>\n",
       "      <td>15</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>136175</th>\n",
       "      <td>b89130febe4af29347ca1be608faf2350540a600</td>\n",
       "      <td>2021-01-16 18:10:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>755</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>NE</td>\n",
       "      <td>North Platte</td>\n",
       "      <td>15</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>135570 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                      Identifier                date_time  \\\n",
       "0       4b62cface5b5ca77d4a85a0ce65e7a0af360e9d1  2021-01-04 19:25:00 UTC   \n",
       "1       a6f5c07aac86f2cc5e78dc0165a33a5edec68476  2021-01-11 17:15:00 UTC   \n",
       "2       ae207699741703a86828265b6556382c300adb71  2021-01-27 16:15:00 UTC   \n",
       "3       2504df671d8f2abfb4e209289ba629963f2db33b  2021-01-17 18:35:00 UTC   \n",
       "4       6a4878bb967916a664ed4d25a33d2d8c92342a59  2021-01-30 13:30:00 UTC   \n",
       "...                                          ...                      ...   \n",
       "136171  b89130febe4af29347ca1be608faf2350540a600  2021-01-16 15:20:00 UTC   \n",
       "136172  b89130febe4af29347ca1be608faf2350540a600  2021-01-16 19:10:00 UTC   \n",
       "136173  b89130febe4af29347ca1be608faf2350540a600  2021-01-16 19:40:00 UTC   \n",
       "136174  b89130febe4af29347ca1be608faf2350540a600  2021-01-16 17:15:00 UTC   \n",
       "136175  b89130febe4af29347ca1be608faf2350540a600  2021-01-16 18:10:00 UTC   \n",
       "\n",
       "       HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0          heat          hold               711                      722   \n",
       "1          heat          hold               705                      743   \n",
       "2          auto          hold               718                      773   \n",
       "3          heat          hold               753                      752   \n",
       "4          heat          hold               689                      696   \n",
       "...         ...           ...               ...                      ...   \n",
       "136171     heat          hold               757                      760   \n",
       "136172     heat          hold               761                      760   \n",
       "136173     heat          hold               760                      760   \n",
       "136174     heat          hold               757                      760   \n",
       "136175     heat          hold               755                      760   \n",
       "\n",
       "        TemperatureExpectedHeat ProvinceState          City  \\\n",
       "0                           722            NE         Omaha   \n",
       "1                           700            NE         Omaha   \n",
       "2                           723            NE       Lincoln   \n",
       "3                           752            NE       Lincoln   \n",
       "4                           696            NE       HAMPTON   \n",
       "...                         ...           ...           ...   \n",
       "136171                      760            NE  North Platte   \n",
       "136172                      760            NE  North Platte   \n",
       "136173                      760            NE  North Platte   \n",
       "136174                      760            NE  North Platte   \n",
       "136175                      760            NE  North Platte   \n",
       "\n",
       "        Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                         0             False         False            False   \n",
       "1                        10             False         False            False   \n",
       "2                        95             False         False            False   \n",
       "3                         0             False         False            False   \n",
       "4                        10             False         False             True   \n",
       "...                     ...               ...           ...              ...   \n",
       "136171                   15              True         False            False   \n",
       "136172                   15              True         False            False   \n",
       "136173                   15              True         False            False   \n",
       "136174                   15              True         False            False   \n",
       "136175                   15              True         False            False   \n",
       "\n",
       "       Auxilliary_Heat_Fuel_Type  \n",
       "0                            Gas  \n",
       "1                            Gas  \n",
       "2                            Gas  \n",
       "3                            Gas  \n",
       "4                       Electric  \n",
       "...                          ...  \n",
       "136171                       Gas  \n",
       "136172                       Gas  \n",
       "136173                       Gas  \n",
       "136174                       Gas  \n",
       "136175                       Gas  \n",
       "\n",
       "[135570 rows x 14 columns]"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "jan_2021.drop(jan_2021[jan_2021['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "jan_2021.drop(jan_2021[jan_2021['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "jan_2021.drop(jan_2021[jan_2021['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "jan_2021.drop(jan_2021[jan_2021['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "jan_2021"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "jan_2021[\"Year\"] = \"2021\"\n",
    "jan_2021[\"Month\"] = \"Jan\"\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "jan_2021 = jan_2021.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "jan_2021_ave = jan_2021.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# jan_2021_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "jan_2021_ave.to_csv(\"data/day/NE/jan/jan_2021_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "\n",
    "### Combine month CSV Files \n",
    "1. Read in files in folders for each state\n",
    "2. Export as combined CSV"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create variable for files in directory\n",
    "files = [f for f in os.listdir(\"data/day/NE/jan/\") if f.endswith(\".csv\")]\n",
    "\n",
    "# files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>Month</th>\n",
       "      <th>Year</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>City</th>\n",
       "      <th>Temperature_ctrl_ave</th>\n",
       "      <th>TemperatureExpectedCool_ave</th>\n",
       "      <th>TemperatureExpectedHeat_ave</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>05320d509a18b2d4022029409e7a267736e78e0b</td>\n",
       "      <td>Jan</td>\n",
       "      <td>2017</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>Lincoln</td>\n",
       "      <td>702.837209</td>\n",
       "      <td>696.116279</td>\n",
       "      <td>694.720930</td>\n",
       "      <td>90.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>05320d509a18b2d4022029409e7a267736e78e0b</td>\n",
       "      <td>Jan</td>\n",
       "      <td>2017</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>Lincoln</td>\n",
       "      <td>681.258824</td>\n",
       "      <td>656.300000</td>\n",
       "      <td>652.817647</td>\n",
       "      <td>90.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0d779ed5bbc93a32506824e72f1b051e9980a0cf</td>\n",
       "      <td>Jan</td>\n",
       "      <td>2017</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>Blair</td>\n",
       "      <td>697.016129</td>\n",
       "      <td>708.387097</td>\n",
       "      <td>707.508065</td>\n",
       "      <td>35.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0d779ed5bbc93a32506824e72f1b051e9980a0cf</td>\n",
       "      <td>Jan</td>\n",
       "      <td>2017</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>Blair</td>\n",
       "      <td>697.666667</td>\n",
       "      <td>703.833333</td>\n",
       "      <td>703.833333</td>\n",
       "      <td>35.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>139fad3ed64a1a20e1def36f6af543b56e705d97</td>\n",
       "      <td>Jan</td>\n",
       "      <td>2017</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>Papillion</td>\n",
       "      <td>659.891892</td>\n",
       "      <td>660.312500</td>\n",
       "      <td>659.981419</td>\n",
       "      <td>5.0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>198</th>\n",
       "      <td>fa41ecba2896263b0616f05220abba8af9cd378d</td>\n",
       "      <td>Jan</td>\n",
       "      <td>2021</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>Geneva</td>\n",
       "      <td>704.378238</td>\n",
       "      <td>709.984456</td>\n",
       "      <td>709.253886</td>\n",
       "      <td>100.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>199</th>\n",
       "      <td>faa85f28edb3a9a38ac6130d63b1c3f892466036</td>\n",
       "      <td>Jan</td>\n",
       "      <td>2021</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>Omaha</td>\n",
       "      <td>704.473267</td>\n",
       "      <td>712.470297</td>\n",
       "      <td>705.557426</td>\n",
       "      <td>55.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>200</th>\n",
       "      <td>faa8c435b84d052676a86ba7d2d6a783d26e6b19</td>\n",
       "      <td>Jan</td>\n",
       "      <td>2021</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>Omaha</td>\n",
       "      <td>690.762376</td>\n",
       "      <td>693.009901</td>\n",
       "      <td>692.514851</td>\n",
       "      <td>20.0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>201</th>\n",
       "      <td>faf3b76e8b0193bcf0256358efb4e593b669a1c7</td>\n",
       "      <td>Jan</td>\n",
       "      <td>2021</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>Hickman</td>\n",
       "      <td>704.577465</td>\n",
       "      <td>710.084507</td>\n",
       "      <td>709.971831</td>\n",
       "      <td>0.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>202</th>\n",
       "      <td>ff8d6679e9ce5c1228cf640311c0110b3aafb5bd</td>\n",
       "      <td>Jan</td>\n",
       "      <td>2021</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>Omaha</td>\n",
       "      <td>717.617963</td>\n",
       "      <td>721.400516</td>\n",
       "      <td>721.400516</td>\n",
       "      <td>0.0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1372 rows × 13 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                   Identifier Month  Year HvacMode  \\\n",
       "0    05320d509a18b2d4022029409e7a267736e78e0b   Jan  2017     heat   \n",
       "1    05320d509a18b2d4022029409e7a267736e78e0b   Jan  2017     heat   \n",
       "2    0d779ed5bbc93a32506824e72f1b051e9980a0cf   Jan  2017     heat   \n",
       "3    0d779ed5bbc93a32506824e72f1b051e9980a0cf   Jan  2017     heat   \n",
       "4    139fad3ed64a1a20e1def36f6af543b56e705d97   Jan  2017     heat   \n",
       "..                                        ...   ...   ...      ...   \n",
       "198  fa41ecba2896263b0616f05220abba8af9cd378d   Jan  2021     heat   \n",
       "199  faa85f28edb3a9a38ac6130d63b1c3f892466036   Jan  2021     heat   \n",
       "200  faa8c435b84d052676a86ba7d2d6a783d26e6b19   Jan  2021     heat   \n",
       "201  faf3b76e8b0193bcf0256358efb4e593b669a1c7   Jan  2021     heat   \n",
       "202  ff8d6679e9ce5c1228cf640311c0110b3aafb5bd   Jan  2021     heat   \n",
       "\n",
       "    CalendarEvent       City  Temperature_ctrl_ave  \\\n",
       "0            auto    Lincoln            702.837209   \n",
       "1            hold    Lincoln            681.258824   \n",
       "2            auto      Blair            697.016129   \n",
       "3            hold      Blair            697.666667   \n",
       "4            hold  Papillion            659.891892   \n",
       "..            ...        ...                   ...   \n",
       "198          hold     Geneva            704.378238   \n",
       "199          hold      Omaha            704.473267   \n",
       "200          hold      Omaha            690.762376   \n",
       "201          hold    Hickman            704.577465   \n",
       "202          hold      Omaha            717.617963   \n",
       "\n",
       "     TemperatureExpectedCool_ave  TemperatureExpectedHeat_ave  \\\n",
       "0                     696.116279                   694.720930   \n",
       "1                     656.300000                   652.817647   \n",
       "2                     708.387097                   707.508065   \n",
       "3                     703.833333                   703.833333   \n",
       "4                     660.312500                   659.981419   \n",
       "..                           ...                          ...   \n",
       "198                   709.984456                   709.253886   \n",
       "199                   712.470297                   705.557426   \n",
       "200                   693.009901                   692.514851   \n",
       "201                   710.084507                   709.971831   \n",
       "202                   721.400516                   721.400516   \n",
       "\n",
       "     Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \n",
       "0                   90.0             False         False            False  \n",
       "1                   90.0             False         False            False  \n",
       "2                   35.0             False         False            False  \n",
       "3                   35.0             False         False            False  \n",
       "4                    5.0              True         False            False  \n",
       "..                   ...               ...           ...              ...  \n",
       "198                100.0             False         False            False  \n",
       "199                 55.0             False         False            False  \n",
       "200                 20.0              True         False            False  \n",
       "201                  0.0             False         False             True  \n",
       "202                  0.0              True         False             True  \n",
       "\n",
       "[1372 rows x 13 columns]"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# https://stackoverflow.com/questions/63886787/how-to-create-a-dataframe-from-multiple-csv-files\n",
    "# Initialize an empty dataframe\n",
    "NE_jan = pd.DataFrame()\n",
    "\n",
    "# Iterate through files and contents, then concatenate the data into the dataframe\n",
    "for file in files:\n",
    "    df = pd.read_csv(\"data/day/NE/jan/\" + file)\n",
    "    NE_jan = pd.concat([NE_jan, df])\n",
    "    \n",
    "NE_jan"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "NE_jan.to_csv(\"Scraper_Output/State_Month_Day/NE/NE_jan.csv\", header=True, index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "\n",
    "## February"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2017 February Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "feb_2017 = pd.read_csv(\"../data_large/NE-day/2017-feb-day-NE.csv\")\n",
    "\n",
    "# feb_2017"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>bbcd2609414c9fbb9e754b8f0f1939cb2a3c47aa</td>\n",
       "      <td>2017-02-10 18:20:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>737</td>\n",
       "      <td>765</td>\n",
       "      <td>735</td>\n",
       "      <td>NE</td>\n",
       "      <td>Lincoln</td>\n",
       "      <td>10</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>bbcd2609414c9fbb9e754b8f0f1939cb2a3c47aa</td>\n",
       "      <td>2017-02-10 14:45:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>722</td>\n",
       "      <td>765</td>\n",
       "      <td>735</td>\n",
       "      <td>NE</td>\n",
       "      <td>Lincoln</td>\n",
       "      <td>10</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>7cfa84db6a98d9989c00390bf61914b066cab4b8</td>\n",
       "      <td>2017-02-19 13:45:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>700</td>\n",
       "      <td>685</td>\n",
       "      <td>685</td>\n",
       "      <td>NE</td>\n",
       "      <td>Omaha</td>\n",
       "      <td>50</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>bbcd2609414c9fbb9e754b8f0f1939cb2a3c47aa</td>\n",
       "      <td>2017-02-14 14:10:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>755</td>\n",
       "      <td>775</td>\n",
       "      <td>755</td>\n",
       "      <td>NE</td>\n",
       "      <td>Lincoln</td>\n",
       "      <td>10</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>bbcd2609414c9fbb9e754b8f0f1939cb2a3c47aa</td>\n",
       "      <td>2017-02-10 14:15:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>721</td>\n",
       "      <td>765</td>\n",
       "      <td>735</td>\n",
       "      <td>NE</td>\n",
       "      <td>Lincoln</td>\n",
       "      <td>10</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>43111</th>\n",
       "      <td>89d1333ab2a46ce0b691b9b804c9dd61e64748e1</td>\n",
       "      <td>2017-02-09 17:55:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>696</td>\n",
       "      <td>760</td>\n",
       "      <td>700</td>\n",
       "      <td>NE</td>\n",
       "      <td>Lincoln</td>\n",
       "      <td>35</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>43112</th>\n",
       "      <td>89d1333ab2a46ce0b691b9b804c9dd61e64748e1</td>\n",
       "      <td>2017-02-11 14:35:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>697</td>\n",
       "      <td>760</td>\n",
       "      <td>700</td>\n",
       "      <td>NE</td>\n",
       "      <td>Lincoln</td>\n",
       "      <td>35</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>43113</th>\n",
       "      <td>8da5c11cefc4000111b5a018878a8c11a229f5f3</td>\n",
       "      <td>2017-02-06 17:35:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>687</td>\n",
       "      <td>760</td>\n",
       "      <td>690</td>\n",
       "      <td>NE</td>\n",
       "      <td>Omaha</td>\n",
       "      <td>30</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>43114</th>\n",
       "      <td>9e191515cec0f18d94d1c2fdeadb8f02695f27e7</td>\n",
       "      <td>2017-02-19 15:20:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>689</td>\n",
       "      <td>760</td>\n",
       "      <td>670</td>\n",
       "      <td>NE</td>\n",
       "      <td>Omaha</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>43115</th>\n",
       "      <td>8da5c11cefc4000111b5a018878a8c11a229f5f3</td>\n",
       "      <td>2017-02-21 14:50:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>676</td>\n",
       "      <td>760</td>\n",
       "      <td>670</td>\n",
       "      <td>NE</td>\n",
       "      <td>Omaha</td>\n",
       "      <td>30</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>40952 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                     Identifier                date_time  \\\n",
       "0      bbcd2609414c9fbb9e754b8f0f1939cb2a3c47aa  2017-02-10 18:20:00 UTC   \n",
       "1      bbcd2609414c9fbb9e754b8f0f1939cb2a3c47aa  2017-02-10 14:45:00 UTC   \n",
       "2      7cfa84db6a98d9989c00390bf61914b066cab4b8  2017-02-19 13:45:00 UTC   \n",
       "3      bbcd2609414c9fbb9e754b8f0f1939cb2a3c47aa  2017-02-14 14:10:00 UTC   \n",
       "4      bbcd2609414c9fbb9e754b8f0f1939cb2a3c47aa  2017-02-10 14:15:00 UTC   \n",
       "...                                         ...                      ...   \n",
       "43111  89d1333ab2a46ce0b691b9b804c9dd61e64748e1  2017-02-09 17:55:00 UTC   \n",
       "43112  89d1333ab2a46ce0b691b9b804c9dd61e64748e1  2017-02-11 14:35:00 UTC   \n",
       "43113  8da5c11cefc4000111b5a018878a8c11a229f5f3  2017-02-06 17:35:00 UTC   \n",
       "43114  9e191515cec0f18d94d1c2fdeadb8f02695f27e7  2017-02-19 15:20:00 UTC   \n",
       "43115  8da5c11cefc4000111b5a018878a8c11a229f5f3  2017-02-21 14:50:00 UTC   \n",
       "\n",
       "      HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0         auto          hold               737                      765   \n",
       "1         auto          hold               722                      765   \n",
       "2         heat          hold               700                      685   \n",
       "3         auto          hold               755                      775   \n",
       "4         auto          hold               721                      765   \n",
       "...        ...           ...               ...                      ...   \n",
       "43111     auto          auto               696                      760   \n",
       "43112     auto          auto               697                      760   \n",
       "43113     auto          hold               687                      760   \n",
       "43114     auto          hold               689                      760   \n",
       "43115     auto          hold               676                      760   \n",
       "\n",
       "       TemperatureExpectedHeat ProvinceState     City  Age_of_Home__years_  \\\n",
       "0                          735            NE  Lincoln                   10   \n",
       "1                          735            NE  Lincoln                   10   \n",
       "2                          685            NE    Omaha                   50   \n",
       "3                          755            NE  Lincoln                   10   \n",
       "4                          735            NE  Lincoln                   10   \n",
       "...                        ...           ...      ...                  ...   \n",
       "43111                      700            NE  Lincoln                   35   \n",
       "43112                      700            NE  Lincoln                   35   \n",
       "43113                      690            NE    Omaha                   30   \n",
       "43114                      670            NE    Omaha                    0   \n",
       "43115                      670            NE    Omaha                   30   \n",
       "\n",
       "       allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                  True         False             True   \n",
       "1                  True         False             True   \n",
       "2                 False         False            False   \n",
       "3                  True         False             True   \n",
       "4                  True         False             True   \n",
       "...                 ...           ...              ...   \n",
       "43111             False         False            False   \n",
       "43112             False         False            False   \n",
       "43113             False         False             True   \n",
       "43114             False         False            False   \n",
       "43115             False         False             True   \n",
       "\n",
       "      Auxilliary_Heat_Fuel_Type  \n",
       "0                      Electric  \n",
       "1                      Electric  \n",
       "2                           Gas  \n",
       "3                      Electric  \n",
       "4                      Electric  \n",
       "...                         ...  \n",
       "43111                       Gas  \n",
       "43112                       Gas  \n",
       "43113                  Electric  \n",
       "43114                       Gas  \n",
       "43115                  Electric  \n",
       "\n",
       "[40952 rows x 14 columns]"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "feb_2017.drop(feb_2017[feb_2017['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "feb_2017.drop(feb_2017[feb_2017['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "feb_2017.drop(feb_2017[feb_2017['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "feb_2017.drop(feb_2017[feb_2017['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "feb_2017"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "feb_2017[\"Year\"] = \"2017\"\n",
    "feb_2017[\"Month\"] = \"feb\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "feb_2017 = feb_2017.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "feb_2017_ave = feb_2017.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# feb_2017_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "feb_2017_ave.to_csv(\"data/day/NE/feb/feb_2017_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2018 February Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "feb_2018 = pd.read_csv(\"../data_large/NE-day/2018-feb-day-NE.csv\")\n",
    "\n",
    "# feb_2018"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>6c71018df5d7d3ec67c41d961ba0a0bbdd3a9e19</td>\n",
       "      <td>2018-02-13 15:15:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>661</td>\n",
       "      <td>667</td>\n",
       "      <td>667</td>\n",
       "      <td>NE</td>\n",
       "      <td>Lincoln</td>\n",
       "      <td>25</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>d0142ffe7ec8913522cd169fed649d6122f3c49c</td>\n",
       "      <td>2018-02-27 18:10:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>707</td>\n",
       "      <td>765</td>\n",
       "      <td>715</td>\n",
       "      <td>NE</td>\n",
       "      <td>Lincoln</td>\n",
       "      <td>35</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>deb5a406ef9cf79c602c1597ec8090cbf2e11913</td>\n",
       "      <td>2018-02-23 16:25:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>712</td>\n",
       "      <td>718</td>\n",
       "      <td>708</td>\n",
       "      <td>NE</td>\n",
       "      <td>Lincoln</td>\n",
       "      <td>20</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2504df671d8f2abfb4e209289ba629963f2db33b</td>\n",
       "      <td>2018-02-14 15:10:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>736</td>\n",
       "      <td>830</td>\n",
       "      <td>740</td>\n",
       "      <td>NE</td>\n",
       "      <td>Lincoln</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>d0040049f0105d39f159341c12d1f054719fb53f</td>\n",
       "      <td>2018-02-19 18:20:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>675</td>\n",
       "      <td>744</td>\n",
       "      <td>674</td>\n",
       "      <td>NE</td>\n",
       "      <td>Elkhorn</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>120959</th>\n",
       "      <td>ff8d6679e9ce5c1228cf640311c0110b3aafb5bd</td>\n",
       "      <td>2018-02-13 17:20:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>759</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>NE</td>\n",
       "      <td>Omaha</td>\n",
       "      <td>0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>120960</th>\n",
       "      <td>ff8d6679e9ce5c1228cf640311c0110b3aafb5bd</td>\n",
       "      <td>2018-02-13 19:40:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>759</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>NE</td>\n",
       "      <td>Omaha</td>\n",
       "      <td>0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>120961</th>\n",
       "      <td>ff8d6679e9ce5c1228cf640311c0110b3aafb5bd</td>\n",
       "      <td>2018-02-13 14:05:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>742</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>NE</td>\n",
       "      <td>Omaha</td>\n",
       "      <td>0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>120962</th>\n",
       "      <td>ff8d6679e9ce5c1228cf640311c0110b3aafb5bd</td>\n",
       "      <td>2018-02-13 14:50:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>759</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>NE</td>\n",
       "      <td>Omaha</td>\n",
       "      <td>0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>120963</th>\n",
       "      <td>ff8d6679e9ce5c1228cf640311c0110b3aafb5bd</td>\n",
       "      <td>2018-02-13 15:10:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>753</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>NE</td>\n",
       "      <td>Omaha</td>\n",
       "      <td>0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>115707 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                      Identifier                date_time  \\\n",
       "0       6c71018df5d7d3ec67c41d961ba0a0bbdd3a9e19  2018-02-13 15:15:00 UTC   \n",
       "1       d0142ffe7ec8913522cd169fed649d6122f3c49c  2018-02-27 18:10:00 UTC   \n",
       "2       deb5a406ef9cf79c602c1597ec8090cbf2e11913  2018-02-23 16:25:00 UTC   \n",
       "3       2504df671d8f2abfb4e209289ba629963f2db33b  2018-02-14 15:10:00 UTC   \n",
       "5       d0040049f0105d39f159341c12d1f054719fb53f  2018-02-19 18:20:00 UTC   \n",
       "...                                          ...                      ...   \n",
       "120959  ff8d6679e9ce5c1228cf640311c0110b3aafb5bd  2018-02-13 17:20:00 UTC   \n",
       "120960  ff8d6679e9ce5c1228cf640311c0110b3aafb5bd  2018-02-13 19:40:00 UTC   \n",
       "120961  ff8d6679e9ce5c1228cf640311c0110b3aafb5bd  2018-02-13 14:05:00 UTC   \n",
       "120962  ff8d6679e9ce5c1228cf640311c0110b3aafb5bd  2018-02-13 14:50:00 UTC   \n",
       "120963  ff8d6679e9ce5c1228cf640311c0110b3aafb5bd  2018-02-13 15:10:00 UTC   \n",
       "\n",
       "       HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0          heat          hold               661                      667   \n",
       "1          auto          auto               707                      765   \n",
       "2          heat          hold               712                      718   \n",
       "3          auto          hold               736                      830   \n",
       "5          heat          hold               675                      744   \n",
       "...         ...           ...               ...                      ...   \n",
       "120959     heat          hold               759                      760   \n",
       "120960     heat          hold               759                      760   \n",
       "120961     heat          hold               742                      760   \n",
       "120962     heat          hold               759                      760   \n",
       "120963     heat          hold               753                      760   \n",
       "\n",
       "        TemperatureExpectedHeat ProvinceState     City  Age_of_Home__years_  \\\n",
       "0                           667            NE  Lincoln                   25   \n",
       "1                           715            NE  Lincoln                   35   \n",
       "2                           708            NE  Lincoln                   20   \n",
       "3                           740            NE  Lincoln                    0   \n",
       "5                           674            NE  Elkhorn                    0   \n",
       "...                         ...           ...      ...                  ...   \n",
       "120959                      760            NE    Omaha                    0   \n",
       "120960                      760            NE    Omaha                    0   \n",
       "120961                      760            NE    Omaha                    0   \n",
       "120962                      760            NE    Omaha                    0   \n",
       "120963                      760            NE    Omaha                    0   \n",
       "\n",
       "        allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                  False         False             True   \n",
       "1                  False         False            False   \n",
       "2                  False         False            False   \n",
       "3                  False         False            False   \n",
       "5                  False         False            False   \n",
       "...                  ...           ...              ...   \n",
       "120959              True         False             True   \n",
       "120960              True         False             True   \n",
       "120961              True         False             True   \n",
       "120962              True         False             True   \n",
       "120963              True         False             True   \n",
       "\n",
       "       Auxilliary_Heat_Fuel_Type  \n",
       "0                       Electric  \n",
       "1                            Gas  \n",
       "2                            Gas  \n",
       "3                            Gas  \n",
       "5                            Gas  \n",
       "...                          ...  \n",
       "120959                  Electric  \n",
       "120960                  Electric  \n",
       "120961                  Electric  \n",
       "120962                  Electric  \n",
       "120963                  Electric  \n",
       "\n",
       "[115707 rows x 14 columns]"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "feb_2018.drop(feb_2018[feb_2018['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "feb_2018.drop(feb_2018[feb_2018['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "feb_2018.drop(feb_2018[feb_2018['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "feb_2018.drop(feb_2018[feb_2018['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "feb_2018"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "feb_2018[\"Year\"] = \"2018\"\n",
    "feb_2018[\"Month\"] = \"feb\"\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "feb_2018 = feb_2018.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "feb_2018_ave = feb_2018.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# feb_2018_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "feb_2018_ave.to_csv(\"data/day/NE/feb/feb_2018_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2019 February Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "feb_2019 = pd.read_csv(\"../data_large/NE-day/2019-feb-day-NE.csv\")\n",
    "\n",
    "# feb_2019"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>c039f7f15fbd5d98b80efba2631acb48bef7d066</td>\n",
       "      <td>2019-02-16 13:55:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>721</td>\n",
       "      <td>775</td>\n",
       "      <td>725</td>\n",
       "      <td>NE</td>\n",
       "      <td>Bennington</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>6c71018df5d7d3ec67c41d961ba0a0bbdd3a9e19</td>\n",
       "      <td>2019-02-10 17:05:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>653</td>\n",
       "      <td>659</td>\n",
       "      <td>659</td>\n",
       "      <td>NE</td>\n",
       "      <td>Lincoln</td>\n",
       "      <td>25</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>a26d25cdd2e63d4ce7c8754f7b9479ca92e16561</td>\n",
       "      <td>2019-02-02 18:35:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>662</td>\n",
       "      <td>672</td>\n",
       "      <td>668</td>\n",
       "      <td>NE</td>\n",
       "      <td>Lincoln</td>\n",
       "      <td>15</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>3b46790ec167583ec99f9fcc4d1aa248ecd4fa77</td>\n",
       "      <td>2019-02-13 14:00:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>718</td>\n",
       "      <td>717</td>\n",
       "      <td>717</td>\n",
       "      <td>NE</td>\n",
       "      <td>Gretna</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>3b46790ec167583ec99f9fcc4d1aa248ecd4fa77</td>\n",
       "      <td>2019-02-04 14:25:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>714</td>\n",
       "      <td>717</td>\n",
       "      <td>717</td>\n",
       "      <td>NE</td>\n",
       "      <td>Gretna</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>144587</th>\n",
       "      <td>a879115cd52da5e02d3f452a5dd21eea4b5dafe9</td>\n",
       "      <td>2019-02-20 18:35:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>759</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>NE</td>\n",
       "      <td>Minatare</td>\n",
       "      <td>20</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>144588</th>\n",
       "      <td>a879115cd52da5e02d3f452a5dd21eea4b5dafe9</td>\n",
       "      <td>2019-02-26 14:35:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>758</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>NE</td>\n",
       "      <td>Minatare</td>\n",
       "      <td>20</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>144589</th>\n",
       "      <td>a879115cd52da5e02d3f452a5dd21eea4b5dafe9</td>\n",
       "      <td>2019-02-26 18:25:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>NE</td>\n",
       "      <td>Minatare</td>\n",
       "      <td>20</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>144590</th>\n",
       "      <td>a879115cd52da5e02d3f452a5dd21eea4b5dafe9</td>\n",
       "      <td>2019-02-26 14:50:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>752</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>NE</td>\n",
       "      <td>Minatare</td>\n",
       "      <td>20</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>144591</th>\n",
       "      <td>a879115cd52da5e02d3f452a5dd21eea4b5dafe9</td>\n",
       "      <td>2019-02-26 15:00:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>757</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>NE</td>\n",
       "      <td>Minatare</td>\n",
       "      <td>20</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>142016 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                      Identifier                date_time  \\\n",
       "0       c039f7f15fbd5d98b80efba2631acb48bef7d066  2019-02-16 13:55:00 UTC   \n",
       "3       6c71018df5d7d3ec67c41d961ba0a0bbdd3a9e19  2019-02-10 17:05:00 UTC   \n",
       "4       a26d25cdd2e63d4ce7c8754f7b9479ca92e16561  2019-02-02 18:35:00 UTC   \n",
       "5       3b46790ec167583ec99f9fcc4d1aa248ecd4fa77  2019-02-13 14:00:00 UTC   \n",
       "7       3b46790ec167583ec99f9fcc4d1aa248ecd4fa77  2019-02-04 14:25:00 UTC   \n",
       "...                                          ...                      ...   \n",
       "144587  a879115cd52da5e02d3f452a5dd21eea4b5dafe9  2019-02-20 18:35:00 UTC   \n",
       "144588  a879115cd52da5e02d3f452a5dd21eea4b5dafe9  2019-02-26 14:35:00 UTC   \n",
       "144589  a879115cd52da5e02d3f452a5dd21eea4b5dafe9  2019-02-26 18:25:00 UTC   \n",
       "144590  a879115cd52da5e02d3f452a5dd21eea4b5dafe9  2019-02-26 14:50:00 UTC   \n",
       "144591  a879115cd52da5e02d3f452a5dd21eea4b5dafe9  2019-02-26 15:00:00 UTC   \n",
       "\n",
       "       HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0          auto          hold               721                      775   \n",
       "3          heat          hold               653                      659   \n",
       "4          heat          auto               662                      672   \n",
       "5          heat          hold               718                      717   \n",
       "7          heat          hold               714                      717   \n",
       "...         ...           ...               ...                      ...   \n",
       "144587     heat          auto               759                      760   \n",
       "144588     heat          auto               758                      760   \n",
       "144589     heat          auto               760                      760   \n",
       "144590     heat          auto               752                      760   \n",
       "144591     heat          auto               757                      760   \n",
       "\n",
       "        TemperatureExpectedHeat ProvinceState        City  \\\n",
       "0                           725            NE  Bennington   \n",
       "3                           659            NE     Lincoln   \n",
       "4                           668            NE     Lincoln   \n",
       "5                           717            NE      Gretna   \n",
       "7                           717            NE      Gretna   \n",
       "...                         ...           ...         ...   \n",
       "144587                      760            NE    Minatare   \n",
       "144588                      760            NE    Minatare   \n",
       "144589                      760            NE    Minatare   \n",
       "144590                      760            NE    Minatare   \n",
       "144591                      760            NE    Minatare   \n",
       "\n",
       "        Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                         0             False         False            False   \n",
       "3                        25             False         False             True   \n",
       "4                        15             False         False            False   \n",
       "5                         0             False         False            False   \n",
       "7                         0             False         False            False   \n",
       "...                     ...               ...           ...              ...   \n",
       "144587                   20             False         False            False   \n",
       "144588                   20             False         False            False   \n",
       "144589                   20             False         False            False   \n",
       "144590                   20             False         False            False   \n",
       "144591                   20             False         False            False   \n",
       "\n",
       "       Auxilliary_Heat_Fuel_Type  \n",
       "0                            Gas  \n",
       "3                       Electric  \n",
       "4                            Gas  \n",
       "5                            Gas  \n",
       "7                            Gas  \n",
       "...                          ...  \n",
       "144587                       Gas  \n",
       "144588                       Gas  \n",
       "144589                       Gas  \n",
       "144590                       Gas  \n",
       "144591                       Gas  \n",
       "\n",
       "[142016 rows x 14 columns]"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "feb_2019.drop(feb_2019[feb_2019['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "feb_2019.drop(feb_2019[feb_2019['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "feb_2019.drop(feb_2019[feb_2019['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "feb_2019.drop(feb_2019[feb_2019['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "feb_2019"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "feb_2019[\"Year\"] = \"2019\"\n",
    "feb_2019[\"Month\"] = \"feb\"\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "feb_2019 = feb_2019.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "feb_2019_ave = feb_2019.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# feb_2019_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "feb_2019_ave.to_csv(\"data/day/NE/feb/feb_2019_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2020 February Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "feb_2020 = pd.read_csv(\"../data_large/NE-day/2020-feb-day-NE.csv\")\n",
    "\n",
    "# feb_2020"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>8148e47f455d15aa74eb84f220afc14d7f182b10</td>\n",
       "      <td>2020-02-22 16:25:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>690</td>\n",
       "      <td>685</td>\n",
       "      <td>685</td>\n",
       "      <td>NE</td>\n",
       "      <td>Omaha</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>124aefa02526f136375928b0c92df70f356d26b9</td>\n",
       "      <td>2020-02-22 19:20:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>659</td>\n",
       "      <td>655</td>\n",
       "      <td>655</td>\n",
       "      <td>NE</td>\n",
       "      <td>Omaha</td>\n",
       "      <td>50</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2504df671d8f2abfb4e209289ba629963f2db33b</td>\n",
       "      <td>2020-02-04 14:35:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>750</td>\n",
       "      <td>775</td>\n",
       "      <td>750</td>\n",
       "      <td>NE</td>\n",
       "      <td>Lincoln</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>c039f7f15fbd5d98b80efba2631acb48bef7d066</td>\n",
       "      <td>2020-02-09 11:30:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>717</td>\n",
       "      <td>775</td>\n",
       "      <td>725</td>\n",
       "      <td>NE</td>\n",
       "      <td>Bennington</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0eb988a29c001ee5efd375446de17a6888c90e4c</td>\n",
       "      <td>2020-02-15 18:45:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>661</td>\n",
       "      <td>640</td>\n",
       "      <td>640</td>\n",
       "      <td>NE</td>\n",
       "      <td>Omaha</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>199262</th>\n",
       "      <td>dc8b24b927b1c859f3d316b2c07c14a69a26181b</td>\n",
       "      <td>2020-02-14 19:50:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>673</td>\n",
       "      <td>760</td>\n",
       "      <td>670</td>\n",
       "      <td>NE</td>\n",
       "      <td>Bennington</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>199263</th>\n",
       "      <td>dc8b24b927b1c859f3d316b2c07c14a69a26181b</td>\n",
       "      <td>2020-02-21 18:20:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>693</td>\n",
       "      <td>760</td>\n",
       "      <td>670</td>\n",
       "      <td>NE</td>\n",
       "      <td>Bennington</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>199264</th>\n",
       "      <td>dc8b24b927b1c859f3d316b2c07c14a69a26181b</td>\n",
       "      <td>2020-02-21 19:00:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>692</td>\n",
       "      <td>760</td>\n",
       "      <td>670</td>\n",
       "      <td>NE</td>\n",
       "      <td>Bennington</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>199265</th>\n",
       "      <td>dc8b24b927b1c859f3d316b2c07c14a69a26181b</td>\n",
       "      <td>2020-02-01 14:30:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>681</td>\n",
       "      <td>760</td>\n",
       "      <td>680</td>\n",
       "      <td>NE</td>\n",
       "      <td>Bennington</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>199266</th>\n",
       "      <td>dc8b24b927b1c859f3d316b2c07c14a69a26181b</td>\n",
       "      <td>2020-02-09 12:50:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>657</td>\n",
       "      <td>760</td>\n",
       "      <td>660</td>\n",
       "      <td>NE</td>\n",
       "      <td>Bennington</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>191481 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                      Identifier                date_time  \\\n",
       "0       8148e47f455d15aa74eb84f220afc14d7f182b10  2020-02-22 16:25:00 UTC   \n",
       "1       124aefa02526f136375928b0c92df70f356d26b9  2020-02-22 19:20:00 UTC   \n",
       "2       2504df671d8f2abfb4e209289ba629963f2db33b  2020-02-04 14:35:00 UTC   \n",
       "3       c039f7f15fbd5d98b80efba2631acb48bef7d066  2020-02-09 11:30:00 UTC   \n",
       "4       0eb988a29c001ee5efd375446de17a6888c90e4c  2020-02-15 18:45:00 UTC   \n",
       "...                                          ...                      ...   \n",
       "199262  dc8b24b927b1c859f3d316b2c07c14a69a26181b  2020-02-14 19:50:00 UTC   \n",
       "199263  dc8b24b927b1c859f3d316b2c07c14a69a26181b  2020-02-21 18:20:00 UTC   \n",
       "199264  dc8b24b927b1c859f3d316b2c07c14a69a26181b  2020-02-21 19:00:00 UTC   \n",
       "199265  dc8b24b927b1c859f3d316b2c07c14a69a26181b  2020-02-01 14:30:00 UTC   \n",
       "199266  dc8b24b927b1c859f3d316b2c07c14a69a26181b  2020-02-09 12:50:00 UTC   \n",
       "\n",
       "       HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0          heat          hold               690                      685   \n",
       "1          heat          hold               659                      655   \n",
       "2          heat          auto               750                      775   \n",
       "3          auto          hold               717                      775   \n",
       "4          heat          hold               661                      640   \n",
       "...         ...           ...               ...                      ...   \n",
       "199262     auto          hold               673                      760   \n",
       "199263     auto          auto               693                      760   \n",
       "199264     auto          auto               692                      760   \n",
       "199265     auto          hold               681                      760   \n",
       "199266     auto          auto               657                      760   \n",
       "\n",
       "        TemperatureExpectedHeat ProvinceState        City  \\\n",
       "0                           685            NE       Omaha   \n",
       "1                           655            NE       Omaha   \n",
       "2                           750            NE     Lincoln   \n",
       "3                           725            NE  Bennington   \n",
       "4                           640            NE       Omaha   \n",
       "...                         ...           ...         ...   \n",
       "199262                      670            NE  Bennington   \n",
       "199263                      670            NE  Bennington   \n",
       "199264                      670            NE  Bennington   \n",
       "199265                      680            NE  Bennington   \n",
       "199266                      660            NE  Bennington   \n",
       "\n",
       "        Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                         5             False         False            False   \n",
       "1                        50              True         False             True   \n",
       "2                         0             False         False            False   \n",
       "3                         0             False         False            False   \n",
       "4                        10             False         False            False   \n",
       "...                     ...               ...           ...              ...   \n",
       "199262                    5             False         False            False   \n",
       "199263                    5             False         False            False   \n",
       "199264                    5             False         False            False   \n",
       "199265                    5             False         False            False   \n",
       "199266                    5             False         False            False   \n",
       "\n",
       "       Auxilliary_Heat_Fuel_Type  \n",
       "0                            Gas  \n",
       "1                       Electric  \n",
       "2                            Gas  \n",
       "3                            Gas  \n",
       "4                            Gas  \n",
       "...                          ...  \n",
       "199262                       Gas  \n",
       "199263                       Gas  \n",
       "199264                       Gas  \n",
       "199265                       Gas  \n",
       "199266                       Gas  \n",
       "\n",
       "[191481 rows x 14 columns]"
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "feb_2020.drop(feb_2020[feb_2020['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "feb_2020.drop(feb_2020[feb_2020['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "feb_2020.drop(feb_2020[feb_2020['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "feb_2020.drop(feb_2020[feb_2020['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "feb_2020"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "feb_2020[\"Year\"] = \"2020\"\n",
    "feb_2020[\"Month\"] = \"feb\"\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "feb_2020 = feb_2020.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "feb_2020_ave = feb_2020.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# feb_2020_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "feb_2020_ave.to_csv(\"data/day/NE/feb/feb_2020_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2021 February Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "feb_2021 = pd.read_csv(\"../data_large/NE-day/2021-feb-day-NE.csv\")\n",
    "\n",
    "# feb_2021"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>ae207699741703a86828265b6556382c300adb71</td>\n",
       "      <td>2021-02-03 16:00:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>721</td>\n",
       "      <td>776</td>\n",
       "      <td>726</td>\n",
       "      <td>NE</td>\n",
       "      <td>Lincoln</td>\n",
       "      <td>95</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>d3ad6d46c43f95c6fbf6e250eeb4696d6a95200d</td>\n",
       "      <td>2021-02-28 17:00:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>653</td>\n",
       "      <td>664</td>\n",
       "      <td>640</td>\n",
       "      <td>NE</td>\n",
       "      <td>North Platte</td>\n",
       "      <td>60</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>e0b9a0d802f823a1cf6a2106c4ac75c05d08b31e</td>\n",
       "      <td>2021-02-21 17:05:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>674</td>\n",
       "      <td>742</td>\n",
       "      <td>672</td>\n",
       "      <td>NE</td>\n",
       "      <td>Lincoln</td>\n",
       "      <td>40</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2504df671d8f2abfb4e209289ba629963f2db33b</td>\n",
       "      <td>2021-02-07 17:50:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>738</td>\n",
       "      <td>743</td>\n",
       "      <td>743</td>\n",
       "      <td>NE</td>\n",
       "      <td>Lincoln</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>2504df671d8f2abfb4e209289ba629963f2db33b</td>\n",
       "      <td>2021-02-04 16:40:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>748</td>\n",
       "      <td>743</td>\n",
       "      <td>743</td>\n",
       "      <td>NE</td>\n",
       "      <td>Lincoln</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>130162</th>\n",
       "      <td>86c502d93e7747d5ce109c6610f9e71a7e2a7b1b</td>\n",
       "      <td>2021-02-03 17:55:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>756</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>NE</td>\n",
       "      <td>Bellevue</td>\n",
       "      <td>58</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>130163</th>\n",
       "      <td>86c502d93e7747d5ce109c6610f9e71a7e2a7b1b</td>\n",
       "      <td>2021-02-03 15:55:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>NE</td>\n",
       "      <td>Bellevue</td>\n",
       "      <td>58</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>130164</th>\n",
       "      <td>86c502d93e7747d5ce109c6610f9e71a7e2a7b1b</td>\n",
       "      <td>2021-02-03 18:00:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>759</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>NE</td>\n",
       "      <td>Bellevue</td>\n",
       "      <td>58</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>130165</th>\n",
       "      <td>86c502d93e7747d5ce109c6610f9e71a7e2a7b1b</td>\n",
       "      <td>2021-02-03 16:15:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>754</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>NE</td>\n",
       "      <td>Bellevue</td>\n",
       "      <td>58</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>130166</th>\n",
       "      <td>86c502d93e7747d5ce109c6610f9e71a7e2a7b1b</td>\n",
       "      <td>2021-02-03 16:00:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>759</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>NE</td>\n",
       "      <td>Bellevue</td>\n",
       "      <td>58</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>129117 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                      Identifier                date_time  \\\n",
       "1       ae207699741703a86828265b6556382c300adb71  2021-02-03 16:00:00 UTC   \n",
       "2       d3ad6d46c43f95c6fbf6e250eeb4696d6a95200d  2021-02-28 17:00:00 UTC   \n",
       "3       e0b9a0d802f823a1cf6a2106c4ac75c05d08b31e  2021-02-21 17:05:00 UTC   \n",
       "4       2504df671d8f2abfb4e209289ba629963f2db33b  2021-02-07 17:50:00 UTC   \n",
       "5       2504df671d8f2abfb4e209289ba629963f2db33b  2021-02-04 16:40:00 UTC   \n",
       "...                                          ...                      ...   \n",
       "130162  86c502d93e7747d5ce109c6610f9e71a7e2a7b1b  2021-02-03 17:55:00 UTC   \n",
       "130163  86c502d93e7747d5ce109c6610f9e71a7e2a7b1b  2021-02-03 15:55:00 UTC   \n",
       "130164  86c502d93e7747d5ce109c6610f9e71a7e2a7b1b  2021-02-03 18:00:00 UTC   \n",
       "130165  86c502d93e7747d5ce109c6610f9e71a7e2a7b1b  2021-02-03 16:15:00 UTC   \n",
       "130166  86c502d93e7747d5ce109c6610f9e71a7e2a7b1b  2021-02-03 16:00:00 UTC   \n",
       "\n",
       "       HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "1          auto          hold               721                      776   \n",
       "2          heat          hold               653                      664   \n",
       "3          auto          hold               674                      742   \n",
       "4          heat          hold               738                      743   \n",
       "5          heat          hold               748                      743   \n",
       "...         ...           ...               ...                      ...   \n",
       "130162     heat          hold               756                      760   \n",
       "130163     heat          hold               760                      760   \n",
       "130164     heat          hold               759                      760   \n",
       "130165     heat          hold               754                      760   \n",
       "130166     heat          hold               759                      760   \n",
       "\n",
       "        TemperatureExpectedHeat ProvinceState          City  \\\n",
       "1                           726            NE       Lincoln   \n",
       "2                           640            NE  North Platte   \n",
       "3                           672            NE       Lincoln   \n",
       "4                           743            NE       Lincoln   \n",
       "5                           743            NE       Lincoln   \n",
       "...                         ...           ...           ...   \n",
       "130162                      760            NE      Bellevue   \n",
       "130163                      760            NE      Bellevue   \n",
       "130164                      760            NE      Bellevue   \n",
       "130165                      760            NE      Bellevue   \n",
       "130166                      760            NE      Bellevue   \n",
       "\n",
       "        Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "1                        95             False         False            False   \n",
       "2                        60             False         False            False   \n",
       "3                        40             False         False             True   \n",
       "4                         0             False         False            False   \n",
       "5                         0             False         False            False   \n",
       "...                     ...               ...           ...              ...   \n",
       "130162                   58             False         False            False   \n",
       "130163                   58             False         False            False   \n",
       "130164                   58             False         False            False   \n",
       "130165                   58             False         False            False   \n",
       "130166                   58             False         False            False   \n",
       "\n",
       "       Auxilliary_Heat_Fuel_Type  \n",
       "1                            Gas  \n",
       "2                            Gas  \n",
       "3                       Electric  \n",
       "4                            Gas  \n",
       "5                            Gas  \n",
       "...                          ...  \n",
       "130162                       Gas  \n",
       "130163                       Gas  \n",
       "130164                       Gas  \n",
       "130165                       Gas  \n",
       "130166                       Gas  \n",
       "\n",
       "[129117 rows x 14 columns]"
      ]
     },
     "execution_count": 60,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "feb_2021.drop(feb_2021[feb_2021['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "feb_2021.drop(feb_2021[feb_2021['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "feb_2021.drop(feb_2021[feb_2021['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "feb_2021.drop(feb_2021[feb_2021['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "feb_2021"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "feb_2021[\"Year\"] = \"2021\"\n",
    "feb_2021[\"Month\"] = \"feb\"\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "feb_2021 = feb_2021.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "feb_2021_ave = feb_2021.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# feb_2021_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "feb_2021_ave.to_csv(\"data/day/NE/feb/feb_2021_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "\n",
    "### Combine month CSV Files \n",
    "1. Read in files in folders for each state\n",
    "2. Export as combined CSV"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create variable for files in directory\n",
    "files = [f for f in os.listdir(\"data/day/NE/feb/\") if f.endswith(\".csv\")]\n",
    "\n",
    "# files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>Month</th>\n",
       "      <th>Year</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>City</th>\n",
       "      <th>Temperature_ctrl_ave</th>\n",
       "      <th>TemperatureExpectedCool_ave</th>\n",
       "      <th>TemperatureExpectedHeat_ave</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>05320d509a18b2d4022029409e7a267736e78e0b</td>\n",
       "      <td>feb</td>\n",
       "      <td>2017</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>Lincoln</td>\n",
       "      <td>706.320000</td>\n",
       "      <td>721.960000</td>\n",
       "      <td>718.640000</td>\n",
       "      <td>90.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>05320d509a18b2d4022029409e7a267736e78e0b</td>\n",
       "      <td>feb</td>\n",
       "      <td>2017</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>Lincoln</td>\n",
       "      <td>707.606195</td>\n",
       "      <td>690.053097</td>\n",
       "      <td>686.526549</td>\n",
       "      <td>90.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0d779ed5bbc93a32506824e72f1b051e9980a0cf</td>\n",
       "      <td>feb</td>\n",
       "      <td>2017</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>Blair</td>\n",
       "      <td>708.043478</td>\n",
       "      <td>714.782609</td>\n",
       "      <td>714.782609</td>\n",
       "      <td>35.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>10484576cdf6fa0ce377a1110f267866bcfea53d</td>\n",
       "      <td>feb</td>\n",
       "      <td>2017</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>Omaha</td>\n",
       "      <td>704.510204</td>\n",
       "      <td>677.448980</td>\n",
       "      <td>677.448980</td>\n",
       "      <td>0.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1368c1b32f1bf0e6653e05e1c3cc7a00b8ac0384</td>\n",
       "      <td>feb</td>\n",
       "      <td>2017</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>Papillion</td>\n",
       "      <td>650.524590</td>\n",
       "      <td>650.065574</td>\n",
       "      <td>630.065574</td>\n",
       "      <td>20.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>208</th>\n",
       "      <td>f9efefb7392498029db8a74ea5f6a2f93d20fa47</td>\n",
       "      <td>feb</td>\n",
       "      <td>2021</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>Bellevue</td>\n",
       "      <td>682.800000</td>\n",
       "      <td>691.200000</td>\n",
       "      <td>689.240000</td>\n",
       "      <td>25.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>209</th>\n",
       "      <td>faa85f28edb3a9a38ac6130d63b1c3f892466036</td>\n",
       "      <td>feb</td>\n",
       "      <td>2021</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>Omaha</td>\n",
       "      <td>701.371681</td>\n",
       "      <td>711.000000</td>\n",
       "      <td>707.723451</td>\n",
       "      <td>55.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>210</th>\n",
       "      <td>faa8c435b84d052676a86ba7d2d6a783d26e6b19</td>\n",
       "      <td>feb</td>\n",
       "      <td>2021</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>Omaha</td>\n",
       "      <td>692.013514</td>\n",
       "      <td>693.756757</td>\n",
       "      <td>693.391892</td>\n",
       "      <td>20.0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>211</th>\n",
       "      <td>faf3b76e8b0193bcf0256358efb4e593b669a1c7</td>\n",
       "      <td>feb</td>\n",
       "      <td>2021</td>\n",
       "      <td>auxHeatOnly</td>\n",
       "      <td>hold</td>\n",
       "      <td>Hickman</td>\n",
       "      <td>706.114286</td>\n",
       "      <td>710.114286</td>\n",
       "      <td>709.971429</td>\n",
       "      <td>0.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>212</th>\n",
       "      <td>ff8d6679e9ce5c1228cf640311c0110b3aafb5bd</td>\n",
       "      <td>feb</td>\n",
       "      <td>2021</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>Omaha</td>\n",
       "      <td>709.668089</td>\n",
       "      <td>713.769625</td>\n",
       "      <td>713.769625</td>\n",
       "      <td>0.0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1279 rows × 13 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                   Identifier Month  Year     HvacMode  \\\n",
       "0    05320d509a18b2d4022029409e7a267736e78e0b   feb  2017         heat   \n",
       "1    05320d509a18b2d4022029409e7a267736e78e0b   feb  2017         heat   \n",
       "2    0d779ed5bbc93a32506824e72f1b051e9980a0cf   feb  2017         heat   \n",
       "3    10484576cdf6fa0ce377a1110f267866bcfea53d   feb  2017         heat   \n",
       "4    1368c1b32f1bf0e6653e05e1c3cc7a00b8ac0384   feb  2017         cool   \n",
       "..                                        ...   ...   ...          ...   \n",
       "208  f9efefb7392498029db8a74ea5f6a2f93d20fa47   feb  2021         heat   \n",
       "209  faa85f28edb3a9a38ac6130d63b1c3f892466036   feb  2021         heat   \n",
       "210  faa8c435b84d052676a86ba7d2d6a783d26e6b19   feb  2021         heat   \n",
       "211  faf3b76e8b0193bcf0256358efb4e593b669a1c7   feb  2021  auxHeatOnly   \n",
       "212  ff8d6679e9ce5c1228cf640311c0110b3aafb5bd   feb  2021         heat   \n",
       "\n",
       "    CalendarEvent       City  Temperature_ctrl_ave  \\\n",
       "0            auto    Lincoln            706.320000   \n",
       "1            hold    Lincoln            707.606195   \n",
       "2            auto      Blair            708.043478   \n",
       "3            hold      Omaha            704.510204   \n",
       "4            hold  Papillion            650.524590   \n",
       "..            ...        ...                   ...   \n",
       "208          hold   Bellevue            682.800000   \n",
       "209          hold      Omaha            701.371681   \n",
       "210          hold      Omaha            692.013514   \n",
       "211          hold    Hickman            706.114286   \n",
       "212          hold      Omaha            709.668089   \n",
       "\n",
       "     TemperatureExpectedCool_ave  TemperatureExpectedHeat_ave  \\\n",
       "0                     721.960000                   718.640000   \n",
       "1                     690.053097                   686.526549   \n",
       "2                     714.782609                   714.782609   \n",
       "3                     677.448980                   677.448980   \n",
       "4                     650.065574                   630.065574   \n",
       "..                           ...                          ...   \n",
       "208                   691.200000                   689.240000   \n",
       "209                   711.000000                   707.723451   \n",
       "210                   693.756757                   693.391892   \n",
       "211                   710.114286                   709.971429   \n",
       "212                   713.769625                   713.769625   \n",
       "\n",
       "     Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \n",
       "0                   90.0             False         False            False  \n",
       "1                   90.0             False         False            False  \n",
       "2                   35.0             False         False            False  \n",
       "3                    0.0             False         False            False  \n",
       "4                   20.0             False         False            False  \n",
       "..                   ...               ...           ...              ...  \n",
       "208                 25.0             False         False            False  \n",
       "209                 55.0             False         False            False  \n",
       "210                 20.0              True         False            False  \n",
       "211                  0.0             False         False             True  \n",
       "212                  0.0              True         False             True  \n",
       "\n",
       "[1279 rows x 13 columns]"
      ]
     },
     "execution_count": 66,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# https://stackoverflow.com/questions/63886787/how-to-create-a-dataframe-from-multiple-csv-files\n",
    "# Initialize an empty dataframe\n",
    "NE_feb = pd.DataFrame()\n",
    "\n",
    "# Iterate through files and contents, then concatenate the data into the dataframe\n",
    "for file in files:\n",
    "    df = pd.read_csv(\"data/day/NE/feb/\" + file)\n",
    "    NE_feb = pd.concat([NE_feb, df])\n",
    "    \n",
    "NE_feb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [],
   "source": [
    "NE_feb.to_csv(\"Scraper_Output/State_Month_Day/NE/NE_feb.csv\", header=True, index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "\n",
    "## March"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "\n",
    "## April"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "\n",
    "## May"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "\n",
    "## June"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2017 June Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "jun_2017 = pd.read_csv(\"../data_large/NE-day/2017-jun-day-NE.csv\")\n",
    "\n",
    "# jun_2017"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>ec27fedc2a3af90b5453dc4df107b5f19ab4cc9c</td>\n",
       "      <td>2017-06-01 15:30:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>729</td>\n",
       "      <td>731</td>\n",
       "      <td>731</td>\n",
       "      <td>NE</td>\n",
       "      <td>Syracuse</td>\n",
       "      <td>55</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>e98b28d1f1c335a8494b1c1e7367c8d02464a2eb</td>\n",
       "      <td>2017-06-02 12:25:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>744</td>\n",
       "      <td>774</td>\n",
       "      <td>694</td>\n",
       "      <td>NE</td>\n",
       "      <td>Lincoln</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>56b6390b43d3384d08426091f5f5d9e4726167d0</td>\n",
       "      <td>2017-06-16 11:25:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>775</td>\n",
       "      <td>781</td>\n",
       "      <td>680</td>\n",
       "      <td>NE</td>\n",
       "      <td>Blair</td>\n",
       "      <td>25</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>56b6390b43d3384d08426091f5f5d9e4726167d0</td>\n",
       "      <td>2017-06-11 11:30:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>773</td>\n",
       "      <td>781</td>\n",
       "      <td>671</td>\n",
       "      <td>NE</td>\n",
       "      <td>Blair</td>\n",
       "      <td>25</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>1985472cdc3f8d3bf064a68f444737c05c724ea8</td>\n",
       "      <td>2017-06-08 18:00:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>748</td>\n",
       "      <td>775</td>\n",
       "      <td>695</td>\n",
       "      <td>NE</td>\n",
       "      <td>Omaha</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>84167</th>\n",
       "      <td>dcd591783d60cc5ebb4cab197cde6801d1d2b70f</td>\n",
       "      <td>2017-06-05 18:40:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>724</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>NE</td>\n",
       "      <td>Lincoln</td>\n",
       "      <td>77</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>84168</th>\n",
       "      <td>dcd591783d60cc5ebb4cab197cde6801d1d2b70f</td>\n",
       "      <td>2017-06-05 18:45:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>724</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>NE</td>\n",
       "      <td>Lincoln</td>\n",
       "      <td>77</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>84169</th>\n",
       "      <td>dcd591783d60cc5ebb4cab197cde6801d1d2b70f</td>\n",
       "      <td>2017-06-05 18:15:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>722</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>NE</td>\n",
       "      <td>Lincoln</td>\n",
       "      <td>77</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>84170</th>\n",
       "      <td>dcd591783d60cc5ebb4cab197cde6801d1d2b70f</td>\n",
       "      <td>2017-06-05 19:15:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>728</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>NE</td>\n",
       "      <td>Lincoln</td>\n",
       "      <td>77</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>84171</th>\n",
       "      <td>dcd591783d60cc5ebb4cab197cde6801d1d2b70f</td>\n",
       "      <td>2017-06-05 18:35:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>723</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>NE</td>\n",
       "      <td>Lincoln</td>\n",
       "      <td>77</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>82549 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                     Identifier                date_time  \\\n",
       "0      ec27fedc2a3af90b5453dc4df107b5f19ab4cc9c  2017-06-01 15:30:00 UTC   \n",
       "6      e98b28d1f1c335a8494b1c1e7367c8d02464a2eb  2017-06-02 12:25:00 UTC   \n",
       "7      56b6390b43d3384d08426091f5f5d9e4726167d0  2017-06-16 11:25:00 UTC   \n",
       "8      56b6390b43d3384d08426091f5f5d9e4726167d0  2017-06-11 11:30:00 UTC   \n",
       "9      1985472cdc3f8d3bf064a68f444737c05c724ea8  2017-06-08 18:00:00 UTC   \n",
       "...                                         ...                      ...   \n",
       "84167  dcd591783d60cc5ebb4cab197cde6801d1d2b70f  2017-06-05 18:40:00 UTC   \n",
       "84168  dcd591783d60cc5ebb4cab197cde6801d1d2b70f  2017-06-05 18:45:00 UTC   \n",
       "84169  dcd591783d60cc5ebb4cab197cde6801d1d2b70f  2017-06-05 18:15:00 UTC   \n",
       "84170  dcd591783d60cc5ebb4cab197cde6801d1d2b70f  2017-06-05 19:15:00 UTC   \n",
       "84171  dcd591783d60cc5ebb4cab197cde6801d1d2b70f  2017-06-05 18:35:00 UTC   \n",
       "\n",
       "      HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0         cool          hold               729                      731   \n",
       "6         auto          auto               744                      774   \n",
       "7         auto          auto               775                      781   \n",
       "8         auto          auto               773                      781   \n",
       "9         auto          hold               748                      775   \n",
       "...        ...           ...               ...                      ...   \n",
       "84167     cool          hold               724                      760   \n",
       "84168     cool          hold               724                      760   \n",
       "84169     cool          hold               722                      760   \n",
       "84170     cool          hold               728                      760   \n",
       "84171     cool          hold               723                      760   \n",
       "\n",
       "       TemperatureExpectedHeat ProvinceState      City  Age_of_Home__years_  \\\n",
       "0                          731            NE  Syracuse                   55   \n",
       "6                          694            NE   Lincoln                    0   \n",
       "7                          680            NE     Blair                   25   \n",
       "8                          671            NE     Blair                   25   \n",
       "9                          695            NE     Omaha                   10   \n",
       "...                        ...           ...       ...                  ...   \n",
       "84167                      760            NE   Lincoln                   77   \n",
       "84168                      760            NE   Lincoln                   77   \n",
       "84169                      760            NE   Lincoln                   77   \n",
       "84170                      760            NE   Lincoln                   77   \n",
       "84171                      760            NE   Lincoln                   77   \n",
       "\n",
       "       allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                 False         False             True   \n",
       "6                 False         False            False   \n",
       "7                 False         False            False   \n",
       "8                 False         False            False   \n",
       "9                 False         False            False   \n",
       "...                 ...           ...              ...   \n",
       "84167             False         False            False   \n",
       "84168             False         False            False   \n",
       "84169             False         False            False   \n",
       "84170             False         False            False   \n",
       "84171             False         False            False   \n",
       "\n",
       "      Auxilliary_Heat_Fuel_Type  \n",
       "0                      Electric  \n",
       "6                           Gas  \n",
       "7                           Gas  \n",
       "8                           Gas  \n",
       "9                           Gas  \n",
       "...                         ...  \n",
       "84167                       Gas  \n",
       "84168                       Gas  \n",
       "84169                       Gas  \n",
       "84170                       Gas  \n",
       "84171                       Gas  \n",
       "\n",
       "[82549 rows x 14 columns]"
      ]
     },
     "execution_count": 69,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "jun_2017.drop(jun_2017[jun_2017['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "jun_2017.drop(jun_2017[jun_2017['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "jun_2017.drop(jun_2017[jun_2017['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "jun_2017.drop(jun_2017[jun_2017['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "jun_2017"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "jun_2017[\"Year\"] = \"2017\"\n",
    "jun_2017[\"Month\"] = \"jun\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "jun_2017 = jun_2017.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "jun_2017_ave = jun_2017.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# jun_2017_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "jun_2017_ave.to_csv(\"data/day/NE/jun/jun_2017_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2018 June Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "jun_2018 = pd.read_csv(\"../data_large/NE-day/2018-jun-day-NE.csv\")\n",
    "\n",
    "# jun_2018"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1cde6ae2ccdf910bb7fd2bd26dac007a23614191</td>\n",
       "      <td>2018-06-17 13:30:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>731</td>\n",
       "      <td>732</td>\n",
       "      <td>676</td>\n",
       "      <td>NE</td>\n",
       "      <td>Lincoln</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>50f777071abb7b1664d734531f57de6ce9007794</td>\n",
       "      <td>2018-06-23 19:15:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>728</td>\n",
       "      <td>729</td>\n",
       "      <td>729</td>\n",
       "      <td>NE</td>\n",
       "      <td>Omaha</td>\n",
       "      <td>30</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>a152c3918557f196745694cf50f718c79cef91f8</td>\n",
       "      <td>2018-06-12 14:05:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>736</td>\n",
       "      <td>733</td>\n",
       "      <td>683</td>\n",
       "      <td>NE</td>\n",
       "      <td>Grand Island</td>\n",
       "      <td>40</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>3fbd860c5b0aa1d896ef522174171b57b8c4c44b</td>\n",
       "      <td>2018-06-10 16:25:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>697</td>\n",
       "      <td>697</td>\n",
       "      <td>697</td>\n",
       "      <td>NE</td>\n",
       "      <td>Lincoln</td>\n",
       "      <td>70</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>a152c3918557f196745694cf50f718c79cef91f8</td>\n",
       "      <td>2018-06-16 15:30:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>735</td>\n",
       "      <td>733</td>\n",
       "      <td>683</td>\n",
       "      <td>NE</td>\n",
       "      <td>Grand Island</td>\n",
       "      <td>40</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>185700</th>\n",
       "      <td>ff8d6679e9ce5c1228cf640311c0110b3aafb5bd</td>\n",
       "      <td>2018-06-17 18:40:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>749</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>NE</td>\n",
       "      <td>Omaha</td>\n",
       "      <td>0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>185701</th>\n",
       "      <td>ff8d6679e9ce5c1228cf640311c0110b3aafb5bd</td>\n",
       "      <td>2018-06-17 17:30:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>741</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>NE</td>\n",
       "      <td>Omaha</td>\n",
       "      <td>0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>185702</th>\n",
       "      <td>ff8d6679e9ce5c1228cf640311c0110b3aafb5bd</td>\n",
       "      <td>2018-06-16 18:30:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>743</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>NE</td>\n",
       "      <td>Omaha</td>\n",
       "      <td>0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>185703</th>\n",
       "      <td>ff8d6679e9ce5c1228cf640311c0110b3aafb5bd</td>\n",
       "      <td>2018-06-17 19:05:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>755</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>NE</td>\n",
       "      <td>Omaha</td>\n",
       "      <td>0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>185704</th>\n",
       "      <td>ff8d6679e9ce5c1228cf640311c0110b3aafb5bd</td>\n",
       "      <td>2018-06-17 19:40:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>761</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>NE</td>\n",
       "      <td>Omaha</td>\n",
       "      <td>0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>183768 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                      Identifier                date_time  \\\n",
       "0       1cde6ae2ccdf910bb7fd2bd26dac007a23614191  2018-06-17 13:30:00 UTC   \n",
       "1       50f777071abb7b1664d734531f57de6ce9007794  2018-06-23 19:15:00 UTC   \n",
       "3       a152c3918557f196745694cf50f718c79cef91f8  2018-06-12 14:05:00 UTC   \n",
       "4       3fbd860c5b0aa1d896ef522174171b57b8c4c44b  2018-06-10 16:25:00 UTC   \n",
       "5       a152c3918557f196745694cf50f718c79cef91f8  2018-06-16 15:30:00 UTC   \n",
       "...                                          ...                      ...   \n",
       "185700  ff8d6679e9ce5c1228cf640311c0110b3aafb5bd  2018-06-17 18:40:00 UTC   \n",
       "185701  ff8d6679e9ce5c1228cf640311c0110b3aafb5bd  2018-06-17 17:30:00 UTC   \n",
       "185702  ff8d6679e9ce5c1228cf640311c0110b3aafb5bd  2018-06-16 18:30:00 UTC   \n",
       "185703  ff8d6679e9ce5c1228cf640311c0110b3aafb5bd  2018-06-17 19:05:00 UTC   \n",
       "185704  ff8d6679e9ce5c1228cf640311c0110b3aafb5bd  2018-06-17 19:40:00 UTC   \n",
       "\n",
       "       HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0          auto          hold               731                      732   \n",
       "1          cool          hold               728                      729   \n",
       "3          auto          auto               736                      733   \n",
       "4          cool          hold               697                      697   \n",
       "5          auto          auto               735                      733   \n",
       "...         ...           ...               ...                      ...   \n",
       "185700     cool          hold               749                      760   \n",
       "185701     cool          hold               741                      760   \n",
       "185702     cool          hold               743                      760   \n",
       "185703     cool          hold               755                      760   \n",
       "185704     cool          hold               761                      760   \n",
       "\n",
       "        TemperatureExpectedHeat ProvinceState          City  \\\n",
       "0                           676            NE       Lincoln   \n",
       "1                           729            NE         Omaha   \n",
       "3                           683            NE  Grand Island   \n",
       "4                           697            NE       Lincoln   \n",
       "5                           683            NE  Grand Island   \n",
       "...                         ...           ...           ...   \n",
       "185700                      760            NE         Omaha   \n",
       "185701                      760            NE         Omaha   \n",
       "185702                      760            NE         Omaha   \n",
       "185703                      760            NE         Omaha   \n",
       "185704                      760            NE         Omaha   \n",
       "\n",
       "        Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                        10             False         False            False   \n",
       "1                        30             False         False            False   \n",
       "3                        40             False         False            False   \n",
       "4                        70             False         False            False   \n",
       "5                        40             False         False            False   \n",
       "...                     ...               ...           ...              ...   \n",
       "185700                    0              True         False             True   \n",
       "185701                    0              True         False             True   \n",
       "185702                    0              True         False             True   \n",
       "185703                    0              True         False             True   \n",
       "185704                    0              True         False             True   \n",
       "\n",
       "       Auxilliary_Heat_Fuel_Type  \n",
       "0                            Gas  \n",
       "1                            Gas  \n",
       "3                            Gas  \n",
       "4                            Gas  \n",
       "5                            Gas  \n",
       "...                          ...  \n",
       "185700                  Electric  \n",
       "185701                  Electric  \n",
       "185702                  Electric  \n",
       "185703                  Electric  \n",
       "185704                  Electric  \n",
       "\n",
       "[183768 rows x 14 columns]"
      ]
     },
     "execution_count": 75,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "jun_2018.drop(jun_2018[jun_2018['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "jun_2018.drop(jun_2018[jun_2018['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "jun_2018.drop(jun_2018[jun_2018['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "jun_2018.drop(jun_2018[jun_2018['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "jun_2018"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "jun_2018[\"Year\"] = \"2018\"\n",
    "jun_2018[\"Month\"] = \"jun\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "jun_2018 = jun_2018.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "jun_2018_ave = jun_2018.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# jun_2018_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "jun_2018_ave.to_csv(\"data/day/NE/jun/jun_2018_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2019 June Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "jun_2019 = pd.read_csv(\"../data_large/NE-day/2019-jun-day-NE.csv\")\n",
    "\n",
    "# jun_2019"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>a8f8fef9a85bf3d53a6f15cbf33ded8fc0130f27</td>\n",
       "      <td>2019-06-08 18:35:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>721</td>\n",
       "      <td>726</td>\n",
       "      <td>676</td>\n",
       "      <td>NE</td>\n",
       "      <td>Omaha</td>\n",
       "      <td>90</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>50f777071abb7b1664d734531f57de6ce9007794</td>\n",
       "      <td>2019-06-10 15:00:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>733</td>\n",
       "      <td>732</td>\n",
       "      <td>732</td>\n",
       "      <td>NE</td>\n",
       "      <td>Omaha</td>\n",
       "      <td>30</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0a7b8a3fc74f19b5d4a368c58557be1f3ca053f7</td>\n",
       "      <td>2019-06-12 14:10:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>680</td>\n",
       "      <td>830</td>\n",
       "      <td>792</td>\n",
       "      <td>NE</td>\n",
       "      <td>Lincoln</td>\n",
       "      <td>69</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>50f777071abb7b1664d734531f57de6ce9007794</td>\n",
       "      <td>2019-06-11 19:05:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>731</td>\n",
       "      <td>732</td>\n",
       "      <td>732</td>\n",
       "      <td>NE</td>\n",
       "      <td>Omaha</td>\n",
       "      <td>30</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>50f777071abb7b1664d734531f57de6ce9007794</td>\n",
       "      <td>2019-06-23 16:10:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>713</td>\n",
       "      <td>706</td>\n",
       "      <td>706</td>\n",
       "      <td>NE</td>\n",
       "      <td>Omaha</td>\n",
       "      <td>30</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>235789</th>\n",
       "      <td>6c2daa77787fdb0d7171782aeefd25a0b112e5f0</td>\n",
       "      <td>2019-06-08 12:40:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>703</td>\n",
       "      <td>730</td>\n",
       "      <td>620</td>\n",
       "      <td>NE</td>\n",
       "      <td>Plattsmouth</td>\n",
       "      <td>10</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>235790</th>\n",
       "      <td>6c2daa77787fdb0d7171782aeefd25a0b112e5f0</td>\n",
       "      <td>2019-06-08 11:40:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>704</td>\n",
       "      <td>730</td>\n",
       "      <td>620</td>\n",
       "      <td>NE</td>\n",
       "      <td>Plattsmouth</td>\n",
       "      <td>10</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>235791</th>\n",
       "      <td>6c2daa77787fdb0d7171782aeefd25a0b112e5f0</td>\n",
       "      <td>2019-06-08 12:00:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>702</td>\n",
       "      <td>730</td>\n",
       "      <td>620</td>\n",
       "      <td>NE</td>\n",
       "      <td>Plattsmouth</td>\n",
       "      <td>10</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>235792</th>\n",
       "      <td>6c2daa77787fdb0d7171782aeefd25a0b112e5f0</td>\n",
       "      <td>2019-06-08 13:50:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>702</td>\n",
       "      <td>730</td>\n",
       "      <td>620</td>\n",
       "      <td>NE</td>\n",
       "      <td>Plattsmouth</td>\n",
       "      <td>10</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>235793</th>\n",
       "      <td>6c2daa77787fdb0d7171782aeefd25a0b112e5f0</td>\n",
       "      <td>2019-06-08 12:55:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>703</td>\n",
       "      <td>730</td>\n",
       "      <td>620</td>\n",
       "      <td>NE</td>\n",
       "      <td>Plattsmouth</td>\n",
       "      <td>10</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>226586 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                      Identifier                date_time  \\\n",
       "0       a8f8fef9a85bf3d53a6f15cbf33ded8fc0130f27  2019-06-08 18:35:00 UTC   \n",
       "1       50f777071abb7b1664d734531f57de6ce9007794  2019-06-10 15:00:00 UTC   \n",
       "2       0a7b8a3fc74f19b5d4a368c58557be1f3ca053f7  2019-06-12 14:10:00 UTC   \n",
       "3       50f777071abb7b1664d734531f57de6ce9007794  2019-06-11 19:05:00 UTC   \n",
       "4       50f777071abb7b1664d734531f57de6ce9007794  2019-06-23 16:10:00 UTC   \n",
       "...                                          ...                      ...   \n",
       "235789  6c2daa77787fdb0d7171782aeefd25a0b112e5f0  2019-06-08 12:40:00 UTC   \n",
       "235790  6c2daa77787fdb0d7171782aeefd25a0b112e5f0  2019-06-08 11:40:00 UTC   \n",
       "235791  6c2daa77787fdb0d7171782aeefd25a0b112e5f0  2019-06-08 12:00:00 UTC   \n",
       "235792  6c2daa77787fdb0d7171782aeefd25a0b112e5f0  2019-06-08 13:50:00 UTC   \n",
       "235793  6c2daa77787fdb0d7171782aeefd25a0b112e5f0  2019-06-08 12:55:00 UTC   \n",
       "\n",
       "       HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0          auto          auto               721                      726   \n",
       "1          cool          hold               733                      732   \n",
       "2          cool          auto               680                      830   \n",
       "3          cool          hold               731                      732   \n",
       "4          cool          hold               713                      706   \n",
       "...         ...           ...               ...                      ...   \n",
       "235789     auto          auto               703                      730   \n",
       "235790     auto          auto               704                      730   \n",
       "235791     auto          auto               702                      730   \n",
       "235792     auto          auto               702                      730   \n",
       "235793     auto          auto               703                      730   \n",
       "\n",
       "        TemperatureExpectedHeat ProvinceState         City  \\\n",
       "0                           676            NE        Omaha   \n",
       "1                           732            NE        Omaha   \n",
       "2                           792            NE      Lincoln   \n",
       "3                           732            NE        Omaha   \n",
       "4                           706            NE        Omaha   \n",
       "...                         ...           ...          ...   \n",
       "235789                      620            NE  Plattsmouth   \n",
       "235790                      620            NE  Plattsmouth   \n",
       "235791                      620            NE  Plattsmouth   \n",
       "235792                      620            NE  Plattsmouth   \n",
       "235793                      620            NE  Plattsmouth   \n",
       "\n",
       "        Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                        90              True         False             True   \n",
       "1                        30             False         False            False   \n",
       "2                        69              True         False            False   \n",
       "3                        30             False         False            False   \n",
       "4                        30             False         False            False   \n",
       "...                     ...               ...           ...              ...   \n",
       "235789                   10              True         False            False   \n",
       "235790                   10              True         False            False   \n",
       "235791                   10              True         False            False   \n",
       "235792                   10              True         False            False   \n",
       "235793                   10              True         False            False   \n",
       "\n",
       "       Auxilliary_Heat_Fuel_Type  \n",
       "0                       Electric  \n",
       "1                            Gas  \n",
       "2                            Gas  \n",
       "3                            Gas  \n",
       "4                            Gas  \n",
       "...                          ...  \n",
       "235789                       Gas  \n",
       "235790                       Gas  \n",
       "235791                       Gas  \n",
       "235792                       Gas  \n",
       "235793                       Gas  \n",
       "\n",
       "[226586 rows x 14 columns]"
      ]
     },
     "execution_count": 81,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "jun_2019.drop(jun_2019[jun_2019['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "jun_2019.drop(jun_2019[jun_2019['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "jun_2019.drop(jun_2019[jun_2019['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "jun_2019.drop(jun_2019[jun_2019['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "jun_2019"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "jun_2019[\"Year\"] = \"2019\"\n",
    "jun_2019[\"Month\"] = \"jun\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "jun_2019 = jun_2019.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "jun_2019_ave = jun_2019.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# jun_2019_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "jun_2019_ave.to_csv(\"data/day/NE/jun/jun_2019_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2020 June Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "jun_2020 = pd.read_csv(\"../data_large/NE-day/2020-jun-day-NE.csv\")\n",
    "\n",
    "# jun_2020"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>6c2daa77787fdb0d7171782aeefd25a0b112e5f0</td>\n",
       "      <td>2020-06-28 15:35:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>669</td>\n",
       "      <td>670</td>\n",
       "      <td>670</td>\n",
       "      <td>NE</td>\n",
       "      <td>Plattsmouth</td>\n",
       "      <td>10</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>6c2daa77787fdb0d7171782aeefd25a0b112e5f0</td>\n",
       "      <td>2020-06-08 19:35:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>671</td>\n",
       "      <td>670</td>\n",
       "      <td>670</td>\n",
       "      <td>NE</td>\n",
       "      <td>Plattsmouth</td>\n",
       "      <td>10</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>6c2daa77787fdb0d7171782aeefd25a0b112e5f0</td>\n",
       "      <td>2020-06-08 14:40:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>676</td>\n",
       "      <td>670</td>\n",
       "      <td>670</td>\n",
       "      <td>NE</td>\n",
       "      <td>Plattsmouth</td>\n",
       "      <td>10</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>6c2daa77787fdb0d7171782aeefd25a0b112e5f0</td>\n",
       "      <td>2020-06-20 13:15:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>660</td>\n",
       "      <td>670</td>\n",
       "      <td>670</td>\n",
       "      <td>NE</td>\n",
       "      <td>Plattsmouth</td>\n",
       "      <td>10</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>6c2daa77787fdb0d7171782aeefd25a0b112e5f0</td>\n",
       "      <td>2020-06-20 19:35:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>673</td>\n",
       "      <td>670</td>\n",
       "      <td>670</td>\n",
       "      <td>NE</td>\n",
       "      <td>Plattsmouth</td>\n",
       "      <td>10</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>246185</th>\n",
       "      <td>6d43e20c2c1846a3448c6d954942cdbf6b9fc622</td>\n",
       "      <td>2020-06-21 09:45:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>730</td>\n",
       "      <td>730</td>\n",
       "      <td>730</td>\n",
       "      <td>NE</td>\n",
       "      <td>Papillion</td>\n",
       "      <td>20</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>246186</th>\n",
       "      <td>6d43e20c2c1846a3448c6d954942cdbf6b9fc622</td>\n",
       "      <td>2020-06-22 09:10:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>732</td>\n",
       "      <td>730</td>\n",
       "      <td>730</td>\n",
       "      <td>NE</td>\n",
       "      <td>Papillion</td>\n",
       "      <td>20</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>246187</th>\n",
       "      <td>6d43e20c2c1846a3448c6d954942cdbf6b9fc622</td>\n",
       "      <td>2020-06-03 18:55:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>736</td>\n",
       "      <td>730</td>\n",
       "      <td>610</td>\n",
       "      <td>NE</td>\n",
       "      <td>Papillion</td>\n",
       "      <td>20</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>246188</th>\n",
       "      <td>6d43e20c2c1846a3448c6d954942cdbf6b9fc622</td>\n",
       "      <td>2020-06-03 19:10:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>731</td>\n",
       "      <td>730</td>\n",
       "      <td>610</td>\n",
       "      <td>NE</td>\n",
       "      <td>Papillion</td>\n",
       "      <td>20</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>246189</th>\n",
       "      <td>6d43e20c2c1846a3448c6d954942cdbf6b9fc622</td>\n",
       "      <td>2020-06-20 13:30:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>723</td>\n",
       "      <td>730</td>\n",
       "      <td>730</td>\n",
       "      <td>NE</td>\n",
       "      <td>Papillion</td>\n",
       "      <td>20</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>234530 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                      Identifier                date_time  \\\n",
       "0       6c2daa77787fdb0d7171782aeefd25a0b112e5f0  2020-06-28 15:35:00 UTC   \n",
       "1       6c2daa77787fdb0d7171782aeefd25a0b112e5f0  2020-06-08 19:35:00 UTC   \n",
       "2       6c2daa77787fdb0d7171782aeefd25a0b112e5f0  2020-06-08 14:40:00 UTC   \n",
       "3       6c2daa77787fdb0d7171782aeefd25a0b112e5f0  2020-06-20 13:15:00 UTC   \n",
       "4       6c2daa77787fdb0d7171782aeefd25a0b112e5f0  2020-06-20 19:35:00 UTC   \n",
       "...                                          ...                      ...   \n",
       "246185  6d43e20c2c1846a3448c6d954942cdbf6b9fc622  2020-06-21 09:45:00 UTC   \n",
       "246186  6d43e20c2c1846a3448c6d954942cdbf6b9fc622  2020-06-22 09:10:00 UTC   \n",
       "246187  6d43e20c2c1846a3448c6d954942cdbf6b9fc622  2020-06-03 18:55:00 UTC   \n",
       "246188  6d43e20c2c1846a3448c6d954942cdbf6b9fc622  2020-06-03 19:10:00 UTC   \n",
       "246189  6d43e20c2c1846a3448c6d954942cdbf6b9fc622  2020-06-20 13:30:00 UTC   \n",
       "\n",
       "       HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0          cool          auto               669                      670   \n",
       "1          cool          auto               671                      670   \n",
       "2          cool          auto               676                      670   \n",
       "3          cool          auto               660                      670   \n",
       "4          cool          auto               673                      670   \n",
       "...         ...           ...               ...                      ...   \n",
       "246185     cool          hold               730                      730   \n",
       "246186     cool          hold               732                      730   \n",
       "246187     auto          auto               736                      730   \n",
       "246188     auto          auto               731                      730   \n",
       "246189     cool          hold               723                      730   \n",
       "\n",
       "        TemperatureExpectedHeat ProvinceState         City  \\\n",
       "0                           670            NE  Plattsmouth   \n",
       "1                           670            NE  Plattsmouth   \n",
       "2                           670            NE  Plattsmouth   \n",
       "3                           670            NE  Plattsmouth   \n",
       "4                           670            NE  Plattsmouth   \n",
       "...                         ...           ...          ...   \n",
       "246185                      730            NE    Papillion   \n",
       "246186                      730            NE    Papillion   \n",
       "246187                      610            NE    Papillion   \n",
       "246188                      610            NE    Papillion   \n",
       "246189                      730            NE    Papillion   \n",
       "\n",
       "        Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                        10              True         False            False   \n",
       "1                        10              True         False            False   \n",
       "2                        10              True         False            False   \n",
       "3                        10              True         False            False   \n",
       "4                        10              True         False            False   \n",
       "...                     ...               ...           ...              ...   \n",
       "246185                   20              True         False            False   \n",
       "246186                   20              True         False            False   \n",
       "246187                   20              True         False            False   \n",
       "246188                   20              True         False            False   \n",
       "246189                   20              True         False            False   \n",
       "\n",
       "       Auxilliary_Heat_Fuel_Type  \n",
       "0                            Gas  \n",
       "1                            Gas  \n",
       "2                            Gas  \n",
       "3                            Gas  \n",
       "4                            Gas  \n",
       "...                          ...  \n",
       "246185                       Gas  \n",
       "246186                       Gas  \n",
       "246187                       Gas  \n",
       "246188                       Gas  \n",
       "246189                       Gas  \n",
       "\n",
       "[234530 rows x 14 columns]"
      ]
     },
     "execution_count": 87,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "jun_2020.drop(jun_2020[jun_2020['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "jun_2020.drop(jun_2020[jun_2020['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "jun_2020.drop(jun_2020[jun_2020['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "jun_2020.drop(jun_2020[jun_2020['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "jun_2020"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "jun_2020[\"Year\"] = \"2020\"\n",
    "jun_2020[\"Month\"] = \"jun\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "jun_2020 = jun_2020.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "jun_2020_ave = jun_2020.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# jun_2020_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "jun_2020_ave.to_csv(\"data/day/NE/jun/jun_2020_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2021 June Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "jun_2021 = pd.read_csv(\"../data_large/NE-day/2021-jun-day-NE.csv\")\n",
    "\n",
    "# jun_2021"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>399c9c9f011895083e1e3063640a4b8313daad01</td>\n",
       "      <td>2021-06-27 18:25:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>744</td>\n",
       "      <td>744</td>\n",
       "      <td>664</td>\n",
       "      <td>NE</td>\n",
       "      <td>Fremont</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>770532d678d67213ddc1339014f8df35b27aa31a</td>\n",
       "      <td>2021-06-14 19:45:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>689</td>\n",
       "      <td>685</td>\n",
       "      <td>635</td>\n",
       "      <td>NE</td>\n",
       "      <td>Plattsmouth</td>\n",
       "      <td>10</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>e1c436fad40c0745964d68bfa66687da305d5b4f</td>\n",
       "      <td>2021-06-05 14:45:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>702</td>\n",
       "      <td>696</td>\n",
       "      <td>696</td>\n",
       "      <td>NE</td>\n",
       "      <td>Omaha</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>e1c436fad40c0745964d68bfa66687da305d5b4f</td>\n",
       "      <td>2021-06-20 19:45:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>708</td>\n",
       "      <td>706</td>\n",
       "      <td>706</td>\n",
       "      <td>NE</td>\n",
       "      <td>Omaha</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2504df671d8f2abfb4e209289ba629963f2db33b</td>\n",
       "      <td>2021-06-14 16:35:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>770</td>\n",
       "      <td>788</td>\n",
       "      <td>788</td>\n",
       "      <td>NE</td>\n",
       "      <td>Lincoln</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>141243</th>\n",
       "      <td>dcd591783d60cc5ebb4cab197cde6801d1d2b70f</td>\n",
       "      <td>2021-06-13 14:50:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>747</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>NE</td>\n",
       "      <td>Lincoln</td>\n",
       "      <td>77</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>141244</th>\n",
       "      <td>dcd591783d60cc5ebb4cab197cde6801d1d2b70f</td>\n",
       "      <td>2021-06-12 15:30:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>738</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>NE</td>\n",
       "      <td>Lincoln</td>\n",
       "      <td>77</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>141245</th>\n",
       "      <td>dcd591783d60cc5ebb4cab197cde6801d1d2b70f</td>\n",
       "      <td>2021-06-12 17:20:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>741</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>NE</td>\n",
       "      <td>Lincoln</td>\n",
       "      <td>77</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>141246</th>\n",
       "      <td>dcd591783d60cc5ebb4cab197cde6801d1d2b70f</td>\n",
       "      <td>2021-06-13 14:05:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>745</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>NE</td>\n",
       "      <td>Lincoln</td>\n",
       "      <td>77</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>141247</th>\n",
       "      <td>dcd591783d60cc5ebb4cab197cde6801d1d2b70f</td>\n",
       "      <td>2021-06-12 16:25:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>740</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>NE</td>\n",
       "      <td>Lincoln</td>\n",
       "      <td>77</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>139761 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                      Identifier                date_time  \\\n",
       "0       399c9c9f011895083e1e3063640a4b8313daad01  2021-06-27 18:25:00 UTC   \n",
       "1       770532d678d67213ddc1339014f8df35b27aa31a  2021-06-14 19:45:00 UTC   \n",
       "2       e1c436fad40c0745964d68bfa66687da305d5b4f  2021-06-05 14:45:00 UTC   \n",
       "3       e1c436fad40c0745964d68bfa66687da305d5b4f  2021-06-20 19:45:00 UTC   \n",
       "4       2504df671d8f2abfb4e209289ba629963f2db33b  2021-06-14 16:35:00 UTC   \n",
       "...                                          ...                      ...   \n",
       "141243  dcd591783d60cc5ebb4cab197cde6801d1d2b70f  2021-06-13 14:50:00 UTC   \n",
       "141244  dcd591783d60cc5ebb4cab197cde6801d1d2b70f  2021-06-12 15:30:00 UTC   \n",
       "141245  dcd591783d60cc5ebb4cab197cde6801d1d2b70f  2021-06-12 17:20:00 UTC   \n",
       "141246  dcd591783d60cc5ebb4cab197cde6801d1d2b70f  2021-06-13 14:05:00 UTC   \n",
       "141247  dcd591783d60cc5ebb4cab197cde6801d1d2b70f  2021-06-12 16:25:00 UTC   \n",
       "\n",
       "       HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0          auto          hold               744                      744   \n",
       "1          auto          hold               689                      685   \n",
       "2          cool          hold               702                      696   \n",
       "3          cool          hold               708                      706   \n",
       "4          cool          hold               770                      788   \n",
       "...         ...           ...               ...                      ...   \n",
       "141243     cool          hold               747                      760   \n",
       "141244     cool          hold               738                      760   \n",
       "141245     cool          hold               741                      760   \n",
       "141246     cool          hold               745                      760   \n",
       "141247     cool          hold               740                      760   \n",
       "\n",
       "        TemperatureExpectedHeat ProvinceState         City  \\\n",
       "0                           664            NE      Fremont   \n",
       "1                           635            NE  Plattsmouth   \n",
       "2                           696            NE        Omaha   \n",
       "3                           706            NE        Omaha   \n",
       "4                           788            NE      Lincoln   \n",
       "...                         ...           ...          ...   \n",
       "141243                      760            NE      Lincoln   \n",
       "141244                      760            NE      Lincoln   \n",
       "141245                      760            NE      Lincoln   \n",
       "141246                      760            NE      Lincoln   \n",
       "141247                      760            NE      Lincoln   \n",
       "\n",
       "        Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                        10             False         False             True   \n",
       "1                        10              True         False            False   \n",
       "2                        10             False         False            False   \n",
       "3                        10             False         False            False   \n",
       "4                         0             False         False            False   \n",
       "...                     ...               ...           ...              ...   \n",
       "141243                   77             False         False            False   \n",
       "141244                   77             False         False            False   \n",
       "141245                   77             False         False            False   \n",
       "141246                   77             False         False            False   \n",
       "141247                   77             False         False            False   \n",
       "\n",
       "       Auxilliary_Heat_Fuel_Type  \n",
       "0                       Electric  \n",
       "1                            Gas  \n",
       "2                            Gas  \n",
       "3                            Gas  \n",
       "4                            Gas  \n",
       "...                          ...  \n",
       "141243                       Gas  \n",
       "141244                       Gas  \n",
       "141245                       Gas  \n",
       "141246                       Gas  \n",
       "141247                       Gas  \n",
       "\n",
       "[139761 rows x 14 columns]"
      ]
     },
     "execution_count": 93,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "jun_2021.drop(jun_2021[jun_2021['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "jun_2021.drop(jun_2021[jun_2021['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "jun_2021.drop(jun_2021[jun_2021['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "jun_2021.drop(jun_2021[jun_2021['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "jun_2021"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "jun_2021[\"Year\"] = \"2021\"\n",
    "jun_2021[\"Month\"] = \"jun\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "jun_2021 = jun_2021.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "jun_2021_ave = jun_2021.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# jun_2021_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "jun_2021_ave.to_csv(\"data/day/NE/jun/jun_2021_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "\n",
    "### Combine month CSV Files \n",
    "1. Read in files in folders for each state\n",
    "2. Export as combined CSV"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create variable for files in directory\n",
    "files = [f for f in os.listdir(\"data/day/NE/jun/\") if f.endswith(\".csv\")]\n",
    "\n",
    "# files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>Month</th>\n",
       "      <th>Year</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>City</th>\n",
       "      <th>Temperature_ctrl_ave</th>\n",
       "      <th>TemperatureExpectedCool_ave</th>\n",
       "      <th>TemperatureExpectedHeat_ave</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0087ac1d7540ff5efa6ae20213627943a2b07fe8</td>\n",
       "      <td>jun</td>\n",
       "      <td>2017</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>Omaha</td>\n",
       "      <td>736.500000</td>\n",
       "      <td>719.500000</td>\n",
       "      <td>693.500000</td>\n",
       "      <td>35.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0087ac1d7540ff5efa6ae20213627943a2b07fe8</td>\n",
       "      <td>jun</td>\n",
       "      <td>2017</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>Omaha</td>\n",
       "      <td>753.615385</td>\n",
       "      <td>706.923077</td>\n",
       "      <td>706.923077</td>\n",
       "      <td>35.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>033afd975854035a4b0ab320768f094d2b716859</td>\n",
       "      <td>jun</td>\n",
       "      <td>2017</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>La Vista</td>\n",
       "      <td>704.320000</td>\n",
       "      <td>748.000000</td>\n",
       "      <td>650.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>05320d509a18b2d4022029409e7a267736e78e0b</td>\n",
       "      <td>jun</td>\n",
       "      <td>2017</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>Lincoln</td>\n",
       "      <td>732.890625</td>\n",
       "      <td>718.718750</td>\n",
       "      <td>700.531250</td>\n",
       "      <td>90.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>05320d509a18b2d4022029409e7a267736e78e0b</td>\n",
       "      <td>jun</td>\n",
       "      <td>2017</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>Lincoln</td>\n",
       "      <td>732.180180</td>\n",
       "      <td>736.938739</td>\n",
       "      <td>736.282883</td>\n",
       "      <td>90.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>219</th>\n",
       "      <td>faa85f28edb3a9a38ac6130d63b1c3f892466036</td>\n",
       "      <td>jun</td>\n",
       "      <td>2021</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>Omaha</td>\n",
       "      <td>727.871901</td>\n",
       "      <td>741.438017</td>\n",
       "      <td>712.714876</td>\n",
       "      <td>55.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>220</th>\n",
       "      <td>faa8c435b84d052676a86ba7d2d6a783d26e6b19</td>\n",
       "      <td>jun</td>\n",
       "      <td>2021</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>Omaha</td>\n",
       "      <td>737.666667</td>\n",
       "      <td>700.000000</td>\n",
       "      <td>680.000000</td>\n",
       "      <td>20.0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>221</th>\n",
       "      <td>faa8c435b84d052676a86ba7d2d6a783d26e6b19</td>\n",
       "      <td>jun</td>\n",
       "      <td>2021</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>Omaha</td>\n",
       "      <td>705.419355</td>\n",
       "      <td>701.072581</td>\n",
       "      <td>701.072581</td>\n",
       "      <td>20.0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>222</th>\n",
       "      <td>faf3b76e8b0193bcf0256358efb4e593b669a1c7</td>\n",
       "      <td>jun</td>\n",
       "      <td>2021</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>Hickman</td>\n",
       "      <td>746.884615</td>\n",
       "      <td>716.000000</td>\n",
       "      <td>666.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>223</th>\n",
       "      <td>faf3b76e8b0193bcf0256358efb4e593b669a1c7</td>\n",
       "      <td>jun</td>\n",
       "      <td>2021</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>Hickman</td>\n",
       "      <td>737.982456</td>\n",
       "      <td>740.210526</td>\n",
       "      <td>730.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1621 rows × 13 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                   Identifier Month  Year HvacMode  \\\n",
       "0    0087ac1d7540ff5efa6ae20213627943a2b07fe8   jun  2017     auto   \n",
       "1    0087ac1d7540ff5efa6ae20213627943a2b07fe8   jun  2017     cool   \n",
       "2    033afd975854035a4b0ab320768f094d2b716859   jun  2017     auto   \n",
       "3    05320d509a18b2d4022029409e7a267736e78e0b   jun  2017     cool   \n",
       "4    05320d509a18b2d4022029409e7a267736e78e0b   jun  2017     cool   \n",
       "..                                        ...   ...   ...      ...   \n",
       "219  faa85f28edb3a9a38ac6130d63b1c3f892466036   jun  2021     cool   \n",
       "220  faa8c435b84d052676a86ba7d2d6a783d26e6b19   jun  2021     auto   \n",
       "221  faa8c435b84d052676a86ba7d2d6a783d26e6b19   jun  2021     cool   \n",
       "222  faf3b76e8b0193bcf0256358efb4e593b669a1c7   jun  2021     auto   \n",
       "223  faf3b76e8b0193bcf0256358efb4e593b669a1c7   jun  2021     cool   \n",
       "\n",
       "    CalendarEvent      City  Temperature_ctrl_ave  \\\n",
       "0            hold     Omaha            736.500000   \n",
       "1            hold     Omaha            753.615385   \n",
       "2            hold  La Vista            704.320000   \n",
       "3            auto   Lincoln            732.890625   \n",
       "4            hold   Lincoln            732.180180   \n",
       "..            ...       ...                   ...   \n",
       "219          hold     Omaha            727.871901   \n",
       "220          hold     Omaha            737.666667   \n",
       "221          hold     Omaha            705.419355   \n",
       "222          hold   Hickman            746.884615   \n",
       "223          hold   Hickman            737.982456   \n",
       "\n",
       "     TemperatureExpectedCool_ave  TemperatureExpectedHeat_ave  \\\n",
       "0                     719.500000                   693.500000   \n",
       "1                     706.923077                   706.923077   \n",
       "2                     748.000000                   650.000000   \n",
       "3                     718.718750                   700.531250   \n",
       "4                     736.938739                   736.282883   \n",
       "..                           ...                          ...   \n",
       "219                   741.438017                   712.714876   \n",
       "220                   700.000000                   680.000000   \n",
       "221                   701.072581                   701.072581   \n",
       "222                   716.000000                   666.000000   \n",
       "223                   740.210526                   730.000000   \n",
       "\n",
       "     Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \n",
       "0                   35.0             False         False            False  \n",
       "1                   35.0             False         False            False  \n",
       "2                    0.0             False         False            False  \n",
       "3                   90.0             False         False            False  \n",
       "4                   90.0             False         False            False  \n",
       "..                   ...               ...           ...              ...  \n",
       "219                 55.0             False         False            False  \n",
       "220                 20.0              True         False            False  \n",
       "221                 20.0              True         False            False  \n",
       "222                  0.0             False         False             True  \n",
       "223                  0.0             False         False             True  \n",
       "\n",
       "[1621 rows x 13 columns]"
      ]
     },
     "execution_count": 99,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# https://stackoverflow.com/questions/63886787/how-to-create-a-dataframe-from-multiple-csv-files\n",
    "# Initialize an empty dataframe\n",
    "NE_jun = pd.DataFrame()\n",
    "\n",
    "# Iterate through files and contents, then concatenate the data into the dataframe\n",
    "for file in files:\n",
    "    df = pd.read_csv(\"data/day/NE/jun/\" + file)\n",
    "    NE_jun = pd.concat([NE_jun, df])\n",
    "    \n",
    "NE_jun"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "metadata": {},
   "outputs": [],
   "source": [
    "NE_jun.to_csv(\"Scraper_Output/State_Month_Day/NE/NE_jun.csv\", header=True, index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "\n",
    "## July"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2017 July Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "jul_2017 = pd.read_csv(\"../data_large/NE-day/2017-jul-day-NE.csv\")\n",
    "\n",
    "# jul_2017"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>e6c7a7449be199e76934eb1df4e97a561d4ad874</td>\n",
       "      <td>2017-07-30 14:15:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>785</td>\n",
       "      <td>840</td>\n",
       "      <td>790</td>\n",
       "      <td>NE</td>\n",
       "      <td>Nebraska City</td>\n",
       "      <td>85</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>399c9c9f011895083e1e3063640a4b8313daad01</td>\n",
       "      <td>2017-07-15 18:00:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>715</td>\n",
       "      <td>713</td>\n",
       "      <td>663</td>\n",
       "      <td>NE</td>\n",
       "      <td>Fremont</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>e5020a56db9e45e942254216808f1607f90aac53</td>\n",
       "      <td>2017-07-02 18:20:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>704</td>\n",
       "      <td>718</td>\n",
       "      <td>670</td>\n",
       "      <td>NE</td>\n",
       "      <td>Bennington</td>\n",
       "      <td>15</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>e6c7a7449be199e76934eb1df4e97a561d4ad874</td>\n",
       "      <td>2017-07-01 14:30:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>747</td>\n",
       "      <td>830</td>\n",
       "      <td>790</td>\n",
       "      <td>NE</td>\n",
       "      <td>Nebraska City</td>\n",
       "      <td>85</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>1985472cdc3f8d3bf064a68f444737c05c724ea8</td>\n",
       "      <td>2017-07-05 17:15:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>751</td>\n",
       "      <td>765</td>\n",
       "      <td>715</td>\n",
       "      <td>NE</td>\n",
       "      <td>Omaha</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>99169</th>\n",
       "      <td>dc8b24b927b1c859f3d316b2c07c14a69a26181b</td>\n",
       "      <td>2017-07-15 18:50:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>630</td>\n",
       "      <td>NE</td>\n",
       "      <td>Bennington</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>99170</th>\n",
       "      <td>dc8b24b927b1c859f3d316b2c07c14a69a26181b</td>\n",
       "      <td>2017-07-12 18:50:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>745</td>\n",
       "      <td>760</td>\n",
       "      <td>630</td>\n",
       "      <td>NE</td>\n",
       "      <td>Bennington</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>99171</th>\n",
       "      <td>dc8b24b927b1c859f3d316b2c07c14a69a26181b</td>\n",
       "      <td>2017-07-13 12:30:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>736</td>\n",
       "      <td>760</td>\n",
       "      <td>630</td>\n",
       "      <td>NE</td>\n",
       "      <td>Bennington</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>99172</th>\n",
       "      <td>dc8b24b927b1c859f3d316b2c07c14a69a26181b</td>\n",
       "      <td>2017-07-14 12:45:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>747</td>\n",
       "      <td>760</td>\n",
       "      <td>630</td>\n",
       "      <td>NE</td>\n",
       "      <td>Bennington</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>99173</th>\n",
       "      <td>dc8b24b927b1c859f3d316b2c07c14a69a26181b</td>\n",
       "      <td>2017-07-14 17:45:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>753</td>\n",
       "      <td>760</td>\n",
       "      <td>630</td>\n",
       "      <td>NE</td>\n",
       "      <td>Bennington</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>97986 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                     Identifier                date_time  \\\n",
       "1      e6c7a7449be199e76934eb1df4e97a561d4ad874  2017-07-30 14:15:00 UTC   \n",
       "2      399c9c9f011895083e1e3063640a4b8313daad01  2017-07-15 18:00:00 UTC   \n",
       "3      e5020a56db9e45e942254216808f1607f90aac53  2017-07-02 18:20:00 UTC   \n",
       "4      e6c7a7449be199e76934eb1df4e97a561d4ad874  2017-07-01 14:30:00 UTC   \n",
       "5      1985472cdc3f8d3bf064a68f444737c05c724ea8  2017-07-05 17:15:00 UTC   \n",
       "...                                         ...                      ...   \n",
       "99169  dc8b24b927b1c859f3d316b2c07c14a69a26181b  2017-07-15 18:50:00 UTC   \n",
       "99170  dc8b24b927b1c859f3d316b2c07c14a69a26181b  2017-07-12 18:50:00 UTC   \n",
       "99171  dc8b24b927b1c859f3d316b2c07c14a69a26181b  2017-07-13 12:30:00 UTC   \n",
       "99172  dc8b24b927b1c859f3d316b2c07c14a69a26181b  2017-07-14 12:45:00 UTC   \n",
       "99173  dc8b24b927b1c859f3d316b2c07c14a69a26181b  2017-07-14 17:45:00 UTC   \n",
       "\n",
       "      HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "1         cool          hold               785                      840   \n",
       "2         auto          hold               715                      713   \n",
       "3         auto          hold               704                      718   \n",
       "4         cool          hold               747                      830   \n",
       "5         auto          hold               751                      765   \n",
       "...        ...           ...               ...                      ...   \n",
       "99169     auto          hold               760                      760   \n",
       "99170     auto          hold               745                      760   \n",
       "99171     auto          hold               736                      760   \n",
       "99172     auto          hold               747                      760   \n",
       "99173     auto          hold               753                      760   \n",
       "\n",
       "       TemperatureExpectedHeat ProvinceState           City  \\\n",
       "1                          790            NE  Nebraska City   \n",
       "2                          663            NE        Fremont   \n",
       "3                          670            NE     Bennington   \n",
       "4                          790            NE  Nebraska City   \n",
       "5                          715            NE          Omaha   \n",
       "...                        ...           ...            ...   \n",
       "99169                      630            NE     Bennington   \n",
       "99170                      630            NE     Bennington   \n",
       "99171                      630            NE     Bennington   \n",
       "99172                      630            NE     Bennington   \n",
       "99173                      630            NE     Bennington   \n",
       "\n",
       "       Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "1                       85             False         False            False   \n",
       "2                       10             False         False             True   \n",
       "3                       15             False         False            False   \n",
       "4                       85             False         False            False   \n",
       "5                       10             False         False            False   \n",
       "...                    ...               ...           ...              ...   \n",
       "99169                    5             False         False            False   \n",
       "99170                    5             False         False            False   \n",
       "99171                    5             False         False            False   \n",
       "99172                    5             False         False            False   \n",
       "99173                    5             False         False            False   \n",
       "\n",
       "      Auxilliary_Heat_Fuel_Type  \n",
       "1                           Gas  \n",
       "2                      Electric  \n",
       "3                           Gas  \n",
       "4                           Gas  \n",
       "5                           Gas  \n",
       "...                         ...  \n",
       "99169                       Gas  \n",
       "99170                       Gas  \n",
       "99171                       Gas  \n",
       "99172                       Gas  \n",
       "99173                       Gas  \n",
       "\n",
       "[97986 rows x 14 columns]"
      ]
     },
     "execution_count": 102,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "jul_2017.drop(jul_2017[jul_2017['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "jul_2017.drop(jul_2017[jul_2017['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "jul_2017.drop(jul_2017[jul_2017['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "jul_2017.drop(jul_2017[jul_2017['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "jul_2017"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "jul_2017[\"Year\"] = \"2017\"\n",
    "jul_2017[\"Month\"] = \"jul\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "jul_2017 = jul_2017.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "jul_2017_ave = jul_2017.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# jul_2017_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "jul_2017_ave.to_csv(\"data/day/NE/jul/jul_2017_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2018 July Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "jul_2018 = pd.read_csv(\"../data_large/NE-day/2018-jul-day-NE.csv\")\n",
    "\n",
    "# jul_2018"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 108,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>6c71018df5d7d3ec67c41d961ba0a0bbdd3a9e19</td>\n",
       "      <td>2018-07-25 14:35:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>688</td>\n",
       "      <td>687</td>\n",
       "      <td>687</td>\n",
       "      <td>NE</td>\n",
       "      <td>Lincoln</td>\n",
       "      <td>25</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>6c71018df5d7d3ec67c41d961ba0a0bbdd3a9e19</td>\n",
       "      <td>2018-07-30 13:50:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>684</td>\n",
       "      <td>687</td>\n",
       "      <td>687</td>\n",
       "      <td>NE</td>\n",
       "      <td>Lincoln</td>\n",
       "      <td>25</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>a152c3918557f196745694cf50f718c79cef91f8</td>\n",
       "      <td>2018-07-08 19:10:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>732</td>\n",
       "      <td>733</td>\n",
       "      <td>683</td>\n",
       "      <td>NE</td>\n",
       "      <td>Grand Island</td>\n",
       "      <td>40</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>24557ca6d177aa3b5a71f9110b182dc2dc436af0</td>\n",
       "      <td>2018-07-31 19:25:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>766</td>\n",
       "      <td>775</td>\n",
       "      <td>775</td>\n",
       "      <td>NE</td>\n",
       "      <td>Omaha</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>6c71018df5d7d3ec67c41d961ba0a0bbdd3a9e19</td>\n",
       "      <td>2018-07-29 17:20:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>686</td>\n",
       "      <td>685</td>\n",
       "      <td>685</td>\n",
       "      <td>NE</td>\n",
       "      <td>Lincoln</td>\n",
       "      <td>25</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>208588</th>\n",
       "      <td>c665a959dfb6cabd5665004bf9a1fc6afef9282e</td>\n",
       "      <td>2018-07-29 16:55:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>737</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>NE</td>\n",
       "      <td>Bellevue</td>\n",
       "      <td>30</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>208589</th>\n",
       "      <td>c665a959dfb6cabd5665004bf9a1fc6afef9282e</td>\n",
       "      <td>2018-07-29 16:10:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>733</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>NE</td>\n",
       "      <td>Bellevue</td>\n",
       "      <td>30</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>208590</th>\n",
       "      <td>c665a959dfb6cabd5665004bf9a1fc6afef9282e</td>\n",
       "      <td>2018-07-29 15:10:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>730</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>NE</td>\n",
       "      <td>Bellevue</td>\n",
       "      <td>30</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>208591</th>\n",
       "      <td>c665a959dfb6cabd5665004bf9a1fc6afef9282e</td>\n",
       "      <td>2018-07-29 16:20:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>733</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>NE</td>\n",
       "      <td>Bellevue</td>\n",
       "      <td>30</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>208592</th>\n",
       "      <td>c665a959dfb6cabd5665004bf9a1fc6afef9282e</td>\n",
       "      <td>2018-07-24 18:30:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>740</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>NE</td>\n",
       "      <td>Bellevue</td>\n",
       "      <td>30</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>204976 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                      Identifier                date_time  \\\n",
       "0       6c71018df5d7d3ec67c41d961ba0a0bbdd3a9e19  2018-07-25 14:35:00 UTC   \n",
       "1       6c71018df5d7d3ec67c41d961ba0a0bbdd3a9e19  2018-07-30 13:50:00 UTC   \n",
       "2       a152c3918557f196745694cf50f718c79cef91f8  2018-07-08 19:10:00 UTC   \n",
       "5       24557ca6d177aa3b5a71f9110b182dc2dc436af0  2018-07-31 19:25:00 UTC   \n",
       "6       6c71018df5d7d3ec67c41d961ba0a0bbdd3a9e19  2018-07-29 17:20:00 UTC   \n",
       "...                                          ...                      ...   \n",
       "208588  c665a959dfb6cabd5665004bf9a1fc6afef9282e  2018-07-29 16:55:00 UTC   \n",
       "208589  c665a959dfb6cabd5665004bf9a1fc6afef9282e  2018-07-29 16:10:00 UTC   \n",
       "208590  c665a959dfb6cabd5665004bf9a1fc6afef9282e  2018-07-29 15:10:00 UTC   \n",
       "208591  c665a959dfb6cabd5665004bf9a1fc6afef9282e  2018-07-29 16:20:00 UTC   \n",
       "208592  c665a959dfb6cabd5665004bf9a1fc6afef9282e  2018-07-24 18:30:00 UTC   \n",
       "\n",
       "       HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0          cool          hold               688                      687   \n",
       "1          cool          hold               684                      687   \n",
       "2          auto          hold               732                      733   \n",
       "5          cool          hold               766                      775   \n",
       "6          cool          hold               686                      685   \n",
       "...         ...           ...               ...                      ...   \n",
       "208588     cool          hold               737                      760   \n",
       "208589     cool          hold               733                      760   \n",
       "208590     cool          hold               730                      760   \n",
       "208591     cool          hold               733                      760   \n",
       "208592     cool          hold               740                      760   \n",
       "\n",
       "        TemperatureExpectedHeat ProvinceState          City  \\\n",
       "0                           687            NE       Lincoln   \n",
       "1                           687            NE       Lincoln   \n",
       "2                           683            NE  Grand Island   \n",
       "5                           775            NE         Omaha   \n",
       "6                           685            NE       Lincoln   \n",
       "...                         ...           ...           ...   \n",
       "208588                      760            NE      Bellevue   \n",
       "208589                      760            NE      Bellevue   \n",
       "208590                      760            NE      Bellevue   \n",
       "208591                      760            NE      Bellevue   \n",
       "208592                      760            NE      Bellevue   \n",
       "\n",
       "        Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                        25             False         False             True   \n",
       "1                        25             False         False             True   \n",
       "2                        40             False         False            False   \n",
       "5                        10             False         False             True   \n",
       "6                        25             False         False             True   \n",
       "...                     ...               ...           ...              ...   \n",
       "208588                   30             False         False            False   \n",
       "208589                   30             False         False            False   \n",
       "208590                   30             False         False            False   \n",
       "208591                   30             False         False            False   \n",
       "208592                   30             False         False            False   \n",
       "\n",
       "       Auxilliary_Heat_Fuel_Type  \n",
       "0                       Electric  \n",
       "1                       Electric  \n",
       "2                            Gas  \n",
       "5                       Electric  \n",
       "6                       Electric  \n",
       "...                          ...  \n",
       "208588                       Gas  \n",
       "208589                       Gas  \n",
       "208590                       Gas  \n",
       "208591                       Gas  \n",
       "208592                       Gas  \n",
       "\n",
       "[204976 rows x 14 columns]"
      ]
     },
     "execution_count": 108,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "jul_2018.drop(jul_2018[jul_2018['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "jul_2018.drop(jul_2018[jul_2018['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "jul_2018.drop(jul_2018[jul_2018['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "jul_2018.drop(jul_2018[jul_2018['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "jul_2018"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "jul_2018[\"Year\"] = \"2018\"\n",
    "jul_2018[\"Month\"] = \"jul\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "jul_2018 = jul_2018.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 111,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "jul_2018_ave = jul_2018.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# jul_2018_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 112,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "jul_2018_ave.to_csv(\"data/day/NE/jul/jul_2018_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2019 July Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 113,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "jul_2019 = pd.read_csv(\"../data_large/NE-day/2019-jul-day-NE.csv\")\n",
    "\n",
    "# jul_2019"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 114,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>50f777071abb7b1664d734531f57de6ce9007794</td>\n",
       "      <td>2019-07-15 12:35:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>680</td>\n",
       "      <td>742</td>\n",
       "      <td>742</td>\n",
       "      <td>NE</td>\n",
       "      <td>Omaha</td>\n",
       "      <td>30</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2fdc63565f7136c15970a1012e3042ff5cef182f</td>\n",
       "      <td>2019-07-29 15:20:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>741</td>\n",
       "      <td>737</td>\n",
       "      <td>687</td>\n",
       "      <td>NE</td>\n",
       "      <td>Omaha</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>01dbf65be10c41aa3c116d6c0a0a9a8e63658bab</td>\n",
       "      <td>2019-07-01 19:40:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>742</td>\n",
       "      <td>741</td>\n",
       "      <td>640</td>\n",
       "      <td>NE</td>\n",
       "      <td>Omaha</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2504df671d8f2abfb4e209289ba629963f2db33b</td>\n",
       "      <td>2019-07-04 17:15:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>730</td>\n",
       "      <td>728</td>\n",
       "      <td>728</td>\n",
       "      <td>NE</td>\n",
       "      <td>Lincoln</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2504df671d8f2abfb4e209289ba629963f2db33b</td>\n",
       "      <td>2019-07-04 18:05:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>730</td>\n",
       "      <td>728</td>\n",
       "      <td>728</td>\n",
       "      <td>NE</td>\n",
       "      <td>Lincoln</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>246820</th>\n",
       "      <td>ff8d6679e9ce5c1228cf640311c0110b3aafb5bd</td>\n",
       "      <td>2019-07-22 14:40:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>705</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>NE</td>\n",
       "      <td>Omaha</td>\n",
       "      <td>0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>246821</th>\n",
       "      <td>ff8d6679e9ce5c1228cf640311c0110b3aafb5bd</td>\n",
       "      <td>2019-07-30 17:00:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>750</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>NE</td>\n",
       "      <td>Omaha</td>\n",
       "      <td>0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>246822</th>\n",
       "      <td>ff8d6679e9ce5c1228cf640311c0110b3aafb5bd</td>\n",
       "      <td>2019-07-22 16:15:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>708</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>NE</td>\n",
       "      <td>Omaha</td>\n",
       "      <td>0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>246823</th>\n",
       "      <td>ff8d6679e9ce5c1228cf640311c0110b3aafb5bd</td>\n",
       "      <td>2019-07-30 19:55:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>765</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>NE</td>\n",
       "      <td>Omaha</td>\n",
       "      <td>0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>246824</th>\n",
       "      <td>ff8d6679e9ce5c1228cf640311c0110b3aafb5bd</td>\n",
       "      <td>2019-07-30 15:00:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>737</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>NE</td>\n",
       "      <td>Omaha</td>\n",
       "      <td>0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>240420 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                      Identifier                date_time  \\\n",
       "0       50f777071abb7b1664d734531f57de6ce9007794  2019-07-15 12:35:00 UTC   \n",
       "1       2fdc63565f7136c15970a1012e3042ff5cef182f  2019-07-29 15:20:00 UTC   \n",
       "2       01dbf65be10c41aa3c116d6c0a0a9a8e63658bab  2019-07-01 19:40:00 UTC   \n",
       "3       2504df671d8f2abfb4e209289ba629963f2db33b  2019-07-04 17:15:00 UTC   \n",
       "4       2504df671d8f2abfb4e209289ba629963f2db33b  2019-07-04 18:05:00 UTC   \n",
       "...                                          ...                      ...   \n",
       "246820  ff8d6679e9ce5c1228cf640311c0110b3aafb5bd  2019-07-22 14:40:00 UTC   \n",
       "246821  ff8d6679e9ce5c1228cf640311c0110b3aafb5bd  2019-07-30 17:00:00 UTC   \n",
       "246822  ff8d6679e9ce5c1228cf640311c0110b3aafb5bd  2019-07-22 16:15:00 UTC   \n",
       "246823  ff8d6679e9ce5c1228cf640311c0110b3aafb5bd  2019-07-30 19:55:00 UTC   \n",
       "246824  ff8d6679e9ce5c1228cf640311c0110b3aafb5bd  2019-07-30 15:00:00 UTC   \n",
       "\n",
       "       HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0          cool          hold               680                      742   \n",
       "1          auto          auto               741                      737   \n",
       "2          auto          auto               742                      741   \n",
       "3          cool          hold               730                      728   \n",
       "4          cool          hold               730                      728   \n",
       "...         ...           ...               ...                      ...   \n",
       "246820     cool          auto               705                      760   \n",
       "246821     cool          hold               750                      760   \n",
       "246822     cool          auto               708                      760   \n",
       "246823     cool          hold               765                      760   \n",
       "246824     cool          hold               737                      760   \n",
       "\n",
       "        TemperatureExpectedHeat ProvinceState     City  Age_of_Home__years_  \\\n",
       "0                           742            NE    Omaha                   30   \n",
       "1                           687            NE    Omaha                    0   \n",
       "2                           640            NE    Omaha                    0   \n",
       "3                           728            NE  Lincoln                    0   \n",
       "4                           728            NE  Lincoln                    0   \n",
       "...                         ...           ...      ...                  ...   \n",
       "246820                      760            NE    Omaha                    0   \n",
       "246821                      760            NE    Omaha                    0   \n",
       "246822                      760            NE    Omaha                    0   \n",
       "246823                      760            NE    Omaha                    0   \n",
       "246824                      760            NE    Omaha                    0   \n",
       "\n",
       "        allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                  False         False            False   \n",
       "1                  False         False            False   \n",
       "2                  False         False             True   \n",
       "3                  False         False            False   \n",
       "4                  False         False            False   \n",
       "...                  ...           ...              ...   \n",
       "246820              True         False             True   \n",
       "246821              True         False             True   \n",
       "246822              True         False             True   \n",
       "246823              True         False             True   \n",
       "246824              True         False             True   \n",
       "\n",
       "       Auxilliary_Heat_Fuel_Type  \n",
       "0                            Gas  \n",
       "1                            Gas  \n",
       "2                       Electric  \n",
       "3                            Gas  \n",
       "4                            Gas  \n",
       "...                          ...  \n",
       "246820                  Electric  \n",
       "246821                  Electric  \n",
       "246822                  Electric  \n",
       "246823                  Electric  \n",
       "246824                  Electric  \n",
       "\n",
       "[240420 rows x 14 columns]"
      ]
     },
     "execution_count": 114,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "jul_2019.drop(jul_2019[jul_2019['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "jul_2019.drop(jul_2019[jul_2019['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "jul_2019.drop(jul_2019[jul_2019['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "jul_2019.drop(jul_2019[jul_2019['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "jul_2019"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 115,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "jul_2019[\"Year\"] = \"2019\"\n",
    "jul_2019[\"Month\"] = \"jul\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 116,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "jul_2019 = jul_2019.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 117,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "jul_2019_ave = jul_2019.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# jul_2019_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 118,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "jul_2019_ave.to_csv(\"data/day/NE/jul/jul_2019_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2020 July Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 119,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "jul_2020 = pd.read_csv(\"../data_large/NE-day/2020-jul-day-NE.csv\")\n",
    "\n",
    "# jul_2020"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 120,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>ae207699741703a86828265b6556382c300adb71</td>\n",
       "      <td>2020-07-08 18:45:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>738</td>\n",
       "      <td>736</td>\n",
       "      <td>736</td>\n",
       "      <td>NE</td>\n",
       "      <td>Lincoln</td>\n",
       "      <td>95</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>c0893a724ad626c64e22f059c4a1e81afa8396fc</td>\n",
       "      <td>2020-07-14 11:40:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>713</td>\n",
       "      <td>714</td>\n",
       "      <td>664</td>\n",
       "      <td>NE</td>\n",
       "      <td>Omaha</td>\n",
       "      <td>80</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>24557ca6d177aa3b5a71f9110b182dc2dc436af0</td>\n",
       "      <td>2020-07-12 11:50:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>776</td>\n",
       "      <td>795</td>\n",
       "      <td>790</td>\n",
       "      <td>NE</td>\n",
       "      <td>Omaha</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>24557ca6d177aa3b5a71f9110b182dc2dc436af0</td>\n",
       "      <td>2020-07-15 11:30:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>755</td>\n",
       "      <td>795</td>\n",
       "      <td>790</td>\n",
       "      <td>NE</td>\n",
       "      <td>Omaha</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>c0893a724ad626c64e22f059c4a1e81afa8396fc</td>\n",
       "      <td>2020-07-21 16:30:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>720</td>\n",
       "      <td>714</td>\n",
       "      <td>664</td>\n",
       "      <td>NE</td>\n",
       "      <td>Omaha</td>\n",
       "      <td>80</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>245558</th>\n",
       "      <td>ff8d6679e9ce5c1228cf640311c0110b3aafb5bd</td>\n",
       "      <td>2020-07-14 09:55:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>761</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>NE</td>\n",
       "      <td>Omaha</td>\n",
       "      <td>0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>245559</th>\n",
       "      <td>ff8d6679e9ce5c1228cf640311c0110b3aafb5bd</td>\n",
       "      <td>2020-07-30 18:55:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>750</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>NE</td>\n",
       "      <td>Omaha</td>\n",
       "      <td>0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>245560</th>\n",
       "      <td>ff8d6679e9ce5c1228cf640311c0110b3aafb5bd</td>\n",
       "      <td>2020-07-14 12:50:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>761</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>NE</td>\n",
       "      <td>Omaha</td>\n",
       "      <td>0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>245561</th>\n",
       "      <td>ff8d6679e9ce5c1228cf640311c0110b3aafb5bd</td>\n",
       "      <td>2020-07-30 18:15:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>746</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>NE</td>\n",
       "      <td>Omaha</td>\n",
       "      <td>0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>245562</th>\n",
       "      <td>ff8d6679e9ce5c1228cf640311c0110b3aafb5bd</td>\n",
       "      <td>2020-07-14 16:30:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>776</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>NE</td>\n",
       "      <td>Omaha</td>\n",
       "      <td>0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>235438 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                      Identifier                date_time  \\\n",
       "0       ae207699741703a86828265b6556382c300adb71  2020-07-08 18:45:00 UTC   \n",
       "1       c0893a724ad626c64e22f059c4a1e81afa8396fc  2020-07-14 11:40:00 UTC   \n",
       "2       24557ca6d177aa3b5a71f9110b182dc2dc436af0  2020-07-12 11:50:00 UTC   \n",
       "3       24557ca6d177aa3b5a71f9110b182dc2dc436af0  2020-07-15 11:30:00 UTC   \n",
       "4       c0893a724ad626c64e22f059c4a1e81afa8396fc  2020-07-21 16:30:00 UTC   \n",
       "...                                          ...                      ...   \n",
       "245558  ff8d6679e9ce5c1228cf640311c0110b3aafb5bd  2020-07-14 09:55:00 UTC   \n",
       "245559  ff8d6679e9ce5c1228cf640311c0110b3aafb5bd  2020-07-30 18:55:00 UTC   \n",
       "245560  ff8d6679e9ce5c1228cf640311c0110b3aafb5bd  2020-07-14 12:50:00 UTC   \n",
       "245561  ff8d6679e9ce5c1228cf640311c0110b3aafb5bd  2020-07-30 18:15:00 UTC   \n",
       "245562  ff8d6679e9ce5c1228cf640311c0110b3aafb5bd  2020-07-14 16:30:00 UTC   \n",
       "\n",
       "       HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0          cool          hold               738                      736   \n",
       "1          auto          auto               713                      714   \n",
       "2          cool          hold               776                      795   \n",
       "3          cool          hold               755                      795   \n",
       "4          auto          auto               720                      714   \n",
       "...         ...           ...               ...                      ...   \n",
       "245558     cool          hold               761                      760   \n",
       "245559     cool          hold               750                      760   \n",
       "245560     cool          hold               761                      760   \n",
       "245561     cool          hold               746                      760   \n",
       "245562     cool          hold               776                      760   \n",
       "\n",
       "        TemperatureExpectedHeat ProvinceState     City  Age_of_Home__years_  \\\n",
       "0                           736            NE  Lincoln                   95   \n",
       "1                           664            NE    Omaha                   80   \n",
       "2                           790            NE    Omaha                   10   \n",
       "3                           790            NE    Omaha                   10   \n",
       "4                           664            NE    Omaha                   80   \n",
       "...                         ...           ...      ...                  ...   \n",
       "245558                      760            NE    Omaha                    0   \n",
       "245559                      760            NE    Omaha                    0   \n",
       "245560                      760            NE    Omaha                    0   \n",
       "245561                      760            NE    Omaha                    0   \n",
       "245562                      760            NE    Omaha                    0   \n",
       "\n",
       "        allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                  False         False            False   \n",
       "1                  False         False            False   \n",
       "2                  False         False             True   \n",
       "3                  False         False             True   \n",
       "4                  False         False            False   \n",
       "...                  ...           ...              ...   \n",
       "245558              True         False             True   \n",
       "245559              True         False             True   \n",
       "245560              True         False             True   \n",
       "245561              True         False             True   \n",
       "245562              True         False             True   \n",
       "\n",
       "       Auxilliary_Heat_Fuel_Type  \n",
       "0                            Gas  \n",
       "1                            Gas  \n",
       "2                       Electric  \n",
       "3                       Electric  \n",
       "4                            Gas  \n",
       "...                          ...  \n",
       "245558                  Electric  \n",
       "245559                  Electric  \n",
       "245560                  Electric  \n",
       "245561                  Electric  \n",
       "245562                  Electric  \n",
       "\n",
       "[235438 rows x 14 columns]"
      ]
     },
     "execution_count": 120,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "jul_2020.drop(jul_2020[jul_2020['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "jul_2020.drop(jul_2020[jul_2020['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "jul_2020.drop(jul_2020[jul_2020['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "jul_2020.drop(jul_2020[jul_2020['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "jul_2020"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 121,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "jul_2020[\"Year\"] = \"2020\"\n",
    "jul_2020[\"Month\"] = \"jul\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 122,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "jul_2020 = jul_2020.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 123,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "jul_2020_ave = jul_2020.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# jul_2020_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 124,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "jul_2020_ave.to_csv(\"data/day/NE/jul/jul_2020_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2021 July Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 125,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "jul_2021 = pd.read_csv(\"../data_large/NE-day/2021-jul-day-NE.csv\")\n",
    "\n",
    "# jul_2021"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 126,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>cf6446120e29e89c9aa494dff46a008db0ee948b</td>\n",
       "      <td>2021-07-06 12:35:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>751</td>\n",
       "      <td>752</td>\n",
       "      <td>752</td>\n",
       "      <td>NE</td>\n",
       "      <td>Bellevue</td>\n",
       "      <td>55</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>d0040049f0105d39f159341c12d1f054719fb53f</td>\n",
       "      <td>2021-07-25 11:50:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>664</td>\n",
       "      <td>665</td>\n",
       "      <td>665</td>\n",
       "      <td>NE</td>\n",
       "      <td>Elkhorn</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>c039f7f15fbd5d98b80efba2631acb48bef7d066</td>\n",
       "      <td>2021-07-14 11:15:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>702</td>\n",
       "      <td>703</td>\n",
       "      <td>653</td>\n",
       "      <td>NE</td>\n",
       "      <td>Bennington</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>faf3b76e8b0193bcf0256358efb4e593b669a1c7</td>\n",
       "      <td>2021-07-14 11:45:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>780</td>\n",
       "      <td>776</td>\n",
       "      <td>730</td>\n",
       "      <td>NE</td>\n",
       "      <td>Hickman</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>faf3b76e8b0193bcf0256358efb4e593b669a1c7</td>\n",
       "      <td>2021-07-27 16:50:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>746</td>\n",
       "      <td>746</td>\n",
       "      <td>730</td>\n",
       "      <td>NE</td>\n",
       "      <td>Hickman</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>145926</th>\n",
       "      <td>c1c075cfc90c4712fe95f5afb28d980c6c427631</td>\n",
       "      <td>2021-07-05 08:10:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>782</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>NE</td>\n",
       "      <td>Lincoln</td>\n",
       "      <td>35</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>145927</th>\n",
       "      <td>c1c075cfc90c4712fe95f5afb28d980c6c427631</td>\n",
       "      <td>2021-07-04 17:50:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>787</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>NE</td>\n",
       "      <td>Lincoln</td>\n",
       "      <td>35</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>145928</th>\n",
       "      <td>c1c075cfc90c4712fe95f5afb28d980c6c427631</td>\n",
       "      <td>2021-07-04 19:45:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>772</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>NE</td>\n",
       "      <td>Lincoln</td>\n",
       "      <td>35</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>145929</th>\n",
       "      <td>c1c075cfc90c4712fe95f5afb28d980c6c427631</td>\n",
       "      <td>2021-07-03 17:15:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>775</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>NE</td>\n",
       "      <td>Lincoln</td>\n",
       "      <td>35</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>145930</th>\n",
       "      <td>c1c075cfc90c4712fe95f5afb28d980c6c427631</td>\n",
       "      <td>2021-07-05 09:20:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>780</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>NE</td>\n",
       "      <td>Lincoln</td>\n",
       "      <td>35</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>143236 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                      Identifier                date_time  \\\n",
       "0       cf6446120e29e89c9aa494dff46a008db0ee948b  2021-07-06 12:35:00 UTC   \n",
       "1       d0040049f0105d39f159341c12d1f054719fb53f  2021-07-25 11:50:00 UTC   \n",
       "2       c039f7f15fbd5d98b80efba2631acb48bef7d066  2021-07-14 11:15:00 UTC   \n",
       "3       faf3b76e8b0193bcf0256358efb4e593b669a1c7  2021-07-14 11:45:00 UTC   \n",
       "4       faf3b76e8b0193bcf0256358efb4e593b669a1c7  2021-07-27 16:50:00 UTC   \n",
       "...                                          ...                      ...   \n",
       "145926  c1c075cfc90c4712fe95f5afb28d980c6c427631  2021-07-05 08:10:00 UTC   \n",
       "145927  c1c075cfc90c4712fe95f5afb28d980c6c427631  2021-07-04 17:50:00 UTC   \n",
       "145928  c1c075cfc90c4712fe95f5afb28d980c6c427631  2021-07-04 19:45:00 UTC   \n",
       "145929  c1c075cfc90c4712fe95f5afb28d980c6c427631  2021-07-03 17:15:00 UTC   \n",
       "145930  c1c075cfc90c4712fe95f5afb28d980c6c427631  2021-07-05 09:20:00 UTC   \n",
       "\n",
       "       HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0          cool          hold               751                      752   \n",
       "1          cool          hold               664                      665   \n",
       "2          auto          hold               702                      703   \n",
       "3          cool          hold               780                      776   \n",
       "4          cool          hold               746                      746   \n",
       "...         ...           ...               ...                      ...   \n",
       "145926     cool          hold               782                      760   \n",
       "145927     cool          hold               787                      760   \n",
       "145928     cool          hold               772                      760   \n",
       "145929     cool          hold               775                      760   \n",
       "145930     cool          hold               780                      760   \n",
       "\n",
       "        TemperatureExpectedHeat ProvinceState        City  \\\n",
       "0                           752            NE    Bellevue   \n",
       "1                           665            NE     Elkhorn   \n",
       "2                           653            NE  Bennington   \n",
       "3                           730            NE     Hickman   \n",
       "4                           730            NE     Hickman   \n",
       "...                         ...           ...         ...   \n",
       "145926                      760            NE     Lincoln   \n",
       "145927                      760            NE     Lincoln   \n",
       "145928                      760            NE     Lincoln   \n",
       "145929                      760            NE     Lincoln   \n",
       "145930                      760            NE     Lincoln   \n",
       "\n",
       "        Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                        55             False         False            False   \n",
       "1                         0             False         False            False   \n",
       "2                         0             False         False            False   \n",
       "3                         0             False         False             True   \n",
       "4                         0             False         False             True   \n",
       "...                     ...               ...           ...              ...   \n",
       "145926                   35             False         False            False   \n",
       "145927                   35             False         False            False   \n",
       "145928                   35             False         False            False   \n",
       "145929                   35             False         False            False   \n",
       "145930                   35             False         False            False   \n",
       "\n",
       "       Auxilliary_Heat_Fuel_Type  \n",
       "0                            Gas  \n",
       "1                            Gas  \n",
       "2                            Gas  \n",
       "3                       Electric  \n",
       "4                       Electric  \n",
       "...                          ...  \n",
       "145926                       Gas  \n",
       "145927                       Gas  \n",
       "145928                       Gas  \n",
       "145929                       Gas  \n",
       "145930                       Gas  \n",
       "\n",
       "[143236 rows x 14 columns]"
      ]
     },
     "execution_count": 126,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "jul_2021.drop(jul_2021[jul_2021['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "jul_2021.drop(jul_2021[jul_2021['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "jul_2021.drop(jul_2021[jul_2021['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "jul_2021.drop(jul_2021[jul_2021['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "\n",
    "jul_2021"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 127,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "jul_2021[\"Year\"] = \"2021\"\n",
    "jul_2021[\"Month\"] = \"jul\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 128,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "jul_2021 = jul_2021.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 129,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "jul_2021_ave = jul_2021.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# jul_2021_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 130,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "jul_2021_ave.to_csv(\"data/day/NE/jul/jul_2021_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "\n",
    "### Combine month CSV Files \n",
    "1. Read in files in folders for each state\n",
    "2. Export as combined CSV"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 131,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create variable for files in directory\n",
    "files = [f for f in os.listdir(\"data/day/NE/jul/\") if f.endswith(\".csv\")]\n",
    "\n",
    "# files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 132,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>Month</th>\n",
       "      <th>Year</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>City</th>\n",
       "      <th>Temperature_ctrl_ave</th>\n",
       "      <th>TemperatureExpectedCool_ave</th>\n",
       "      <th>TemperatureExpectedHeat_ave</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>033afd975854035a4b0ab320768f094d2b716859</td>\n",
       "      <td>jul</td>\n",
       "      <td>2017</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>La Vista</td>\n",
       "      <td>673.833333</td>\n",
       "      <td>670.041667</td>\n",
       "      <td>620.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>05320d509a18b2d4022029409e7a267736e78e0b</td>\n",
       "      <td>jul</td>\n",
       "      <td>2017</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>Lincoln</td>\n",
       "      <td>734.333333</td>\n",
       "      <td>755.000000</td>\n",
       "      <td>692.000000</td>\n",
       "      <td>90.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>05320d509a18b2d4022029409e7a267736e78e0b</td>\n",
       "      <td>jul</td>\n",
       "      <td>2017</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>Lincoln</td>\n",
       "      <td>729.119512</td>\n",
       "      <td>734.965854</td>\n",
       "      <td>734.034146</td>\n",
       "      <td>90.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0ad90268dcfa7e7aa1d92768a5d17e1841ceaf17</td>\n",
       "      <td>jul</td>\n",
       "      <td>2017</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>McCool Junction</td>\n",
       "      <td>759.255230</td>\n",
       "      <td>768.619247</td>\n",
       "      <td>768.573222</td>\n",
       "      <td>15.0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0d779ed5bbc93a32506824e72f1b051e9980a0cf</td>\n",
       "      <td>jul</td>\n",
       "      <td>2017</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>Blair</td>\n",
       "      <td>769.466667</td>\n",
       "      <td>760.000000</td>\n",
       "      <td>690.000000</td>\n",
       "      <td>35.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>193</th>\n",
       "      <td>fa41ecba2896263b0616f05220abba8af9cd378d</td>\n",
       "      <td>jul</td>\n",
       "      <td>2021</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>Geneva</td>\n",
       "      <td>744.916667</td>\n",
       "      <td>740.375000</td>\n",
       "      <td>738.583333</td>\n",
       "      <td>100.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>194</th>\n",
       "      <td>faa85f28edb3a9a38ac6130d63b1c3f892466036</td>\n",
       "      <td>jul</td>\n",
       "      <td>2021</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>Omaha</td>\n",
       "      <td>717.452055</td>\n",
       "      <td>687.735160</td>\n",
       "      <td>685.063927</td>\n",
       "      <td>55.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>195</th>\n",
       "      <td>faa8c435b84d052676a86ba7d2d6a783d26e6b19</td>\n",
       "      <td>jul</td>\n",
       "      <td>2021</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>Omaha</td>\n",
       "      <td>695.957746</td>\n",
       "      <td>691.957746</td>\n",
       "      <td>691.943662</td>\n",
       "      <td>20.0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>196</th>\n",
       "      <td>faf3b76e8b0193bcf0256358efb4e593b669a1c7</td>\n",
       "      <td>jul</td>\n",
       "      <td>2021</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>Hickman</td>\n",
       "      <td>762.708908</td>\n",
       "      <td>763.499373</td>\n",
       "      <td>729.982434</td>\n",
       "      <td>0.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>197</th>\n",
       "      <td>ff8d6679e9ce5c1228cf640311c0110b3aafb5bd</td>\n",
       "      <td>jul</td>\n",
       "      <td>2021</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>Omaha</td>\n",
       "      <td>723.204791</td>\n",
       "      <td>721.312210</td>\n",
       "      <td>721.312210</td>\n",
       "      <td>0.0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1483 rows × 13 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                   Identifier Month  Year HvacMode  \\\n",
       "0    033afd975854035a4b0ab320768f094d2b716859   jul  2017     auto   \n",
       "1    05320d509a18b2d4022029409e7a267736e78e0b   jul  2017     cool   \n",
       "2    05320d509a18b2d4022029409e7a267736e78e0b   jul  2017     cool   \n",
       "3    0ad90268dcfa7e7aa1d92768a5d17e1841ceaf17   jul  2017     cool   \n",
       "4    0d779ed5bbc93a32506824e72f1b051e9980a0cf   jul  2017     cool   \n",
       "..                                        ...   ...   ...      ...   \n",
       "193  fa41ecba2896263b0616f05220abba8af9cd378d   jul  2021     cool   \n",
       "194  faa85f28edb3a9a38ac6130d63b1c3f892466036   jul  2021     cool   \n",
       "195  faa8c435b84d052676a86ba7d2d6a783d26e6b19   jul  2021     cool   \n",
       "196  faf3b76e8b0193bcf0256358efb4e593b669a1c7   jul  2021     cool   \n",
       "197  ff8d6679e9ce5c1228cf640311c0110b3aafb5bd   jul  2021     cool   \n",
       "\n",
       "    CalendarEvent             City  Temperature_ctrl_ave  \\\n",
       "0            hold         La Vista            673.833333   \n",
       "1            auto          Lincoln            734.333333   \n",
       "2            hold          Lincoln            729.119512   \n",
       "3            hold  McCool Junction            759.255230   \n",
       "4            auto            Blair            769.466667   \n",
       "..            ...              ...                   ...   \n",
       "193          hold           Geneva            744.916667   \n",
       "194          hold            Omaha            717.452055   \n",
       "195          hold            Omaha            695.957746   \n",
       "196          hold          Hickman            762.708908   \n",
       "197          hold            Omaha            723.204791   \n",
       "\n",
       "     TemperatureExpectedCool_ave  TemperatureExpectedHeat_ave  \\\n",
       "0                     670.041667                   620.000000   \n",
       "1                     755.000000                   692.000000   \n",
       "2                     734.965854                   734.034146   \n",
       "3                     768.619247                   768.573222   \n",
       "4                     760.000000                   690.000000   \n",
       "..                           ...                          ...   \n",
       "193                   740.375000                   738.583333   \n",
       "194                   687.735160                   685.063927   \n",
       "195                   691.957746                   691.943662   \n",
       "196                   763.499373                   729.982434   \n",
       "197                   721.312210                   721.312210   \n",
       "\n",
       "     Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \n",
       "0                    0.0             False         False            False  \n",
       "1                   90.0             False         False            False  \n",
       "2                   90.0             False         False            False  \n",
       "3                   15.0              True         False             True  \n",
       "4                   35.0             False         False            False  \n",
       "..                   ...               ...           ...              ...  \n",
       "193                100.0             False         False            False  \n",
       "194                 55.0             False         False            False  \n",
       "195                 20.0              True         False            False  \n",
       "196                  0.0             False         False             True  \n",
       "197                  0.0              True         False             True  \n",
       "\n",
       "[1483 rows x 13 columns]"
      ]
     },
     "execution_count": 132,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# https://stackoverflow.com/questions/63886787/how-to-create-a-dataframe-from-multiple-csv-files\n",
    "# Initialize an empty dataframe\n",
    "NE_jul = pd.DataFrame()\n",
    "\n",
    "# Iterate through files and contents, then concatenate the data into the dataframe\n",
    "for file in files:\n",
    "    df = pd.read_csv(\"data/day/NE/jul/\" + file)\n",
    "    NE_jul = pd.concat([NE_jul, df])\n",
    "    \n",
    "NE_jul"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 133,
   "metadata": {},
   "outputs": [],
   "source": [
    "NE_jul.to_csv(\"Scraper_Output/State_Month_Day/NE/NE_jul.csv\", header=True, index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "\n",
    "## August"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2017 August Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 134,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "aug_2017 = pd.read_csv(\"../data_large/NE-day/2017-aug-day-NE.csv\")\n",
    "\n",
    "# aug_2017"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 135,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>50f777071abb7b1664d734531f57de6ce9007794</td>\n",
       "      <td>2017-08-31 19:15:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>760</td>\n",
       "      <td>752</td>\n",
       "      <td>752</td>\n",
       "      <td>NE</td>\n",
       "      <td>Omaha</td>\n",
       "      <td>30</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>50f777071abb7b1664d734531f57de6ce9007794</td>\n",
       "      <td>2017-08-29 18:10:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>757</td>\n",
       "      <td>752</td>\n",
       "      <td>752</td>\n",
       "      <td>NE</td>\n",
       "      <td>Omaha</td>\n",
       "      <td>30</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>3bc5a1ea03c7caced8b4b1fa3d54e9ec8c3f951b</td>\n",
       "      <td>2017-08-12 13:10:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>740</td>\n",
       "      <td>840</td>\n",
       "      <td>700</td>\n",
       "      <td>NE</td>\n",
       "      <td>North Platte</td>\n",
       "      <td>30</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>50f777071abb7b1664d734531f57de6ce9007794</td>\n",
       "      <td>2017-08-31 15:20:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>741</td>\n",
       "      <td>742</td>\n",
       "      <td>742</td>\n",
       "      <td>NE</td>\n",
       "      <td>Omaha</td>\n",
       "      <td>30</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>50f777071abb7b1664d734531f57de6ce9007794</td>\n",
       "      <td>2017-08-15 16:10:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>730</td>\n",
       "      <td>731</td>\n",
       "      <td>731</td>\n",
       "      <td>NE</td>\n",
       "      <td>Omaha</td>\n",
       "      <td>30</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>84442</th>\n",
       "      <td>c11897bdaace2de8866bb11023b1f476b267fa27</td>\n",
       "      <td>2017-08-31 15:50:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>756</td>\n",
       "      <td>760</td>\n",
       "      <td>700</td>\n",
       "      <td>NE</td>\n",
       "      <td>Omaha</td>\n",
       "      <td>15</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>84443</th>\n",
       "      <td>c11897bdaace2de8866bb11023b1f476b267fa27</td>\n",
       "      <td>2017-08-30 18:35:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>764</td>\n",
       "      <td>760</td>\n",
       "      <td>700</td>\n",
       "      <td>NE</td>\n",
       "      <td>Omaha</td>\n",
       "      <td>15</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>84444</th>\n",
       "      <td>c11897bdaace2de8866bb11023b1f476b267fa27</td>\n",
       "      <td>2017-08-12 18:00:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>761</td>\n",
       "      <td>760</td>\n",
       "      <td>700</td>\n",
       "      <td>NE</td>\n",
       "      <td>Omaha</td>\n",
       "      <td>15</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>84445</th>\n",
       "      <td>c11897bdaace2de8866bb11023b1f476b267fa27</td>\n",
       "      <td>2017-08-11 17:25:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>764</td>\n",
       "      <td>760</td>\n",
       "      <td>700</td>\n",
       "      <td>NE</td>\n",
       "      <td>Omaha</td>\n",
       "      <td>15</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>84446</th>\n",
       "      <td>c11897bdaace2de8866bb11023b1f476b267fa27</td>\n",
       "      <td>2017-08-27 18:05:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>700</td>\n",
       "      <td>NE</td>\n",
       "      <td>Omaha</td>\n",
       "      <td>15</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>81341 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                     Identifier                date_time  \\\n",
       "0      50f777071abb7b1664d734531f57de6ce9007794  2017-08-31 19:15:00 UTC   \n",
       "1      50f777071abb7b1664d734531f57de6ce9007794  2017-08-29 18:10:00 UTC   \n",
       "4      3bc5a1ea03c7caced8b4b1fa3d54e9ec8c3f951b  2017-08-12 13:10:00 UTC   \n",
       "5      50f777071abb7b1664d734531f57de6ce9007794  2017-08-31 15:20:00 UTC   \n",
       "7      50f777071abb7b1664d734531f57de6ce9007794  2017-08-15 16:10:00 UTC   \n",
       "...                                         ...                      ...   \n",
       "84442  c11897bdaace2de8866bb11023b1f476b267fa27  2017-08-31 15:50:00 UTC   \n",
       "84443  c11897bdaace2de8866bb11023b1f476b267fa27  2017-08-30 18:35:00 UTC   \n",
       "84444  c11897bdaace2de8866bb11023b1f476b267fa27  2017-08-12 18:00:00 UTC   \n",
       "84445  c11897bdaace2de8866bb11023b1f476b267fa27  2017-08-11 17:25:00 UTC   \n",
       "84446  c11897bdaace2de8866bb11023b1f476b267fa27  2017-08-27 18:05:00 UTC   \n",
       "\n",
       "      HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0         cool          hold               760                      752   \n",
       "1         cool          hold               757                      752   \n",
       "4         cool          auto               740                      840   \n",
       "5         cool          hold               741                      742   \n",
       "7         cool          hold               730                      731   \n",
       "...        ...           ...               ...                      ...   \n",
       "84442     auto          hold               756                      760   \n",
       "84443     auto          hold               764                      760   \n",
       "84444     auto          hold               761                      760   \n",
       "84445     auto          hold               764                      760   \n",
       "84446     auto          hold               760                      760   \n",
       "\n",
       "       TemperatureExpectedHeat ProvinceState          City  \\\n",
       "0                          752            NE         Omaha   \n",
       "1                          752            NE         Omaha   \n",
       "4                          700            NE  North Platte   \n",
       "5                          742            NE         Omaha   \n",
       "7                          731            NE         Omaha   \n",
       "...                        ...           ...           ...   \n",
       "84442                      700            NE         Omaha   \n",
       "84443                      700            NE         Omaha   \n",
       "84444                      700            NE         Omaha   \n",
       "84445                      700            NE         Omaha   \n",
       "84446                      700            NE         Omaha   \n",
       "\n",
       "       Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                       30             False         False            False   \n",
       "1                       30             False         False            False   \n",
       "4                       30             False         False            False   \n",
       "5                       30             False         False            False   \n",
       "7                       30             False         False            False   \n",
       "...                    ...               ...           ...              ...   \n",
       "84442                   15             False         False            False   \n",
       "84443                   15             False         False            False   \n",
       "84444                   15             False         False            False   \n",
       "84445                   15             False         False            False   \n",
       "84446                   15             False         False            False   \n",
       "\n",
       "      Auxilliary_Heat_Fuel_Type  \n",
       "0                           Gas  \n",
       "1                           Gas  \n",
       "4                           Gas  \n",
       "5                           Gas  \n",
       "7                           Gas  \n",
       "...                         ...  \n",
       "84442                       Gas  \n",
       "84443                       Gas  \n",
       "84444                       Gas  \n",
       "84445                       Gas  \n",
       "84446                       Gas  \n",
       "\n",
       "[81341 rows x 14 columns]"
      ]
     },
     "execution_count": 135,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "aug_2017.drop(aug_2017[aug_2017['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "aug_2017.drop(aug_2017[aug_2017['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "aug_2017.drop(aug_2017[aug_2017['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "aug_2017.drop(aug_2017[aug_2017['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "aug_2017"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 136,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "aug_2017[\"Year\"] = \"2017\"\n",
    "aug_2017[\"Month\"] = \"aug\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 137,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "aug_2017 = aug_2017.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 138,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "aug_2017_ave = aug_2017.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# aug_2017_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 139,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "aug_2017_ave.to_csv(\"data/day/NE/aug/aug_2017_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2018 August Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 140,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "aug_2018 = pd.read_csv(\"../data_large/NE-day/2018-aug-day-NE.csv\")\n",
    "\n",
    "# aug_2018"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 141,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>197f6257e48f1d14dffab296917ddf2c3090264b</td>\n",
       "      <td>2018-08-25 16:25:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>736</td>\n",
       "      <td>732</td>\n",
       "      <td>720</td>\n",
       "      <td>NE</td>\n",
       "      <td>Omaha</td>\n",
       "      <td>15</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>24557ca6d177aa3b5a71f9110b182dc2dc436af0</td>\n",
       "      <td>2018-08-02 13:25:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>765</td>\n",
       "      <td>775</td>\n",
       "      <td>775</td>\n",
       "      <td>NE</td>\n",
       "      <td>Omaha</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>6c71018df5d7d3ec67c41d961ba0a0bbdd3a9e19</td>\n",
       "      <td>2018-08-06 17:05:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>687</td>\n",
       "      <td>687</td>\n",
       "      <td>687</td>\n",
       "      <td>NE</td>\n",
       "      <td>Lincoln</td>\n",
       "      <td>25</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>6c71018df5d7d3ec67c41d961ba0a0bbdd3a9e19</td>\n",
       "      <td>2018-08-02 13:55:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>683</td>\n",
       "      <td>687</td>\n",
       "      <td>687</td>\n",
       "      <td>NE</td>\n",
       "      <td>Lincoln</td>\n",
       "      <td>25</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>50f777071abb7b1664d734531f57de6ce9007794</td>\n",
       "      <td>2018-08-13 16:05:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>725</td>\n",
       "      <td>719</td>\n",
       "      <td>719</td>\n",
       "      <td>NE</td>\n",
       "      <td>Omaha</td>\n",
       "      <td>30</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>190345</th>\n",
       "      <td>ff8d6679e9ce5c1228cf640311c0110b3aafb5bd</td>\n",
       "      <td>2018-08-26 13:05:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>732</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>NE</td>\n",
       "      <td>Omaha</td>\n",
       "      <td>0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>190346</th>\n",
       "      <td>ff8d6679e9ce5c1228cf640311c0110b3aafb5bd</td>\n",
       "      <td>2018-08-26 18:45:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>763</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>NE</td>\n",
       "      <td>Omaha</td>\n",
       "      <td>0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>190347</th>\n",
       "      <td>ff8d6679e9ce5c1228cf640311c0110b3aafb5bd</td>\n",
       "      <td>2018-08-26 15:35:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>744</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>NE</td>\n",
       "      <td>Omaha</td>\n",
       "      <td>0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>190348</th>\n",
       "      <td>ff8d6679e9ce5c1228cf640311c0110b3aafb5bd</td>\n",
       "      <td>2018-08-26 16:45:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>748</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>NE</td>\n",
       "      <td>Omaha</td>\n",
       "      <td>0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>190349</th>\n",
       "      <td>ff8d6679e9ce5c1228cf640311c0110b3aafb5bd</td>\n",
       "      <td>2018-08-26 15:25:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>743</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>NE</td>\n",
       "      <td>Omaha</td>\n",
       "      <td>0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>187126 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                      Identifier                date_time  \\\n",
       "0       197f6257e48f1d14dffab296917ddf2c3090264b  2018-08-25 16:25:00 UTC   \n",
       "1       24557ca6d177aa3b5a71f9110b182dc2dc436af0  2018-08-02 13:25:00 UTC   \n",
       "2       6c71018df5d7d3ec67c41d961ba0a0bbdd3a9e19  2018-08-06 17:05:00 UTC   \n",
       "3       6c71018df5d7d3ec67c41d961ba0a0bbdd3a9e19  2018-08-02 13:55:00 UTC   \n",
       "4       50f777071abb7b1664d734531f57de6ce9007794  2018-08-13 16:05:00 UTC   \n",
       "...                                          ...                      ...   \n",
       "190345  ff8d6679e9ce5c1228cf640311c0110b3aafb5bd  2018-08-26 13:05:00 UTC   \n",
       "190346  ff8d6679e9ce5c1228cf640311c0110b3aafb5bd  2018-08-26 18:45:00 UTC   \n",
       "190347  ff8d6679e9ce5c1228cf640311c0110b3aafb5bd  2018-08-26 15:35:00 UTC   \n",
       "190348  ff8d6679e9ce5c1228cf640311c0110b3aafb5bd  2018-08-26 16:45:00 UTC   \n",
       "190349  ff8d6679e9ce5c1228cf640311c0110b3aafb5bd  2018-08-26 15:25:00 UTC   \n",
       "\n",
       "       HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0          cool          hold               736                      732   \n",
       "1          cool          hold               765                      775   \n",
       "2          cool          hold               687                      687   \n",
       "3          cool          hold               683                      687   \n",
       "4          cool          hold               725                      719   \n",
       "...         ...           ...               ...                      ...   \n",
       "190345     cool          hold               732                      760   \n",
       "190346     cool          hold               763                      760   \n",
       "190347     cool          hold               744                      760   \n",
       "190348     cool          hold               748                      760   \n",
       "190349     cool          hold               743                      760   \n",
       "\n",
       "        TemperatureExpectedHeat ProvinceState     City  Age_of_Home__years_  \\\n",
       "0                           720            NE    Omaha                   15   \n",
       "1                           775            NE    Omaha                   10   \n",
       "2                           687            NE  Lincoln                   25   \n",
       "3                           687            NE  Lincoln                   25   \n",
       "4                           719            NE    Omaha                   30   \n",
       "...                         ...           ...      ...                  ...   \n",
       "190345                      760            NE    Omaha                    0   \n",
       "190346                      760            NE    Omaha                    0   \n",
       "190347                      760            NE    Omaha                    0   \n",
       "190348                      760            NE    Omaha                    0   \n",
       "190349                      760            NE    Omaha                    0   \n",
       "\n",
       "        allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                  False         False            False   \n",
       "1                  False         False             True   \n",
       "2                  False         False             True   \n",
       "3                  False         False             True   \n",
       "4                  False         False            False   \n",
       "...                  ...           ...              ...   \n",
       "190345              True         False             True   \n",
       "190346              True         False             True   \n",
       "190347              True         False             True   \n",
       "190348              True         False             True   \n",
       "190349              True         False             True   \n",
       "\n",
       "       Auxilliary_Heat_Fuel_Type  \n",
       "0                            Gas  \n",
       "1                       Electric  \n",
       "2                       Electric  \n",
       "3                       Electric  \n",
       "4                            Gas  \n",
       "...                          ...  \n",
       "190345                  Electric  \n",
       "190346                  Electric  \n",
       "190347                  Electric  \n",
       "190348                  Electric  \n",
       "190349                  Electric  \n",
       "\n",
       "[187126 rows x 14 columns]"
      ]
     },
     "execution_count": 141,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "aug_2018.drop(aug_2018[aug_2018['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "aug_2018.drop(aug_2018[aug_2018['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "aug_2018.drop(aug_2018[aug_2018['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "aug_2018.drop(aug_2018[aug_2018['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "aug_2018"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 142,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "aug_2018[\"Year\"] = \"2018\"\n",
    "aug_2018[\"Month\"] = \"aug\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 143,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "aug_2018 = aug_2018.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 144,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "aug_2018_ave = aug_2018.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# aug_2018_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 145,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "aug_2018_ave.to_csv(\"data/day/NE/aug/aug_2018_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2019 August Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 146,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "aug_2019 = pd.read_csv(\"../data_large/NE-day/2019-aug-day-NE.csv\")\n",
    "\n",
    "# aug_2019"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 147,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>64468e2b4ce43bb003d5ebb297af917298d85ddf</td>\n",
       "      <td>2019-08-27 12:25:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>751</td>\n",
       "      <td>765</td>\n",
       "      <td>715</td>\n",
       "      <td>NE</td>\n",
       "      <td>Lincoln</td>\n",
       "      <td>39</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>e6c7a7449be199e76934eb1df4e97a561d4ad874</td>\n",
       "      <td>2019-08-02 11:10:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>765</td>\n",
       "      <td>830</td>\n",
       "      <td>790</td>\n",
       "      <td>NE</td>\n",
       "      <td>Nebraska City</td>\n",
       "      <td>85</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>bb17dd6b682b51daaffefc2585ef323aab7828e3</td>\n",
       "      <td>2019-08-04 18:05:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>725</td>\n",
       "      <td>721</td>\n",
       "      <td>721</td>\n",
       "      <td>NE</td>\n",
       "      <td>Omaha</td>\n",
       "      <td>50</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>e6c7a7449be199e76934eb1df4e97a561d4ad874</td>\n",
       "      <td>2019-08-31 14:15:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>753</td>\n",
       "      <td>830</td>\n",
       "      <td>790</td>\n",
       "      <td>NE</td>\n",
       "      <td>Nebraska City</td>\n",
       "      <td>85</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>7b9c42484b70b04b51eca1379ebd7dbca68ac432</td>\n",
       "      <td>2019-08-06 08:35:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>721</td>\n",
       "      <td>716</td>\n",
       "      <td>646</td>\n",
       "      <td>NE</td>\n",
       "      <td>Papillion</td>\n",
       "      <td>5</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>246991</th>\n",
       "      <td>ff8d6679e9ce5c1228cf640311c0110b3aafb5bd</td>\n",
       "      <td>2019-08-06 08:35:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>764</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>NE</td>\n",
       "      <td>Omaha</td>\n",
       "      <td>0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>246992</th>\n",
       "      <td>ff8d6679e9ce5c1228cf640311c0110b3aafb5bd</td>\n",
       "      <td>2019-08-05 16:15:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>765</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>NE</td>\n",
       "      <td>Omaha</td>\n",
       "      <td>0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>246993</th>\n",
       "      <td>ff8d6679e9ce5c1228cf640311c0110b3aafb5bd</td>\n",
       "      <td>2019-08-06 11:40:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>754</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>NE</td>\n",
       "      <td>Omaha</td>\n",
       "      <td>0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>246994</th>\n",
       "      <td>ff8d6679e9ce5c1228cf640311c0110b3aafb5bd</td>\n",
       "      <td>2019-08-19 13:00:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>699</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>NE</td>\n",
       "      <td>Omaha</td>\n",
       "      <td>0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>246995</th>\n",
       "      <td>ff8d6679e9ce5c1228cf640311c0110b3aafb5bd</td>\n",
       "      <td>2019-08-06 08:05:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>764</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>NE</td>\n",
       "      <td>Omaha</td>\n",
       "      <td>0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>239633 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                      Identifier                date_time  \\\n",
       "0       64468e2b4ce43bb003d5ebb297af917298d85ddf  2019-08-27 12:25:00 UTC   \n",
       "1       e6c7a7449be199e76934eb1df4e97a561d4ad874  2019-08-02 11:10:00 UTC   \n",
       "3       bb17dd6b682b51daaffefc2585ef323aab7828e3  2019-08-04 18:05:00 UTC   \n",
       "4       e6c7a7449be199e76934eb1df4e97a561d4ad874  2019-08-31 14:15:00 UTC   \n",
       "5       7b9c42484b70b04b51eca1379ebd7dbca68ac432  2019-08-06 08:35:00 UTC   \n",
       "...                                          ...                      ...   \n",
       "246991  ff8d6679e9ce5c1228cf640311c0110b3aafb5bd  2019-08-06 08:35:00 UTC   \n",
       "246992  ff8d6679e9ce5c1228cf640311c0110b3aafb5bd  2019-08-05 16:15:00 UTC   \n",
       "246993  ff8d6679e9ce5c1228cf640311c0110b3aafb5bd  2019-08-06 11:40:00 UTC   \n",
       "246994  ff8d6679e9ce5c1228cf640311c0110b3aafb5bd  2019-08-19 13:00:00 UTC   \n",
       "246995  ff8d6679e9ce5c1228cf640311c0110b3aafb5bd  2019-08-06 08:05:00 UTC   \n",
       "\n",
       "       HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0          auto          hold               751                      765   \n",
       "1          cool          hold               765                      830   \n",
       "3          cool          hold               725                      721   \n",
       "4          cool          hold               753                      830   \n",
       "5          auto          hold               721                      716   \n",
       "...         ...           ...               ...                      ...   \n",
       "246991     cool          hold               764                      760   \n",
       "246992     cool          hold               765                      760   \n",
       "246993     cool          hold               754                      760   \n",
       "246994     cool          hold               699                      760   \n",
       "246995     cool          hold               764                      760   \n",
       "\n",
       "        TemperatureExpectedHeat ProvinceState           City  \\\n",
       "0                           715            NE        Lincoln   \n",
       "1                           790            NE  Nebraska City   \n",
       "3                           721            NE          Omaha   \n",
       "4                           790            NE  Nebraska City   \n",
       "5                           646            NE      Papillion   \n",
       "...                         ...           ...            ...   \n",
       "246991                      760            NE          Omaha   \n",
       "246992                      760            NE          Omaha   \n",
       "246993                      760            NE          Omaha   \n",
       "246994                      760            NE          Omaha   \n",
       "246995                      760            NE          Omaha   \n",
       "\n",
       "        Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                        39             False         False            False   \n",
       "1                        85             False         False            False   \n",
       "3                        50             False         False             True   \n",
       "4                        85             False         False            False   \n",
       "5                         5              True         False            False   \n",
       "...                     ...               ...           ...              ...   \n",
       "246991                    0              True         False             True   \n",
       "246992                    0              True         False             True   \n",
       "246993                    0              True         False             True   \n",
       "246994                    0              True         False             True   \n",
       "246995                    0              True         False             True   \n",
       "\n",
       "       Auxilliary_Heat_Fuel_Type  \n",
       "0                            Gas  \n",
       "1                            Gas  \n",
       "3                       Electric  \n",
       "4                            Gas  \n",
       "5                            Gas  \n",
       "...                          ...  \n",
       "246991                  Electric  \n",
       "246992                  Electric  \n",
       "246993                  Electric  \n",
       "246994                  Electric  \n",
       "246995                  Electric  \n",
       "\n",
       "[239633 rows x 14 columns]"
      ]
     },
     "execution_count": 147,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "aug_2019.drop(aug_2019[aug_2019['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "aug_2019.drop(aug_2019[aug_2019['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "aug_2019.drop(aug_2019[aug_2019['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "aug_2019.drop(aug_2019[aug_2019['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "aug_2019"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 148,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "aug_2019[\"Year\"] = \"2019\"\n",
    "aug_2019[\"Month\"] = \"aug\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 149,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "aug_2019 = aug_2019.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 150,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "aug_2019_ave = aug_2019.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# aug_2019_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 151,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "aug_2019_ave.to_csv(\"data/day/NE/aug/aug_2019_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2020 August Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 152,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "aug_2020 = pd.read_csv(\"../data_large/NE-day/2020-aug-day-NE.csv\")\n",
    "\n",
    "# aug_2020"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 153,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>6e5cf30b657c43e09ec50c7717cf699cfcf35178</td>\n",
       "      <td>2020-08-16 15:30:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>731</td>\n",
       "      <td>728</td>\n",
       "      <td>728</td>\n",
       "      <td>NE</td>\n",
       "      <td>Elkhorn</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>e5020a56db9e45e942254216808f1607f90aac53</td>\n",
       "      <td>2020-08-08 19:55:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>691</td>\n",
       "      <td>683</td>\n",
       "      <td>613</td>\n",
       "      <td>NE</td>\n",
       "      <td>Bennington</td>\n",
       "      <td>15</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2fdc63565f7136c15970a1012e3042ff5cef182f</td>\n",
       "      <td>2020-08-01 17:50:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>733</td>\n",
       "      <td>830</td>\n",
       "      <td>680</td>\n",
       "      <td>NE</td>\n",
       "      <td>Omaha</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>9f26605658880b3798b118e33b72a6c686383954</td>\n",
       "      <td>2020-08-22 15:30:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>690</td>\n",
       "      <td>707</td>\n",
       "      <td>657</td>\n",
       "      <td>NE</td>\n",
       "      <td>Papillion</td>\n",
       "      <td>20</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2bd35be62bb091959037e6ffb751504af73c21da</td>\n",
       "      <td>2020-08-24 17:40:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>818</td>\n",
       "      <td>840</td>\n",
       "      <td>790</td>\n",
       "      <td>NE</td>\n",
       "      <td>Omaha</td>\n",
       "      <td>40</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>238575</th>\n",
       "      <td>ff8d6679e9ce5c1228cf640311c0110b3aafb5bd</td>\n",
       "      <td>2020-08-11 17:15:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>744</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>NE</td>\n",
       "      <td>Omaha</td>\n",
       "      <td>0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>238576</th>\n",
       "      <td>ff8d6679e9ce5c1228cf640311c0110b3aafb5bd</td>\n",
       "      <td>2020-08-11 18:35:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>752</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>NE</td>\n",
       "      <td>Omaha</td>\n",
       "      <td>0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>238577</th>\n",
       "      <td>ff8d6679e9ce5c1228cf640311c0110b3aafb5bd</td>\n",
       "      <td>2020-08-04 19:40:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>NE</td>\n",
       "      <td>Omaha</td>\n",
       "      <td>0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>238578</th>\n",
       "      <td>ff8d6679e9ce5c1228cf640311c0110b3aafb5bd</td>\n",
       "      <td>2020-08-01 17:15:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>735</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>NE</td>\n",
       "      <td>Omaha</td>\n",
       "      <td>0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>238579</th>\n",
       "      <td>ff8d6679e9ce5c1228cf640311c0110b3aafb5bd</td>\n",
       "      <td>2020-08-05 07:40:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>753</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>NE</td>\n",
       "      <td>Omaha</td>\n",
       "      <td>0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>234379 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                      Identifier                date_time  \\\n",
       "0       6e5cf30b657c43e09ec50c7717cf699cfcf35178  2020-08-16 15:30:00 UTC   \n",
       "1       e5020a56db9e45e942254216808f1607f90aac53  2020-08-08 19:55:00 UTC   \n",
       "2       2fdc63565f7136c15970a1012e3042ff5cef182f  2020-08-01 17:50:00 UTC   \n",
       "3       9f26605658880b3798b118e33b72a6c686383954  2020-08-22 15:30:00 UTC   \n",
       "4       2bd35be62bb091959037e6ffb751504af73c21da  2020-08-24 17:40:00 UTC   \n",
       "...                                          ...                      ...   \n",
       "238575  ff8d6679e9ce5c1228cf640311c0110b3aafb5bd  2020-08-11 17:15:00 UTC   \n",
       "238576  ff8d6679e9ce5c1228cf640311c0110b3aafb5bd  2020-08-11 18:35:00 UTC   \n",
       "238577  ff8d6679e9ce5c1228cf640311c0110b3aafb5bd  2020-08-04 19:40:00 UTC   \n",
       "238578  ff8d6679e9ce5c1228cf640311c0110b3aafb5bd  2020-08-01 17:15:00 UTC   \n",
       "238579  ff8d6679e9ce5c1228cf640311c0110b3aafb5bd  2020-08-05 07:40:00 UTC   \n",
       "\n",
       "       HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0          cool          auto               731                      728   \n",
       "1          auto          hold               691                      683   \n",
       "2          auto          auto               733                      830   \n",
       "3          auto          hold               690                      707   \n",
       "4          cool          hold               818                      840   \n",
       "...         ...           ...               ...                      ...   \n",
       "238575     cool          hold               744                      760   \n",
       "238576     cool          hold               752                      760   \n",
       "238577     cool          hold               760                      760   \n",
       "238578     cool          auto               735                      760   \n",
       "238579     cool          auto               753                      760   \n",
       "\n",
       "        TemperatureExpectedHeat ProvinceState        City  \\\n",
       "0                           728            NE     Elkhorn   \n",
       "1                           613            NE  Bennington   \n",
       "2                           680            NE       Omaha   \n",
       "3                           657            NE   Papillion   \n",
       "4                           790            NE       Omaha   \n",
       "...                         ...           ...         ...   \n",
       "238575                      760            NE       Omaha   \n",
       "238576                      760            NE       Omaha   \n",
       "238577                      760            NE       Omaha   \n",
       "238578                      760            NE       Omaha   \n",
       "238579                      760            NE       Omaha   \n",
       "\n",
       "        Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                         0             False         False            False   \n",
       "1                        15             False         False            False   \n",
       "2                         0             False         False            False   \n",
       "3                        20             False         False            False   \n",
       "4                        40              True         False            False   \n",
       "...                     ...               ...           ...              ...   \n",
       "238575                    0              True         False             True   \n",
       "238576                    0              True         False             True   \n",
       "238577                    0              True         False             True   \n",
       "238578                    0              True         False             True   \n",
       "238579                    0              True         False             True   \n",
       "\n",
       "       Auxilliary_Heat_Fuel_Type  \n",
       "0                            Gas  \n",
       "1                            Gas  \n",
       "2                            Gas  \n",
       "3                            Gas  \n",
       "4                            Gas  \n",
       "...                          ...  \n",
       "238575                  Electric  \n",
       "238576                  Electric  \n",
       "238577                  Electric  \n",
       "238578                  Electric  \n",
       "238579                  Electric  \n",
       "\n",
       "[234379 rows x 14 columns]"
      ]
     },
     "execution_count": 153,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "aug_2020.drop(aug_2020[aug_2020['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "aug_2020.drop(aug_2020[aug_2020['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "aug_2020.drop(aug_2020[aug_2020['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "aug_2020.drop(aug_2020[aug_2020['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "aug_2020"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 154,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "aug_2020[\"Year\"] = \"2020\"\n",
    "aug_2020[\"Month\"] = \"aug\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 155,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "aug_2020 = aug_2020.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 156,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "aug_2020_ave = aug_2020.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# aug_2020_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 157,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "aug_2020_ave.to_csv(\"data/day/NE/aug/aug_2020_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "scrolled": true
   },
   "source": [
    "---\n",
    "\n",
    "### Combine month CSV Files \n",
    "1. Read in files in folders for each state\n",
    "2. Export as combined CSV"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 158,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create variable for files in directory\n",
    "files = [f for f in os.listdir(\"data/day/NE/aug/\") if f.endswith(\".csv\")]\n",
    "\n",
    "# files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 159,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>Month</th>\n",
       "      <th>Year</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>City</th>\n",
       "      <th>Temperature_ctrl_ave</th>\n",
       "      <th>TemperatureExpectedCool_ave</th>\n",
       "      <th>TemperatureExpectedHeat_ave</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0087ac1d7540ff5efa6ae20213627943a2b07fe8</td>\n",
       "      <td>aug</td>\n",
       "      <td>2017</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>Omaha</td>\n",
       "      <td>741.040000</td>\n",
       "      <td>740.000000</td>\n",
       "      <td>680.000000</td>\n",
       "      <td>35.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>03fb21c8de5320f353144ac52422f977cfb769f1</td>\n",
       "      <td>aug</td>\n",
       "      <td>2017</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>York</td>\n",
       "      <td>735.913706</td>\n",
       "      <td>734.137056</td>\n",
       "      <td>706.218274</td>\n",
       "      <td>10.0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>05320d509a18b2d4022029409e7a267736e78e0b</td>\n",
       "      <td>aug</td>\n",
       "      <td>2017</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>Lincoln</td>\n",
       "      <td>719.923077</td>\n",
       "      <td>715.726496</td>\n",
       "      <td>700.000000</td>\n",
       "      <td>90.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>05320d509a18b2d4022029409e7a267736e78e0b</td>\n",
       "      <td>aug</td>\n",
       "      <td>2017</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>Lincoln</td>\n",
       "      <td>721.363946</td>\n",
       "      <td>726.908163</td>\n",
       "      <td>726.353741</td>\n",
       "      <td>90.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0ad90268dcfa7e7aa1d92768a5d17e1841ceaf17</td>\n",
       "      <td>aug</td>\n",
       "      <td>2017</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>McCool Junction</td>\n",
       "      <td>756.729167</td>\n",
       "      <td>775.466667</td>\n",
       "      <td>775.245833</td>\n",
       "      <td>15.0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>375</th>\n",
       "      <td>faa8c435b84d052676a86ba7d2d6a783d26e6b19</td>\n",
       "      <td>aug</td>\n",
       "      <td>2020</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>Omaha</td>\n",
       "      <td>709.611111</td>\n",
       "      <td>709.805556</td>\n",
       "      <td>709.430556</td>\n",
       "      <td>20.0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>376</th>\n",
       "      <td>faa8c435b84d052676a86ba7d2d6a783d26e6b19</td>\n",
       "      <td>aug</td>\n",
       "      <td>2020</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>Omaha</td>\n",
       "      <td>708.000000</td>\n",
       "      <td>710.000000</td>\n",
       "      <td>710.000000</td>\n",
       "      <td>20.0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>377</th>\n",
       "      <td>fe6909cb9f272a9b66f3c1b365252ff7168d5760</td>\n",
       "      <td>aug</td>\n",
       "      <td>2020</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>omaha</td>\n",
       "      <td>710.076923</td>\n",
       "      <td>682.384615</td>\n",
       "      <td>615.282051</td>\n",
       "      <td>0.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>378</th>\n",
       "      <td>ff8d6679e9ce5c1228cf640311c0110b3aafb5bd</td>\n",
       "      <td>aug</td>\n",
       "      <td>2020</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>Omaha</td>\n",
       "      <td>727.692643</td>\n",
       "      <td>732.138404</td>\n",
       "      <td>732.138404</td>\n",
       "      <td>0.0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>379</th>\n",
       "      <td>ff8d6679e9ce5c1228cf640311c0110b3aafb5bd</td>\n",
       "      <td>aug</td>\n",
       "      <td>2020</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>Omaha</td>\n",
       "      <td>735.604531</td>\n",
       "      <td>738.222451</td>\n",
       "      <td>738.222451</td>\n",
       "      <td>0.0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1243 rows × 13 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                   Identifier Month  Year HvacMode  \\\n",
       "0    0087ac1d7540ff5efa6ae20213627943a2b07fe8   aug  2017     auto   \n",
       "1    03fb21c8de5320f353144ac52422f977cfb769f1   aug  2017     cool   \n",
       "2    05320d509a18b2d4022029409e7a267736e78e0b   aug  2017     cool   \n",
       "3    05320d509a18b2d4022029409e7a267736e78e0b   aug  2017     cool   \n",
       "4    0ad90268dcfa7e7aa1d92768a5d17e1841ceaf17   aug  2017     cool   \n",
       "..                                        ...   ...   ...      ...   \n",
       "375  faa8c435b84d052676a86ba7d2d6a783d26e6b19   aug  2020     cool   \n",
       "376  faa8c435b84d052676a86ba7d2d6a783d26e6b19   aug  2020     cool   \n",
       "377  fe6909cb9f272a9b66f3c1b365252ff7168d5760   aug  2020     auto   \n",
       "378  ff8d6679e9ce5c1228cf640311c0110b3aafb5bd   aug  2020     cool   \n",
       "379  ff8d6679e9ce5c1228cf640311c0110b3aafb5bd   aug  2020     cool   \n",
       "\n",
       "    CalendarEvent             City  Temperature_ctrl_ave  \\\n",
       "0            hold            Omaha            741.040000   \n",
       "1            hold             York            735.913706   \n",
       "2            auto          Lincoln            719.923077   \n",
       "3            hold          Lincoln            721.363946   \n",
       "4            hold  McCool Junction            756.729167   \n",
       "..            ...              ...                   ...   \n",
       "375          auto            Omaha            709.611111   \n",
       "376          hold            Omaha            708.000000   \n",
       "377          hold            omaha            710.076923   \n",
       "378          auto            Omaha            727.692643   \n",
       "379          hold            Omaha            735.604531   \n",
       "\n",
       "     TemperatureExpectedCool_ave  TemperatureExpectedHeat_ave  \\\n",
       "0                     740.000000                   680.000000   \n",
       "1                     734.137056                   706.218274   \n",
       "2                     715.726496                   700.000000   \n",
       "3                     726.908163                   726.353741   \n",
       "4                     775.466667                   775.245833   \n",
       "..                           ...                          ...   \n",
       "375                   709.805556                   709.430556   \n",
       "376                   710.000000                   710.000000   \n",
       "377                   682.384615                   615.282051   \n",
       "378                   732.138404                   732.138404   \n",
       "379                   738.222451                   738.222451   \n",
       "\n",
       "     Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \n",
       "0                   35.0             False         False            False  \n",
       "1                   10.0              True         False             True  \n",
       "2                   90.0             False         False            False  \n",
       "3                   90.0             False         False            False  \n",
       "4                   15.0              True         False             True  \n",
       "..                   ...               ...           ...              ...  \n",
       "375                 20.0              True         False            False  \n",
       "376                 20.0              True         False            False  \n",
       "377                  0.0             False         False            False  \n",
       "378                  0.0              True         False             True  \n",
       "379                  0.0              True         False             True  \n",
       "\n",
       "[1243 rows x 13 columns]"
      ]
     },
     "execution_count": 159,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# https://stackoverflow.com/questions/63886787/how-to-create-a-dataframe-from-multiple-csv-files\n",
    "# Initialize an empty dataframe\n",
    "NE_aug = pd.DataFrame()\n",
    "\n",
    "# Iterate through files and contents, then concatenate the data into the dataframe\n",
    "for file in files:\n",
    "    df = pd.read_csv(\"data/day/NE/aug/\" + file)\n",
    "    NE_aug = pd.concat([NE_aug, df])\n",
    "    \n",
    "NE_aug"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 160,
   "metadata": {},
   "outputs": [],
   "source": [
    "NE_aug.to_csv(\"Scraper_Output/State_Month_Day/NE/NE_aug.csv\", header=True, index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "\n",
    "## September"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "\n",
    "## October"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "\n",
    "## November"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "\n",
    "## December"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2017 December Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 161,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "dec_2017 = pd.read_csv(\"../data_large/NE-day/2017-dec-day-NE.csv\")\n",
    "\n",
    "# dec_2017"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 162,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1985472cdc3f8d3bf064a68f444737c05c724ea8</td>\n",
       "      <td>2017-12-16 18:05:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>728</td>\n",
       "      <td>785</td>\n",
       "      <td>705</td>\n",
       "      <td>NE</td>\n",
       "      <td>Omaha</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1985472cdc3f8d3bf064a68f444737c05c724ea8</td>\n",
       "      <td>2017-12-30 17:50:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>722</td>\n",
       "      <td>775</td>\n",
       "      <td>725</td>\n",
       "      <td>NE</td>\n",
       "      <td>Omaha</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>40fc9b58f9fc26baac1b9d96ce344d71518f6e49</td>\n",
       "      <td>2017-12-17 13:20:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>738</td>\n",
       "      <td>788</td>\n",
       "      <td>720</td>\n",
       "      <td>NE</td>\n",
       "      <td>Bellevue</td>\n",
       "      <td>0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1985472cdc3f8d3bf064a68f444737c05c724ea8</td>\n",
       "      <td>2017-12-25 15:35:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>716</td>\n",
       "      <td>765</td>\n",
       "      <td>715</td>\n",
       "      <td>NE</td>\n",
       "      <td>Omaha</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>4f608a05003feda5fb362f497866f36d31acd324</td>\n",
       "      <td>2017-12-03 15:50:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>719</td>\n",
       "      <td>840</td>\n",
       "      <td>710</td>\n",
       "      <td>NE</td>\n",
       "      <td>Lincoln</td>\n",
       "      <td>115</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>104707</th>\n",
       "      <td>dc8b24b927b1c859f3d316b2c07c14a69a26181b</td>\n",
       "      <td>2017-12-20 12:30:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>671</td>\n",
       "      <td>760</td>\n",
       "      <td>670</td>\n",
       "      <td>NE</td>\n",
       "      <td>Bennington</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>104708</th>\n",
       "      <td>dc8b24b927b1c859f3d316b2c07c14a69a26181b</td>\n",
       "      <td>2017-12-19 12:35:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>689</td>\n",
       "      <td>760</td>\n",
       "      <td>680</td>\n",
       "      <td>NE</td>\n",
       "      <td>Bennington</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>104709</th>\n",
       "      <td>dc8b24b927b1c859f3d316b2c07c14a69a26181b</td>\n",
       "      <td>2017-12-20 18:15:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>714</td>\n",
       "      <td>760</td>\n",
       "      <td>700</td>\n",
       "      <td>NE</td>\n",
       "      <td>Bennington</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>104710</th>\n",
       "      <td>dc8b24b927b1c859f3d316b2c07c14a69a26181b</td>\n",
       "      <td>2017-12-19 16:55:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>704</td>\n",
       "      <td>760</td>\n",
       "      <td>700</td>\n",
       "      <td>NE</td>\n",
       "      <td>Bennington</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>104711</th>\n",
       "      <td>dc8b24b927b1c859f3d316b2c07c14a69a26181b</td>\n",
       "      <td>2017-12-18 16:00:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>720</td>\n",
       "      <td>760</td>\n",
       "      <td>710</td>\n",
       "      <td>NE</td>\n",
       "      <td>Bennington</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>100307 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                      Identifier                date_time  \\\n",
       "0       1985472cdc3f8d3bf064a68f444737c05c724ea8  2017-12-16 18:05:00 UTC   \n",
       "1       1985472cdc3f8d3bf064a68f444737c05c724ea8  2017-12-30 17:50:00 UTC   \n",
       "2       40fc9b58f9fc26baac1b9d96ce344d71518f6e49  2017-12-17 13:20:00 UTC   \n",
       "4       1985472cdc3f8d3bf064a68f444737c05c724ea8  2017-12-25 15:35:00 UTC   \n",
       "6       4f608a05003feda5fb362f497866f36d31acd324  2017-12-03 15:50:00 UTC   \n",
       "...                                          ...                      ...   \n",
       "104707  dc8b24b927b1c859f3d316b2c07c14a69a26181b  2017-12-20 12:30:00 UTC   \n",
       "104708  dc8b24b927b1c859f3d316b2c07c14a69a26181b  2017-12-19 12:35:00 UTC   \n",
       "104709  dc8b24b927b1c859f3d316b2c07c14a69a26181b  2017-12-20 18:15:00 UTC   \n",
       "104710  dc8b24b927b1c859f3d316b2c07c14a69a26181b  2017-12-19 16:55:00 UTC   \n",
       "104711  dc8b24b927b1c859f3d316b2c07c14a69a26181b  2017-12-18 16:00:00 UTC   \n",
       "\n",
       "       HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0          auto          hold               728                      785   \n",
       "1          auto          hold               722                      775   \n",
       "2          heat          hold               738                      788   \n",
       "4          auto          hold               716                      765   \n",
       "6          auto          hold               719                      840   \n",
       "...         ...           ...               ...                      ...   \n",
       "104707     auto          hold               671                      760   \n",
       "104708     auto          hold               689                      760   \n",
       "104709     auto          hold               714                      760   \n",
       "104710     auto          hold               704                      760   \n",
       "104711     auto          hold               720                      760   \n",
       "\n",
       "        TemperatureExpectedHeat ProvinceState        City  \\\n",
       "0                           705            NE       Omaha   \n",
       "1                           725            NE       Omaha   \n",
       "2                           720            NE    Bellevue   \n",
       "4                           715            NE       Omaha   \n",
       "6                           710            NE     Lincoln   \n",
       "...                         ...           ...         ...   \n",
       "104707                      670            NE  Bennington   \n",
       "104708                      680            NE  Bennington   \n",
       "104709                      700            NE  Bennington   \n",
       "104710                      700            NE  Bennington   \n",
       "104711                      710            NE  Bennington   \n",
       "\n",
       "        Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                        10             False         False            False   \n",
       "1                        10             False         False            False   \n",
       "2                         0              True         False            False   \n",
       "4                        10             False         False            False   \n",
       "6                       115             False         False            False   \n",
       "...                     ...               ...           ...              ...   \n",
       "104707                    5             False         False            False   \n",
       "104708                    5             False         False            False   \n",
       "104709                    5             False         False            False   \n",
       "104710                    5             False         False            False   \n",
       "104711                    5             False         False            False   \n",
       "\n",
       "       Auxilliary_Heat_Fuel_Type  \n",
       "0                            Gas  \n",
       "1                            Gas  \n",
       "2                            Gas  \n",
       "4                            Gas  \n",
       "6                            Gas  \n",
       "...                          ...  \n",
       "104707                       Gas  \n",
       "104708                       Gas  \n",
       "104709                       Gas  \n",
       "104710                       Gas  \n",
       "104711                       Gas  \n",
       "\n",
       "[100307 rows x 14 columns]"
      ]
     },
     "execution_count": 162,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "dec_2017.drop(dec_2017[dec_2017['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "dec_2017.drop(dec_2017[dec_2017['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "dec_2017.drop(dec_2017[dec_2017['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "dec_2017.drop(dec_2017[dec_2017['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "dec_2017"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 163,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "dec_2017[\"Year\"] = \"2017\"\n",
    "dec_2017[\"Month\"] = \"dec\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 164,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "dec_2017 = dec_2017.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 165,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "dec_2017_ave = dec_2017.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# dec_2017_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 166,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "dec_2017_ave.to_csv(\"data/day/NE/dec/dec_2017_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2018 December Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 167,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "dec_2018 = pd.read_csv(\"../data_large/NE-day/2018-dec-day-NE.csv\")\n",
    "\n",
    "# dec_2018"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 168,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>908220d248d3d0de476e4ced851377b2c920ba94</td>\n",
       "      <td>2018-12-16 13:20:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>711</td>\n",
       "      <td>791</td>\n",
       "      <td>680</td>\n",
       "      <td>NE</td>\n",
       "      <td>Papillion</td>\n",
       "      <td>0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>7b9c42484b70b04b51eca1379ebd7dbca68ac432</td>\n",
       "      <td>2018-12-02 12:55:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>734</td>\n",
       "      <td>782</td>\n",
       "      <td>732</td>\n",
       "      <td>NE</td>\n",
       "      <td>Papillion</td>\n",
       "      <td>5</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1cde6ae2ccdf910bb7fd2bd26dac007a23614191</td>\n",
       "      <td>2018-12-11 14:45:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>712</td>\n",
       "      <td>765</td>\n",
       "      <td>715</td>\n",
       "      <td>NE</td>\n",
       "      <td>Lincoln</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>249107bc009e59a0fd5f8dff7cf865f4bde925c3</td>\n",
       "      <td>2018-12-29 19:35:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>680</td>\n",
       "      <td>685</td>\n",
       "      <td>685</td>\n",
       "      <td>NE</td>\n",
       "      <td>Lincoln</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>02674609e156f6807ee78e1c9ac2851c16854804</td>\n",
       "      <td>2018-12-08 18:30:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>648</td>\n",
       "      <td>655</td>\n",
       "      <td>655</td>\n",
       "      <td>NE</td>\n",
       "      <td>Omaha</td>\n",
       "      <td>50</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>199801</th>\n",
       "      <td>f15f623c199dc4f547c63da94db6260301a8cc81</td>\n",
       "      <td>2018-12-07 14:40:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>686</td>\n",
       "      <td>760</td>\n",
       "      <td>690</td>\n",
       "      <td>NE</td>\n",
       "      <td>Springfield</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>199802</th>\n",
       "      <td>f15f623c199dc4f547c63da94db6260301a8cc81</td>\n",
       "      <td>2018-12-16 12:55:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>688</td>\n",
       "      <td>760</td>\n",
       "      <td>690</td>\n",
       "      <td>NE</td>\n",
       "      <td>Springfield</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>199803</th>\n",
       "      <td>f15f623c199dc4f547c63da94db6260301a8cc81</td>\n",
       "      <td>2018-12-27 18:25:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>694</td>\n",
       "      <td>760</td>\n",
       "      <td>690</td>\n",
       "      <td>NE</td>\n",
       "      <td>Springfield</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>199804</th>\n",
       "      <td>f15f623c199dc4f547c63da94db6260301a8cc81</td>\n",
       "      <td>2018-12-16 14:55:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>688</td>\n",
       "      <td>760</td>\n",
       "      <td>690</td>\n",
       "      <td>NE</td>\n",
       "      <td>Springfield</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>199805</th>\n",
       "      <td>f15f623c199dc4f547c63da94db6260301a8cc81</td>\n",
       "      <td>2018-12-28 16:20:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>691</td>\n",
       "      <td>760</td>\n",
       "      <td>690</td>\n",
       "      <td>NE</td>\n",
       "      <td>Springfield</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>196241 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                      Identifier                date_time  \\\n",
       "0       908220d248d3d0de476e4ced851377b2c920ba94  2018-12-16 13:20:00 UTC   \n",
       "1       7b9c42484b70b04b51eca1379ebd7dbca68ac432  2018-12-02 12:55:00 UTC   \n",
       "2       1cde6ae2ccdf910bb7fd2bd26dac007a23614191  2018-12-11 14:45:00 UTC   \n",
       "3       249107bc009e59a0fd5f8dff7cf865f4bde925c3  2018-12-29 19:35:00 UTC   \n",
       "4       02674609e156f6807ee78e1c9ac2851c16854804  2018-12-08 18:30:00 UTC   \n",
       "...                                          ...                      ...   \n",
       "199801  f15f623c199dc4f547c63da94db6260301a8cc81  2018-12-07 14:40:00 UTC   \n",
       "199802  f15f623c199dc4f547c63da94db6260301a8cc81  2018-12-16 12:55:00 UTC   \n",
       "199803  f15f623c199dc4f547c63da94db6260301a8cc81  2018-12-27 18:25:00 UTC   \n",
       "199804  f15f623c199dc4f547c63da94db6260301a8cc81  2018-12-16 14:55:00 UTC   \n",
       "199805  f15f623c199dc4f547c63da94db6260301a8cc81  2018-12-28 16:20:00 UTC   \n",
       "\n",
       "       HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0          heat          auto               711                      791   \n",
       "1          auto          hold               734                      782   \n",
       "2          auto          hold               712                      765   \n",
       "3          heat          hold               680                      685   \n",
       "4          heat          hold               648                      655   \n",
       "...         ...           ...               ...                      ...   \n",
       "199801     heat          auto               686                      760   \n",
       "199802     heat          auto               688                      760   \n",
       "199803     heat          auto               694                      760   \n",
       "199804     heat          auto               688                      760   \n",
       "199805     heat          auto               691                      760   \n",
       "\n",
       "        TemperatureExpectedHeat ProvinceState         City  \\\n",
       "0                           680            NE    Papillion   \n",
       "1                           732            NE    Papillion   \n",
       "2                           715            NE      Lincoln   \n",
       "3                           685            NE      Lincoln   \n",
       "4                           655            NE        Omaha   \n",
       "...                         ...           ...          ...   \n",
       "199801                      690            NE  Springfield   \n",
       "199802                      690            NE  Springfield   \n",
       "199803                      690            NE  Springfield   \n",
       "199804                      690            NE  Springfield   \n",
       "199805                      690            NE  Springfield   \n",
       "\n",
       "        Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                         0              True         False            False   \n",
       "1                         5              True         False            False   \n",
       "2                        10             False         False            False   \n",
       "3                         0             False         False            False   \n",
       "4                        50             False         False             True   \n",
       "...                     ...               ...           ...              ...   \n",
       "199801                    0             False         False             True   \n",
       "199802                    0             False         False             True   \n",
       "199803                    0             False         False             True   \n",
       "199804                    0             False         False             True   \n",
       "199805                    0             False         False             True   \n",
       "\n",
       "       Auxilliary_Heat_Fuel_Type  \n",
       "0                            Gas  \n",
       "1                            Gas  \n",
       "2                            Gas  \n",
       "3                            Gas  \n",
       "4                       Electric  \n",
       "...                          ...  \n",
       "199801                  Electric  \n",
       "199802                  Electric  \n",
       "199803                  Electric  \n",
       "199804                  Electric  \n",
       "199805                  Electric  \n",
       "\n",
       "[196241 rows x 14 columns]"
      ]
     },
     "execution_count": 168,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "dec_2018.drop(dec_2018[dec_2018['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "dec_2018.drop(dec_2018[dec_2018['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "dec_2018.drop(dec_2018[dec_2018['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "dec_2018.drop(dec_2018[dec_2018['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "dec_2018"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 169,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "dec_2018[\"Year\"] = \"2018\"\n",
    "dec_2018[\"Month\"] = \"dec\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 170,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "dec_2018 = dec_2018.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 171,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "dec_2018_ave = dec_2018.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# dec_2018_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 172,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "dec_2018_ave.to_csv(\"data/day/NE/dec/dec_2018_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2019 December Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 173,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "dec_2019 = pd.read_csv(\"../data_large/NE-day/2019-dec-day-NE.csv\")\n",
    "\n",
    "# dec_2019"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 174,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>124aefa02526f136375928b0c92df70f356d26b9</td>\n",
       "      <td>2019-12-28 12:35:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>673</td>\n",
       "      <td>675</td>\n",
       "      <td>675</td>\n",
       "      <td>NE</td>\n",
       "      <td>Omaha</td>\n",
       "      <td>50</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>124aefa02526f136375928b0c92df70f356d26b9</td>\n",
       "      <td>2019-12-13 16:30:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>682</td>\n",
       "      <td>685</td>\n",
       "      <td>685</td>\n",
       "      <td>NE</td>\n",
       "      <td>Omaha</td>\n",
       "      <td>50</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>c039f7f15fbd5d98b80efba2631acb48bef7d066</td>\n",
       "      <td>2019-12-08 14:10:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>704</td>\n",
       "      <td>765</td>\n",
       "      <td>705</td>\n",
       "      <td>NE</td>\n",
       "      <td>Bennington</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>6a4be48209bd11b657a04f995d5afec116bbb937</td>\n",
       "      <td>2019-12-17 13:15:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>711</td>\n",
       "      <td>728</td>\n",
       "      <td>704</td>\n",
       "      <td>NE</td>\n",
       "      <td>Omaha</td>\n",
       "      <td>20</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>6a5f3cda47d1f1ebb68eba316cdb8b6e88c5b112</td>\n",
       "      <td>2019-12-08 14:30:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>713</td>\n",
       "      <td>756</td>\n",
       "      <td>716</td>\n",
       "      <td>NE</td>\n",
       "      <td>Hastings</td>\n",
       "      <td>40</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>217586</th>\n",
       "      <td>e50f2e1393d5251e0586c8b339bca1f26e3430e6</td>\n",
       "      <td>2019-12-11 14:25:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>695</td>\n",
       "      <td>760</td>\n",
       "      <td>700</td>\n",
       "      <td>NE</td>\n",
       "      <td>Lincoln</td>\n",
       "      <td>27</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>217587</th>\n",
       "      <td>e50f2e1393d5251e0586c8b339bca1f26e3430e6</td>\n",
       "      <td>2019-12-18 16:35:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>699</td>\n",
       "      <td>760</td>\n",
       "      <td>700</td>\n",
       "      <td>NE</td>\n",
       "      <td>Lincoln</td>\n",
       "      <td>27</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>217588</th>\n",
       "      <td>e50f2e1393d5251e0586c8b339bca1f26e3430e6</td>\n",
       "      <td>2019-12-09 19:10:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>677</td>\n",
       "      <td>760</td>\n",
       "      <td>680</td>\n",
       "      <td>NE</td>\n",
       "      <td>Lincoln</td>\n",
       "      <td>27</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>217589</th>\n",
       "      <td>e50f2e1393d5251e0586c8b339bca1f26e3430e6</td>\n",
       "      <td>2019-12-08 18:45:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>698</td>\n",
       "      <td>760</td>\n",
       "      <td>700</td>\n",
       "      <td>NE</td>\n",
       "      <td>Lincoln</td>\n",
       "      <td>27</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>217590</th>\n",
       "      <td>e50f2e1393d5251e0586c8b339bca1f26e3430e6</td>\n",
       "      <td>2019-12-16 19:15:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>697</td>\n",
       "      <td>760</td>\n",
       "      <td>700</td>\n",
       "      <td>NE</td>\n",
       "      <td>Lincoln</td>\n",
       "      <td>27</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>209422 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                      Identifier                date_time  \\\n",
       "0       124aefa02526f136375928b0c92df70f356d26b9  2019-12-28 12:35:00 UTC   \n",
       "1       124aefa02526f136375928b0c92df70f356d26b9  2019-12-13 16:30:00 UTC   \n",
       "2       c039f7f15fbd5d98b80efba2631acb48bef7d066  2019-12-08 14:10:00 UTC   \n",
       "3       6a4be48209bd11b657a04f995d5afec116bbb937  2019-12-17 13:15:00 UTC   \n",
       "4       6a5f3cda47d1f1ebb68eba316cdb8b6e88c5b112  2019-12-08 14:30:00 UTC   \n",
       "...                                          ...                      ...   \n",
       "217586  e50f2e1393d5251e0586c8b339bca1f26e3430e6  2019-12-11 14:25:00 UTC   \n",
       "217587  e50f2e1393d5251e0586c8b339bca1f26e3430e6  2019-12-18 16:35:00 UTC   \n",
       "217588  e50f2e1393d5251e0586c8b339bca1f26e3430e6  2019-12-09 19:10:00 UTC   \n",
       "217589  e50f2e1393d5251e0586c8b339bca1f26e3430e6  2019-12-08 18:45:00 UTC   \n",
       "217590  e50f2e1393d5251e0586c8b339bca1f26e3430e6  2019-12-16 19:15:00 UTC   \n",
       "\n",
       "       HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0          heat          hold               673                      675   \n",
       "1          heat          hold               682                      685   \n",
       "2          auto          hold               704                      765   \n",
       "3          heat          hold               711                      728   \n",
       "4          auto          hold               713                      756   \n",
       "...         ...           ...               ...                      ...   \n",
       "217586     auto          hold               695                      760   \n",
       "217587     auto          hold               699                      760   \n",
       "217588     auto          hold               677                      760   \n",
       "217589     auto          hold               698                      760   \n",
       "217590     auto          hold               697                      760   \n",
       "\n",
       "        TemperatureExpectedHeat ProvinceState        City  \\\n",
       "0                           675            NE       Omaha   \n",
       "1                           685            NE       Omaha   \n",
       "2                           705            NE  Bennington   \n",
       "3                           704            NE       Omaha   \n",
       "4                           716            NE    Hastings   \n",
       "...                         ...           ...         ...   \n",
       "217586                      700            NE     Lincoln   \n",
       "217587                      700            NE     Lincoln   \n",
       "217588                      680            NE     Lincoln   \n",
       "217589                      700            NE     Lincoln   \n",
       "217590                      700            NE     Lincoln   \n",
       "\n",
       "        Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                        50              True         False             True   \n",
       "1                        50              True         False             True   \n",
       "2                         0             False         False            False   \n",
       "3                        20             False         False            False   \n",
       "4                        40              True         False            False   \n",
       "...                     ...               ...           ...              ...   \n",
       "217586                   27             False         False            False   \n",
       "217587                   27             False         False            False   \n",
       "217588                   27             False         False            False   \n",
       "217589                   27             False         False            False   \n",
       "217590                   27             False         False            False   \n",
       "\n",
       "       Auxilliary_Heat_Fuel_Type  \n",
       "0                       Electric  \n",
       "1                       Electric  \n",
       "2                            Gas  \n",
       "3                            Gas  \n",
       "4                            Gas  \n",
       "...                          ...  \n",
       "217586                       Gas  \n",
       "217587                       Gas  \n",
       "217588                       Gas  \n",
       "217589                       Gas  \n",
       "217590                       Gas  \n",
       "\n",
       "[209422 rows x 14 columns]"
      ]
     },
     "execution_count": 174,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "dec_2019.drop(dec_2019[dec_2019['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "dec_2019.drop(dec_2019[dec_2019['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "dec_2019.drop(dec_2019[dec_2019['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "dec_2019.drop(dec_2019[dec_2019['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "dec_2019"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 175,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "dec_2019[\"Year\"] = \"2019\"\n",
    "dec_2019[\"Month\"] = \"dec\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 176,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "dec_2019 = dec_2019.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 177,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "dec_2019_ave = dec_2019.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# dec_2019_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 178,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "dec_2019_ave.to_csv(\"data/day/NE/dec/dec_2019_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2020 December Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 179,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "dec_2020 = pd.read_csv(\"../data_large/NE-day/2020-dec-day-NE.csv\")\n",
    "\n",
    "# dec_2020"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 180,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>6a4878bb967916a664ed4d25a33d2d8c92342a59</td>\n",
       "      <td>2020-12-10 15:15:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>702</td>\n",
       "      <td>706</td>\n",
       "      <td>706</td>\n",
       "      <td>NE</td>\n",
       "      <td>HAMPTON</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2504df671d8f2abfb4e209289ba629963f2db33b</td>\n",
       "      <td>2020-12-20 17:35:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>743</td>\n",
       "      <td>743</td>\n",
       "      <td>743</td>\n",
       "      <td>NE</td>\n",
       "      <td>Lincoln</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>6a4878bb967916a664ed4d25a33d2d8c92342a59</td>\n",
       "      <td>2020-12-12 12:55:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>705</td>\n",
       "      <td>706</td>\n",
       "      <td>706</td>\n",
       "      <td>NE</td>\n",
       "      <td>HAMPTON</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>6a4878bb967916a664ed4d25a33d2d8c92342a59</td>\n",
       "      <td>2020-12-19 19:45:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>701</td>\n",
       "      <td>706</td>\n",
       "      <td>706</td>\n",
       "      <td>NE</td>\n",
       "      <td>HAMPTON</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5182408f94362ec1153ac7567db8bdc5cb2f1aea</td>\n",
       "      <td>2020-12-16 13:35:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>677</td>\n",
       "      <td>732</td>\n",
       "      <td>680</td>\n",
       "      <td>NE</td>\n",
       "      <td>Kearney</td>\n",
       "      <td>35</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>198113</th>\n",
       "      <td>ea8146a43e83f42479f4cc045bfee4fa30d0f6eb</td>\n",
       "      <td>2020-12-24 14:55:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>759</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>NE</td>\n",
       "      <td>Lemoyne</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>198114</th>\n",
       "      <td>ea8146a43e83f42479f4cc045bfee4fa30d0f6eb</td>\n",
       "      <td>2020-12-24 14:25:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>761</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>NE</td>\n",
       "      <td>Lemoyne</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>198115</th>\n",
       "      <td>ea8146a43e83f42479f4cc045bfee4fa30d0f6eb</td>\n",
       "      <td>2020-12-24 14:35:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>756</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>NE</td>\n",
       "      <td>Lemoyne</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>198116</th>\n",
       "      <td>ea8146a43e83f42479f4cc045bfee4fa30d0f6eb</td>\n",
       "      <td>2020-12-24 14:20:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>NE</td>\n",
       "      <td>Lemoyne</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>198117</th>\n",
       "      <td>f245bdd207dcc0e6cef19201496ee235fed7dbfd</td>\n",
       "      <td>2020-12-02 18:50:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>700</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>NE</td>\n",
       "      <td>Kearney</td>\n",
       "      <td>0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>194022 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                      Identifier                date_time  \\\n",
       "0       6a4878bb967916a664ed4d25a33d2d8c92342a59  2020-12-10 15:15:00 UTC   \n",
       "1       2504df671d8f2abfb4e209289ba629963f2db33b  2020-12-20 17:35:00 UTC   \n",
       "2       6a4878bb967916a664ed4d25a33d2d8c92342a59  2020-12-12 12:55:00 UTC   \n",
       "3       6a4878bb967916a664ed4d25a33d2d8c92342a59  2020-12-19 19:45:00 UTC   \n",
       "4       5182408f94362ec1153ac7567db8bdc5cb2f1aea  2020-12-16 13:35:00 UTC   \n",
       "...                                          ...                      ...   \n",
       "198113  ea8146a43e83f42479f4cc045bfee4fa30d0f6eb  2020-12-24 14:55:00 UTC   \n",
       "198114  ea8146a43e83f42479f4cc045bfee4fa30d0f6eb  2020-12-24 14:25:00 UTC   \n",
       "198115  ea8146a43e83f42479f4cc045bfee4fa30d0f6eb  2020-12-24 14:35:00 UTC   \n",
       "198116  ea8146a43e83f42479f4cc045bfee4fa30d0f6eb  2020-12-24 14:20:00 UTC   \n",
       "198117  f245bdd207dcc0e6cef19201496ee235fed7dbfd  2020-12-02 18:50:00 UTC   \n",
       "\n",
       "       HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0          heat          hold               702                      706   \n",
       "1          heat          hold               743                      743   \n",
       "2          heat          hold               705                      706   \n",
       "3          heat          hold               701                      706   \n",
       "4          heat          hold               677                      732   \n",
       "...         ...           ...               ...                      ...   \n",
       "198113     heat          hold               759                      760   \n",
       "198114     heat          hold               761                      760   \n",
       "198115     heat          hold               756                      760   \n",
       "198116     heat          hold               760                      760   \n",
       "198117     heat          hold               700                      760   \n",
       "\n",
       "        TemperatureExpectedHeat ProvinceState     City  Age_of_Home__years_  \\\n",
       "0                           706            NE  HAMPTON                   10   \n",
       "1                           743            NE  Lincoln                    0   \n",
       "2                           706            NE  HAMPTON                   10   \n",
       "3                           706            NE  HAMPTON                   10   \n",
       "4                           680            NE  Kearney                   35   \n",
       "...                         ...           ...      ...                  ...   \n",
       "198113                      760            NE  Lemoyne                    0   \n",
       "198114                      760            NE  Lemoyne                    0   \n",
       "198115                      760            NE  Lemoyne                    0   \n",
       "198116                      760            NE  Lemoyne                    0   \n",
       "198117                      760            NE  Kearney                    0   \n",
       "\n",
       "        allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                  False         False             True   \n",
       "1                  False         False            False   \n",
       "2                  False         False             True   \n",
       "3                  False         False             True   \n",
       "4                  False         False            False   \n",
       "...                  ...           ...              ...   \n",
       "198113             False         False            False   \n",
       "198114             False         False            False   \n",
       "198115             False         False            False   \n",
       "198116             False         False            False   \n",
       "198117              True         False             True   \n",
       "\n",
       "       Auxilliary_Heat_Fuel_Type  \n",
       "0                       Electric  \n",
       "1                            Gas  \n",
       "2                       Electric  \n",
       "3                       Electric  \n",
       "4                            Gas  \n",
       "...                          ...  \n",
       "198113                       Gas  \n",
       "198114                       Gas  \n",
       "198115                       Gas  \n",
       "198116                       Gas  \n",
       "198117                  Electric  \n",
       "\n",
       "[194022 rows x 14 columns]"
      ]
     },
     "execution_count": 180,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "dec_2020.drop(dec_2020[dec_2020['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "dec_2020.drop(dec_2020[dec_2020['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "dec_2020.drop(dec_2020[dec_2020['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "dec_2020.drop(dec_2020[dec_2020['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "dec_2020"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 181,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "dec_2020[\"Year\"] = \"2020\"\n",
    "dec_2020[\"Month\"] = \"dec\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 182,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "dec_2020 = dec_2020.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 183,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "dec_2020_ave = dec_2020.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# dec_2020_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 184,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "dec_2020_ave.to_csv(\"data/day/NE/dec/dec_2020_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "\n",
    "### Combine month CSV Files \n",
    "1. Read in files in folders for each state\n",
    "2. Export as combined CSV"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 185,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create variable for files in directory\n",
    "files = [f for f in os.listdir(\"data/day/NE/dec/\") if f.endswith(\".csv\")]\n",
    "\n",
    "# files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 186,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>Month</th>\n",
       "      <th>Year</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>City</th>\n",
       "      <th>Temperature_ctrl_ave</th>\n",
       "      <th>TemperatureExpectedCool_ave</th>\n",
       "      <th>TemperatureExpectedHeat_ave</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>05320d509a18b2d4022029409e7a267736e78e0b</td>\n",
       "      <td>dec</td>\n",
       "      <td>2017</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>Lincoln</td>\n",
       "      <td>706.447368</td>\n",
       "      <td>728.157895</td>\n",
       "      <td>727.947368</td>\n",
       "      <td>90.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>05320d509a18b2d4022029409e7a267736e78e0b</td>\n",
       "      <td>dec</td>\n",
       "      <td>2017</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>Lincoln</td>\n",
       "      <td>722.527473</td>\n",
       "      <td>731.934066</td>\n",
       "      <td>731.846154</td>\n",
       "      <td>90.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>07ab14a1e62abe24dbdc3c8c819ab2ba7d93c8bd</td>\n",
       "      <td>dec</td>\n",
       "      <td>2017</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>Elkhorn</td>\n",
       "      <td>681.927798</td>\n",
       "      <td>696.025271</td>\n",
       "      <td>677.830325</td>\n",
       "      <td>20.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>07ab14a1e62abe24dbdc3c8c819ab2ba7d93c8bd</td>\n",
       "      <td>dec</td>\n",
       "      <td>2017</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>Elkhorn</td>\n",
       "      <td>687.792169</td>\n",
       "      <td>688.494578</td>\n",
       "      <td>688.494578</td>\n",
       "      <td>20.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0888869bed67808acc6deba6f1d8c0ebc69e5d79</td>\n",
       "      <td>dec</td>\n",
       "      <td>2017</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>Omaha</td>\n",
       "      <td>695.849315</td>\n",
       "      <td>696.712329</td>\n",
       "      <td>696.712329</td>\n",
       "      <td>15.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>358</th>\n",
       "      <td>faf3b76e8b0193bcf0256358efb4e593b669a1c7</td>\n",
       "      <td>dec</td>\n",
       "      <td>2020</td>\n",
       "      <td>auxHeatOnly</td>\n",
       "      <td>hold</td>\n",
       "      <td>Hickman</td>\n",
       "      <td>704.368421</td>\n",
       "      <td>711.789474</td>\n",
       "      <td>709.105263</td>\n",
       "      <td>0.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>359</th>\n",
       "      <td>faf3b76e8b0193bcf0256358efb4e593b669a1c7</td>\n",
       "      <td>dec</td>\n",
       "      <td>2020</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>Hickman</td>\n",
       "      <td>696.130435</td>\n",
       "      <td>700.521739</td>\n",
       "      <td>699.869565</td>\n",
       "      <td>0.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>360</th>\n",
       "      <td>faf3b76e8b0193bcf0256358efb4e593b669a1c7</td>\n",
       "      <td>dec</td>\n",
       "      <td>2020</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>Hickman</td>\n",
       "      <td>698.282609</td>\n",
       "      <td>702.489130</td>\n",
       "      <td>702.146739</td>\n",
       "      <td>0.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>361</th>\n",
       "      <td>ff8d6679e9ce5c1228cf640311c0110b3aafb5bd</td>\n",
       "      <td>dec</td>\n",
       "      <td>2020</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>Omaha</td>\n",
       "      <td>718.434537</td>\n",
       "      <td>721.476298</td>\n",
       "      <td>721.476298</td>\n",
       "      <td>0.0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>362</th>\n",
       "      <td>ff8d6679e9ce5c1228cf640311c0110b3aafb5bd</td>\n",
       "      <td>dec</td>\n",
       "      <td>2020</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>Omaha</td>\n",
       "      <td>717.307727</td>\n",
       "      <td>720.608307</td>\n",
       "      <td>720.608307</td>\n",
       "      <td>0.0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1367 rows × 13 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                   Identifier Month  Year     HvacMode  \\\n",
       "0    05320d509a18b2d4022029409e7a267736e78e0b   dec  2017         heat   \n",
       "1    05320d509a18b2d4022029409e7a267736e78e0b   dec  2017         heat   \n",
       "2    07ab14a1e62abe24dbdc3c8c819ab2ba7d93c8bd   dec  2017         heat   \n",
       "3    07ab14a1e62abe24dbdc3c8c819ab2ba7d93c8bd   dec  2017         heat   \n",
       "4    0888869bed67808acc6deba6f1d8c0ebc69e5d79   dec  2017         heat   \n",
       "..                                        ...   ...   ...          ...   \n",
       "358  faf3b76e8b0193bcf0256358efb4e593b669a1c7   dec  2020  auxHeatOnly   \n",
       "359  faf3b76e8b0193bcf0256358efb4e593b669a1c7   dec  2020         heat   \n",
       "360  faf3b76e8b0193bcf0256358efb4e593b669a1c7   dec  2020         heat   \n",
       "361  ff8d6679e9ce5c1228cf640311c0110b3aafb5bd   dec  2020         heat   \n",
       "362  ff8d6679e9ce5c1228cf640311c0110b3aafb5bd   dec  2020         heat   \n",
       "\n",
       "    CalendarEvent     City  Temperature_ctrl_ave  TemperatureExpectedCool_ave  \\\n",
       "0            auto  Lincoln            706.447368                   728.157895   \n",
       "1            hold  Lincoln            722.527473                   731.934066   \n",
       "2            auto  Elkhorn            681.927798                   696.025271   \n",
       "3            hold  Elkhorn            687.792169                   688.494578   \n",
       "4            hold    Omaha            695.849315                   696.712329   \n",
       "..            ...      ...                   ...                          ...   \n",
       "358          hold  Hickman            704.368421                   711.789474   \n",
       "359          auto  Hickman            696.130435                   700.521739   \n",
       "360          hold  Hickman            698.282609                   702.489130   \n",
       "361          auto    Omaha            718.434537                   721.476298   \n",
       "362          hold    Omaha            717.307727                   720.608307   \n",
       "\n",
       "     TemperatureExpectedHeat_ave  Age_of_Home__years_  allowCompWithAux  \\\n",
       "0                     727.947368                 90.0             False   \n",
       "1                     731.846154                 90.0             False   \n",
       "2                     677.830325                 20.0             False   \n",
       "3                     688.494578                 20.0             False   \n",
       "4                     696.712329                 15.0             False   \n",
       "..                           ...                  ...               ...   \n",
       "358                   709.105263                  0.0             False   \n",
       "359                   699.869565                  0.0             False   \n",
       "360                   702.146739                  0.0             False   \n",
       "361                   721.476298                  0.0              True   \n",
       "362                   720.608307                  0.0              True   \n",
       "\n",
       "     Has_Electric  Has_a_Heat_Pump  \n",
       "0           False            False  \n",
       "1           False            False  \n",
       "2           False            False  \n",
       "3           False            False  \n",
       "4           False            False  \n",
       "..            ...              ...  \n",
       "358         False             True  \n",
       "359         False             True  \n",
       "360         False             True  \n",
       "361         False             True  \n",
       "362         False             True  \n",
       "\n",
       "[1367 rows x 13 columns]"
      ]
     },
     "execution_count": 186,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# https://stackoverflow.com/questions/63886787/how-to-create-a-dataframe-from-multiple-csv-files\n",
    "# Initialize an empty dataframe\n",
    "NE_dec = pd.DataFrame()\n",
    "\n",
    "# Iterate through files and contents, then concatenate the data into the dataframe\n",
    "for file in files:\n",
    "    df = pd.read_csv(\"data/day/NE/dec/\" + file)\n",
    "    NE_dec = pd.concat([NE_dec, df])\n",
    "    \n",
    "NE_dec"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 187,
   "metadata": {},
   "outputs": [],
   "source": [
    "NE_dec.to_csv(\"Scraper_Output/State_Month_Day/NE/NE_dec.csv\", header=True, index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "----"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "----"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "\n",
    "### Combine state CSV Files \n",
    "1. Read in files in folders for each state\n",
    "2. Export as combined CSV"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 188,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create variable for files in directory\n",
    "files = [f for f in os.listdir(\"Scraper_Output/State_Month_Day/NE/\") if f.endswith(\".csv\")]\n",
    "\n",
    "# files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 189,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>Month</th>\n",
       "      <th>Year</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>City</th>\n",
       "      <th>Temperature_ctrl_ave</th>\n",
       "      <th>TemperatureExpectedCool_ave</th>\n",
       "      <th>TemperatureExpectedHeat_ave</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0087ac1d7540ff5efa6ae20213627943a2b07fe8</td>\n",
       "      <td>aug</td>\n",
       "      <td>2017</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>Omaha</td>\n",
       "      <td>741.040000</td>\n",
       "      <td>740.000000</td>\n",
       "      <td>680.000000</td>\n",
       "      <td>35.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>03fb21c8de5320f353144ac52422f977cfb769f1</td>\n",
       "      <td>aug</td>\n",
       "      <td>2017</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>York</td>\n",
       "      <td>735.913706</td>\n",
       "      <td>734.137056</td>\n",
       "      <td>706.218274</td>\n",
       "      <td>10.0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>05320d509a18b2d4022029409e7a267736e78e0b</td>\n",
       "      <td>aug</td>\n",
       "      <td>2017</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>Lincoln</td>\n",
       "      <td>719.923077</td>\n",
       "      <td>715.726496</td>\n",
       "      <td>700.000000</td>\n",
       "      <td>90.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>05320d509a18b2d4022029409e7a267736e78e0b</td>\n",
       "      <td>aug</td>\n",
       "      <td>2017</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>Lincoln</td>\n",
       "      <td>721.363946</td>\n",
       "      <td>726.908163</td>\n",
       "      <td>726.353741</td>\n",
       "      <td>90.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0ad90268dcfa7e7aa1d92768a5d17e1841ceaf17</td>\n",
       "      <td>aug</td>\n",
       "      <td>2017</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>McCool Junction</td>\n",
       "      <td>756.729167</td>\n",
       "      <td>775.466667</td>\n",
       "      <td>775.245833</td>\n",
       "      <td>15.0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1616</th>\n",
       "      <td>faa85f28edb3a9a38ac6130d63b1c3f892466036</td>\n",
       "      <td>jun</td>\n",
       "      <td>2021</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>Omaha</td>\n",
       "      <td>727.871901</td>\n",
       "      <td>741.438017</td>\n",
       "      <td>712.714876</td>\n",
       "      <td>55.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1617</th>\n",
       "      <td>faa8c435b84d052676a86ba7d2d6a783d26e6b19</td>\n",
       "      <td>jun</td>\n",
       "      <td>2021</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>Omaha</td>\n",
       "      <td>737.666667</td>\n",
       "      <td>700.000000</td>\n",
       "      <td>680.000000</td>\n",
       "      <td>20.0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1618</th>\n",
       "      <td>faa8c435b84d052676a86ba7d2d6a783d26e6b19</td>\n",
       "      <td>jun</td>\n",
       "      <td>2021</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>Omaha</td>\n",
       "      <td>705.419355</td>\n",
       "      <td>701.072581</td>\n",
       "      <td>701.072581</td>\n",
       "      <td>20.0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1619</th>\n",
       "      <td>faf3b76e8b0193bcf0256358efb4e593b669a1c7</td>\n",
       "      <td>jun</td>\n",
       "      <td>2021</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>Hickman</td>\n",
       "      <td>746.884615</td>\n",
       "      <td>716.000000</td>\n",
       "      <td>666.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1620</th>\n",
       "      <td>faf3b76e8b0193bcf0256358efb4e593b669a1c7</td>\n",
       "      <td>jun</td>\n",
       "      <td>2021</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>Hickman</td>\n",
       "      <td>737.982456</td>\n",
       "      <td>740.210526</td>\n",
       "      <td>730.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>8365 rows × 13 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                    Identifier Month  Year HvacMode  \\\n",
       "0     0087ac1d7540ff5efa6ae20213627943a2b07fe8   aug  2017     auto   \n",
       "1     03fb21c8de5320f353144ac52422f977cfb769f1   aug  2017     cool   \n",
       "2     05320d509a18b2d4022029409e7a267736e78e0b   aug  2017     cool   \n",
       "3     05320d509a18b2d4022029409e7a267736e78e0b   aug  2017     cool   \n",
       "4     0ad90268dcfa7e7aa1d92768a5d17e1841ceaf17   aug  2017     cool   \n",
       "...                                        ...   ...   ...      ...   \n",
       "1616  faa85f28edb3a9a38ac6130d63b1c3f892466036   jun  2021     cool   \n",
       "1617  faa8c435b84d052676a86ba7d2d6a783d26e6b19   jun  2021     auto   \n",
       "1618  faa8c435b84d052676a86ba7d2d6a783d26e6b19   jun  2021     cool   \n",
       "1619  faf3b76e8b0193bcf0256358efb4e593b669a1c7   jun  2021     auto   \n",
       "1620  faf3b76e8b0193bcf0256358efb4e593b669a1c7   jun  2021     cool   \n",
       "\n",
       "     CalendarEvent             City  Temperature_ctrl_ave  \\\n",
       "0             hold            Omaha            741.040000   \n",
       "1             hold             York            735.913706   \n",
       "2             auto          Lincoln            719.923077   \n",
       "3             hold          Lincoln            721.363946   \n",
       "4             hold  McCool Junction            756.729167   \n",
       "...            ...              ...                   ...   \n",
       "1616          hold            Omaha            727.871901   \n",
       "1617          hold            Omaha            737.666667   \n",
       "1618          hold            Omaha            705.419355   \n",
       "1619          hold          Hickman            746.884615   \n",
       "1620          hold          Hickman            737.982456   \n",
       "\n",
       "      TemperatureExpectedCool_ave  TemperatureExpectedHeat_ave  \\\n",
       "0                      740.000000                   680.000000   \n",
       "1                      734.137056                   706.218274   \n",
       "2                      715.726496                   700.000000   \n",
       "3                      726.908163                   726.353741   \n",
       "4                      775.466667                   775.245833   \n",
       "...                           ...                          ...   \n",
       "1616                   741.438017                   712.714876   \n",
       "1617                   700.000000                   680.000000   \n",
       "1618                   701.072581                   701.072581   \n",
       "1619                   716.000000                   666.000000   \n",
       "1620                   740.210526                   730.000000   \n",
       "\n",
       "      Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \n",
       "0                    35.0             False         False            False  \n",
       "1                    10.0              True         False             True  \n",
       "2                    90.0             False         False            False  \n",
       "3                    90.0             False         False            False  \n",
       "4                    15.0              True         False             True  \n",
       "...                   ...               ...           ...              ...  \n",
       "1616                 55.0             False         False            False  \n",
       "1617                 20.0              True         False            False  \n",
       "1618                 20.0              True         False            False  \n",
       "1619                  0.0             False         False             True  \n",
       "1620                  0.0             False         False             True  \n",
       "\n",
       "[8365 rows x 13 columns]"
      ]
     },
     "execution_count": 189,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# https://stackoverflow.com/questions/63886787/how-to-create-a-dataframe-from-multiple-csv-files\n",
    "# Initialize an empty dataframe\n",
    "NE_all = pd.DataFrame()\n",
    "\n",
    "# Iterate through files and contents, then concatenate the data into the dataframe\n",
    "for file in files:\n",
    "    df = pd.read_csv(\"Scraper_Output/State_Month_Day/NE/\" + file)\n",
    "    NE_all = pd.concat([NE_all, df])\n",
    "    \n",
    "NE_all"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 190,
   "metadata": {},
   "outputs": [],
   "source": [
    "NE_all.to_csv(\"Scraper_Output/State_Month_Day/NE_all_day.csv\", header=True, index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 191,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Unique jan_2017: ['NE']\n",
      "Unique jan_2018: ['NE']\n",
      "Unique jan_2019: ['NE']\n",
      "Unique jan_2020: ['NE']\n",
      "Unique jan_2021: ['NE']\n",
      "Unique feb_2017: ['NE']\n",
      "Unique feb_2018: ['NE']\n",
      "Unique feb_2019: ['NE']\n",
      "Unique feb_2020: ['NE']\n",
      "Unique feb_2021: ['NE']\n",
      "Unique jun_2017: ['NE']\n",
      "Unique jun_2018: ['NE']\n",
      "Unique jun_2019: ['NE']\n",
      "Unique jun_2020: ['NE']\n",
      "Unique jun_2021: ['NE']\n",
      "Unique jul_2017: ['NE']\n",
      "Unique jul_2018: ['NE']\n",
      "Unique jul_2019: ['NE']\n",
      "Unique jul_2020: ['NE']\n",
      "Unique jul_2021: ['NE']\n",
      "Unique aug_2017: ['NE']\n",
      "Unique aug_2018: ['NE']\n",
      "Unique aug_2019: ['NE']\n",
      "Unique aug_2020: ['NE']\n",
      "Unique dec_2017: ['NE']\n",
      "Unique dec_2018: ['NE']\n",
      "Unique dec_2019: ['NE']\n",
      "Unique dec_2020: ['NE']\n"
     ]
    }
   ],
   "source": [
    "# Datacheck to mNEe sure state was selected correctly in BQ sql queries\n",
    "\n",
    "print(f\"Unique jan_2017: {jan_2017['ProvinceState'].unique()}\")\n",
    "print(f\"Unique jan_2018: {jan_2018['ProvinceState'].unique()}\")\n",
    "print(f\"Unique jan_2019: {jan_2019['ProvinceState'].unique()}\")\n",
    "print(f\"Unique jan_2020: {jan_2020['ProvinceState'].unique()}\")\n",
    "print(f\"Unique jan_2021: {jan_2021['ProvinceState'].unique()}\")\n",
    "print(f\"Unique feb_2017: {feb_2017['ProvinceState'].unique()}\")\n",
    "print(f\"Unique feb_2018: {feb_2018['ProvinceState'].unique()}\")\n",
    "print(f\"Unique feb_2019: {feb_2019['ProvinceState'].unique()}\")\n",
    "print(f\"Unique feb_2020: {feb_2020['ProvinceState'].unique()}\")\n",
    "print(f\"Unique feb_2021: {feb_2021['ProvinceState'].unique()}\")\n",
    "print(f\"Unique jun_2017: {jun_2017['ProvinceState'].unique()}\")\n",
    "print(f\"Unique jun_2018: {jun_2018['ProvinceState'].unique()}\")\n",
    "print(f\"Unique jun_2019: {jun_2019['ProvinceState'].unique()}\")\n",
    "print(f\"Unique jun_2020: {jun_2020['ProvinceState'].unique()}\")\n",
    "print(f\"Unique jun_2021: {jun_2021['ProvinceState'].unique()}\")\n",
    "print(f\"Unique jul_2017: {jul_2017['ProvinceState'].unique()}\")\n",
    "print(f\"Unique jul_2018: {jul_2018['ProvinceState'].unique()}\")\n",
    "print(f\"Unique jul_2019: {jul_2019['ProvinceState'].unique()}\")\n",
    "print(f\"Unique jul_2020: {jul_2020['ProvinceState'].unique()}\")\n",
    "print(f\"Unique jul_2021: {jul_2021['ProvinceState'].unique()}\")\n",
    "print(f\"Unique aug_2017: {aug_2017['ProvinceState'].unique()}\")\n",
    "print(f\"Unique aug_2018: {aug_2018['ProvinceState'].unique()}\")\n",
    "print(f\"Unique aug_2019: {aug_2019['ProvinceState'].unique()}\")\n",
    "print(f\"Unique aug_2020: {aug_2020['ProvinceState'].unique()}\")\n",
    "print(f\"Unique dec_2017: {dec_2017['ProvinceState'].unique()}\")\n",
    "print(f\"Unique dec_2018: {dec_2018['ProvinceState'].unique()}\")\n",
    "print(f\"Unique dec_2019: {dec_2019['ProvinceState'].unique()}\")\n",
    "print(f\"Unique dec_2020: {dec_2020['ProvinceState'].unique()}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.10"
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}