DYD_NC-day.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# DYD Thermostat Data \n",
    "\n",
    "## Preprocess\n",
    "\n",
    "1. Generated CSV file from queries in BigQueary\n",
    "\n",
    "2. Data separated into states\n",
    "\n",
    "3. Aggregated the data in Pandas by month\n",
    "\n",
    "4. Combine 4 years\n",
    "\n",
    "5. Group by Identifier\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Dependencies\n",
    "import pandas as pd\n",
    "import os\n",
    "import numpy as np\n",
    "from pathlib import Path\n",
    "from datetime import datetime"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "## January"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2017 January Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "jan_2017 = pd.read_csv(\"../data_large/NC-day/2017-jan-day-NC.csv\")\n",
    "\n",
    "# jan_2017"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>3a6c99e2c5bdb4afcff1db623695263b0728305c</td>\n",
       "      <td>2017-01-23 12:30:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>665</td>\n",
       "      <td>760</td>\n",
       "      <td>650</td>\n",
       "      <td>NC</td>\n",
       "      <td>Frisco</td>\n",
       "      <td>25</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>b561319e1d44f031114a08344d28812d66c7fc1f</td>\n",
       "      <td>2017-01-27 19:15:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>704</td>\n",
       "      <td>720</td>\n",
       "      <td>640</td>\n",
       "      <td>NC</td>\n",
       "      <td>Charlotte</td>\n",
       "      <td>10</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>75c53f0c421691c90dec629f5de252904a81ddce</td>\n",
       "      <td>2017-01-01 17:15:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>702</td>\n",
       "      <td>750</td>\n",
       "      <td>700</td>\n",
       "      <td>NC</td>\n",
       "      <td>Wilmington</td>\n",
       "      <td>120</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>24aadbbd65aa6cfde235f48acda995afabd4fca6</td>\n",
       "      <td>2017-01-05 13:55:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>656</td>\n",
       "      <td>780</td>\n",
       "      <td>660</td>\n",
       "      <td>NC</td>\n",
       "      <td>Saluda</td>\n",
       "      <td>0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>bcb4c4251c7b5a0a1dd895a0adbfefc6a9dea053</td>\n",
       "      <td>2017-01-07 10:50:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>610</td>\n",
       "      <td>650</td>\n",
       "      <td>610</td>\n",
       "      <td>NC</td>\n",
       "      <td>Garner</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>483501</th>\n",
       "      <td>ffe96e4925748a4af930a70df17244664778168e</td>\n",
       "      <td>2017-01-20 14:05:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>725</td>\n",
       "      <td>780</td>\n",
       "      <td>730</td>\n",
       "      <td>NC</td>\n",
       "      <td>Cary</td>\n",
       "      <td>6</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>483502</th>\n",
       "      <td>ffe96e4925748a4af930a70df17244664778168e</td>\n",
       "      <td>2017-01-14 11:30:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>736</td>\n",
       "      <td>780</td>\n",
       "      <td>730</td>\n",
       "      <td>NC</td>\n",
       "      <td>Cary</td>\n",
       "      <td>6</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>483503</th>\n",
       "      <td>ffe96e4925748a4af930a70df17244664778168e</td>\n",
       "      <td>2017-01-06 17:15:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>712</td>\n",
       "      <td>755</td>\n",
       "      <td>705</td>\n",
       "      <td>NC</td>\n",
       "      <td>Cary</td>\n",
       "      <td>6</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>483504</th>\n",
       "      <td>ffe96e4925748a4af930a70df17244664778168e</td>\n",
       "      <td>2017-01-14 16:55:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>737</td>\n",
       "      <td>780</td>\n",
       "      <td>730</td>\n",
       "      <td>NC</td>\n",
       "      <td>Cary</td>\n",
       "      <td>6</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>483505</th>\n",
       "      <td>ffe96e4925748a4af930a70df17244664778168e</td>\n",
       "      <td>2017-01-22 15:10:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>728</td>\n",
       "      <td>780</td>\n",
       "      <td>730</td>\n",
       "      <td>NC</td>\n",
       "      <td>Cary</td>\n",
       "      <td>6</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>453226 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                      Identifier                date_time  \\\n",
       "0       3a6c99e2c5bdb4afcff1db623695263b0728305c  2017-01-23 12:30:00 UTC   \n",
       "1       b561319e1d44f031114a08344d28812d66c7fc1f  2017-01-27 19:15:00 UTC   \n",
       "2       75c53f0c421691c90dec629f5de252904a81ddce  2017-01-01 17:15:00 UTC   \n",
       "3       24aadbbd65aa6cfde235f48acda995afabd4fca6  2017-01-05 13:55:00 UTC   \n",
       "5       bcb4c4251c7b5a0a1dd895a0adbfefc6a9dea053  2017-01-07 10:50:00 UTC   \n",
       "...                                          ...                      ...   \n",
       "483501  ffe96e4925748a4af930a70df17244664778168e  2017-01-20 14:05:00 UTC   \n",
       "483502  ffe96e4925748a4af930a70df17244664778168e  2017-01-14 11:30:00 UTC   \n",
       "483503  ffe96e4925748a4af930a70df17244664778168e  2017-01-06 17:15:00 UTC   \n",
       "483504  ffe96e4925748a4af930a70df17244664778168e  2017-01-14 16:55:00 UTC   \n",
       "483505  ffe96e4925748a4af930a70df17244664778168e  2017-01-22 15:10:00 UTC   \n",
       "\n",
       "       HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0          auto          auto               665                      760   \n",
       "1          auto          hold               704                      720   \n",
       "2          auto          hold               702                      750   \n",
       "3          auto          auto               656                      780   \n",
       "5          heat          auto               610                      650   \n",
       "...         ...           ...               ...                      ...   \n",
       "483501     auto          hold               725                      780   \n",
       "483502     auto          hold               736                      780   \n",
       "483503     auto          auto               712                      755   \n",
       "483504     auto          hold               737                      780   \n",
       "483505     auto          hold               728                      780   \n",
       "\n",
       "        TemperatureExpectedHeat ProvinceState        City  \\\n",
       "0                           650            NC      Frisco   \n",
       "1                           640            NC   Charlotte   \n",
       "2                           700            NC  Wilmington   \n",
       "3                           660            NC      Saluda   \n",
       "5                           610            NC      Garner   \n",
       "...                         ...           ...         ...   \n",
       "483501                      730            NC        Cary   \n",
       "483502                      730            NC        Cary   \n",
       "483503                      705            NC        Cary   \n",
       "483504                      730            NC        Cary   \n",
       "483505                      730            NC        Cary   \n",
       "\n",
       "        Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                        25             False         False            False   \n",
       "1                        10              True         False             True   \n",
       "2                       120              True         False             True   \n",
       "3                         0              True         False             True   \n",
       "5                         0             False         False             True   \n",
       "...                     ...               ...           ...              ...   \n",
       "483501                    6             False         False            False   \n",
       "483502                    6             False         False            False   \n",
       "483503                    6             False         False            False   \n",
       "483504                    6             False         False            False   \n",
       "483505                    6             False         False            False   \n",
       "\n",
       "       Auxilliary_Heat_Fuel_Type  \n",
       "0                            Gas  \n",
       "1                       Electric  \n",
       "2                       Electric  \n",
       "3                       Electric  \n",
       "5                       Electric  \n",
       "...                          ...  \n",
       "483501                       Gas  \n",
       "483502                       Gas  \n",
       "483503                       Gas  \n",
       "483504                       Gas  \n",
       "483505                       Gas  \n",
       "\n",
       "[453226 rows x 14 columns]"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "jan_2017.drop(jan_2017[jan_2017['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "jan_2017.drop(jan_2017[jan_2017['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "jan_2017.drop(jan_2017[jan_2017['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "jan_2017.drop(jan_2017[jan_2017['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "jan_2017"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "jan_2017[\"Year\"] = \"2017\"\n",
    "jan_2017[\"Month\"] = \"Jan\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "jan_2017 = jan_2017.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>Temperature_ctrl_ave</th>\n",
       "      <th>TemperatureExpectedCool_ave</th>\n",
       "      <th>TemperatureExpectedHeat_ave</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Identifier</th>\n",
       "      <th>Month</th>\n",
       "      <th>Year</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>City</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>00bbbdb5c3b6cb8b9e78191017ecdb97dad87e47</th>\n",
       "      <th>Jan</th>\n",
       "      <th>2017</th>\n",
       "      <th>auto</th>\n",
       "      <th>auto</th>\n",
       "      <th>Jacksonville</th>\n",
       "      <td>689.997824</td>\n",
       "      <td>762.501632</td>\n",
       "      <td>687.490751</td>\n",
       "      <td>5.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>00e5498cd188bdf09a9a1a902e6084b200cbbdef</th>\n",
       "      <th>Jan</th>\n",
       "      <th>2017</th>\n",
       "      <th>auto</th>\n",
       "      <th>hold</th>\n",
       "      <th>Wade</th>\n",
       "      <td>686.206897</td>\n",
       "      <td>751.310345</td>\n",
       "      <td>688.275862</td>\n",
       "      <td>55.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>013708850f6d20343f635dc3c0130399b66b0578</th>\n",
       "      <th>Jan</th>\n",
       "      <th>2017</th>\n",
       "      <th>auto</th>\n",
       "      <th>hold</th>\n",
       "      <th>Concord</th>\n",
       "      <td>690.990385</td>\n",
       "      <td>750.528846</td>\n",
       "      <td>690.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>016c1867b7cc7622e183e8d74de2b93ddd9449a3</th>\n",
       "      <th>Jan</th>\n",
       "      <th>2017</th>\n",
       "      <th>heat</th>\n",
       "      <th>auto</th>\n",
       "      <th>Dobson</th>\n",
       "      <td>720.290960</td>\n",
       "      <td>710.104520</td>\n",
       "      <td>709.816384</td>\n",
       "      <td>15.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0172751a7d4d785df2216952e1be26ff7861a414</th>\n",
       "      <th>Jan</th>\n",
       "      <th>2017</th>\n",
       "      <th>heat</th>\n",
       "      <th>auto</th>\n",
       "      <th>Greensboro</th>\n",
       "      <td>708.029650</td>\n",
       "      <td>707.734501</td>\n",
       "      <td>705.371968</td>\n",
       "      <td>65.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <th>...</th>\n",
       "      <th>...</th>\n",
       "      <th>...</th>\n",
       "      <th>...</th>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">feabebfe6370877e7c62203ee78d24c1e38f6610</th>\n",
       "      <th rowspan=\"2\" valign=\"top\">Jan</th>\n",
       "      <th rowspan=\"2\" valign=\"top\">2017</th>\n",
       "      <th rowspan=\"2\" valign=\"top\">heat</th>\n",
       "      <th>auto</th>\n",
       "      <th>Huntersville</th>\n",
       "      <td>702.812747</td>\n",
       "      <td>693.426114</td>\n",
       "      <td>693.313311</td>\n",
       "      <td>0.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>hold</th>\n",
       "      <th>Huntersville</th>\n",
       "      <td>698.149123</td>\n",
       "      <td>689.263158</td>\n",
       "      <td>689.263158</td>\n",
       "      <td>0.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ff53874bcab6f2d086a50ca0766bcadebd61e62e</th>\n",
       "      <th>Jan</th>\n",
       "      <th>2017</th>\n",
       "      <th>heat</th>\n",
       "      <th>auto</th>\n",
       "      <th>Matthews</th>\n",
       "      <td>686.722222</td>\n",
       "      <td>769.152778</td>\n",
       "      <td>688.638889</td>\n",
       "      <td>40.0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">ffe96e4925748a4af930a70df17244664778168e</th>\n",
       "      <th rowspan=\"2\" valign=\"top\">Jan</th>\n",
       "      <th rowspan=\"2\" valign=\"top\">2017</th>\n",
       "      <th rowspan=\"2\" valign=\"top\">auto</th>\n",
       "      <th>auto</th>\n",
       "      <th>Cary</th>\n",
       "      <td>713.375000</td>\n",
       "      <td>763.333333</td>\n",
       "      <td>713.333333</td>\n",
       "      <td>6.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>hold</th>\n",
       "      <th>Cary</th>\n",
       "      <td>730.317149</td>\n",
       "      <td>780.000000</td>\n",
       "      <td>730.000000</td>\n",
       "      <td>6.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>941 rows × 7 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                                                         Temperature_ctrl_ave  \\\n",
       "Identifier                               Month Year HvacMode CalendarEvent City                                 \n",
       "00bbbdb5c3b6cb8b9e78191017ecdb97dad87e47 Jan   2017 auto     auto          Jacksonville            689.997824   \n",
       "00e5498cd188bdf09a9a1a902e6084b200cbbdef Jan   2017 auto     hold          Wade                    686.206897   \n",
       "013708850f6d20343f635dc3c0130399b66b0578 Jan   2017 auto     hold          Concord                 690.990385   \n",
       "016c1867b7cc7622e183e8d74de2b93ddd9449a3 Jan   2017 heat     auto          Dobson                  720.290960   \n",
       "0172751a7d4d785df2216952e1be26ff7861a414 Jan   2017 heat     auto          Greensboro              708.029650   \n",
       "...                                                                                                       ...   \n",
       "feabebfe6370877e7c62203ee78d24c1e38f6610 Jan   2017 heat     auto          Huntersville            702.812747   \n",
       "                                                             hold          Huntersville            698.149123   \n",
       "ff53874bcab6f2d086a50ca0766bcadebd61e62e Jan   2017 heat     auto          Matthews                686.722222   \n",
       "ffe96e4925748a4af930a70df17244664778168e Jan   2017 auto     auto          Cary                    713.375000   \n",
       "                                                             hold          Cary                    730.317149   \n",
       "\n",
       "                                                                                         TemperatureExpectedCool_ave  \\\n",
       "Identifier                               Month Year HvacMode CalendarEvent City                                        \n",
       "00bbbdb5c3b6cb8b9e78191017ecdb97dad87e47 Jan   2017 auto     auto          Jacksonville                   762.501632   \n",
       "00e5498cd188bdf09a9a1a902e6084b200cbbdef Jan   2017 auto     hold          Wade                           751.310345   \n",
       "013708850f6d20343f635dc3c0130399b66b0578 Jan   2017 auto     hold          Concord                        750.528846   \n",
       "016c1867b7cc7622e183e8d74de2b93ddd9449a3 Jan   2017 heat     auto          Dobson                         710.104520   \n",
       "0172751a7d4d785df2216952e1be26ff7861a414 Jan   2017 heat     auto          Greensboro                     707.734501   \n",
       "...                                                                                                              ...   \n",
       "feabebfe6370877e7c62203ee78d24c1e38f6610 Jan   2017 heat     auto          Huntersville                   693.426114   \n",
       "                                                             hold          Huntersville                   689.263158   \n",
       "ff53874bcab6f2d086a50ca0766bcadebd61e62e Jan   2017 heat     auto          Matthews                       769.152778   \n",
       "ffe96e4925748a4af930a70df17244664778168e Jan   2017 auto     auto          Cary                           763.333333   \n",
       "                                                             hold          Cary                           780.000000   \n",
       "\n",
       "                                                                                         TemperatureExpectedHeat_ave  \\\n",
       "Identifier                               Month Year HvacMode CalendarEvent City                                        \n",
       "00bbbdb5c3b6cb8b9e78191017ecdb97dad87e47 Jan   2017 auto     auto          Jacksonville                   687.490751   \n",
       "00e5498cd188bdf09a9a1a902e6084b200cbbdef Jan   2017 auto     hold          Wade                           688.275862   \n",
       "013708850f6d20343f635dc3c0130399b66b0578 Jan   2017 auto     hold          Concord                        690.000000   \n",
       "016c1867b7cc7622e183e8d74de2b93ddd9449a3 Jan   2017 heat     auto          Dobson                         709.816384   \n",
       "0172751a7d4d785df2216952e1be26ff7861a414 Jan   2017 heat     auto          Greensboro                     705.371968   \n",
       "...                                                                                                              ...   \n",
       "feabebfe6370877e7c62203ee78d24c1e38f6610 Jan   2017 heat     auto          Huntersville                   693.313311   \n",
       "                                                             hold          Huntersville                   689.263158   \n",
       "ff53874bcab6f2d086a50ca0766bcadebd61e62e Jan   2017 heat     auto          Matthews                       688.638889   \n",
       "ffe96e4925748a4af930a70df17244664778168e Jan   2017 auto     auto          Cary                           713.333333   \n",
       "                                                             hold          Cary                           730.000000   \n",
       "\n",
       "                                                                                         Age_of_Home__years_  \\\n",
       "Identifier                               Month Year HvacMode CalendarEvent City                                \n",
       "00bbbdb5c3b6cb8b9e78191017ecdb97dad87e47 Jan   2017 auto     auto          Jacksonville                  5.0   \n",
       "00e5498cd188bdf09a9a1a902e6084b200cbbdef Jan   2017 auto     hold          Wade                         55.0   \n",
       "013708850f6d20343f635dc3c0130399b66b0578 Jan   2017 auto     hold          Concord                       0.0   \n",
       "016c1867b7cc7622e183e8d74de2b93ddd9449a3 Jan   2017 heat     auto          Dobson                       15.0   \n",
       "0172751a7d4d785df2216952e1be26ff7861a414 Jan   2017 heat     auto          Greensboro                   65.0   \n",
       "...                                                                                                      ...   \n",
       "feabebfe6370877e7c62203ee78d24c1e38f6610 Jan   2017 heat     auto          Huntersville                  0.0   \n",
       "                                                             hold          Huntersville                  0.0   \n",
       "ff53874bcab6f2d086a50ca0766bcadebd61e62e Jan   2017 heat     auto          Matthews                     40.0   \n",
       "ffe96e4925748a4af930a70df17244664778168e Jan   2017 auto     auto          Cary                          6.0   \n",
       "                                                             hold          Cary                          6.0   \n",
       "\n",
       "                                                                                         allowCompWithAux  \\\n",
       "Identifier                               Month Year HvacMode CalendarEvent City                             \n",
       "00bbbdb5c3b6cb8b9e78191017ecdb97dad87e47 Jan   2017 auto     auto          Jacksonville             False   \n",
       "00e5498cd188bdf09a9a1a902e6084b200cbbdef Jan   2017 auto     hold          Wade                     False   \n",
       "013708850f6d20343f635dc3c0130399b66b0578 Jan   2017 auto     hold          Concord                  False   \n",
       "016c1867b7cc7622e183e8d74de2b93ddd9449a3 Jan   2017 heat     auto          Dobson                   False   \n",
       "0172751a7d4d785df2216952e1be26ff7861a414 Jan   2017 heat     auto          Greensboro               False   \n",
       "...                                                                                                   ...   \n",
       "feabebfe6370877e7c62203ee78d24c1e38f6610 Jan   2017 heat     auto          Huntersville             False   \n",
       "                                                             hold          Huntersville             False   \n",
       "ff53874bcab6f2d086a50ca0766bcadebd61e62e Jan   2017 heat     auto          Matthews                  True   \n",
       "ffe96e4925748a4af930a70df17244664778168e Jan   2017 auto     auto          Cary                     False   \n",
       "                                                             hold          Cary                     False   \n",
       "\n",
       "                                                                                         Has_Electric  \\\n",
       "Identifier                               Month Year HvacMode CalendarEvent City                         \n",
       "00bbbdb5c3b6cb8b9e78191017ecdb97dad87e47 Jan   2017 auto     auto          Jacksonville         False   \n",
       "00e5498cd188bdf09a9a1a902e6084b200cbbdef Jan   2017 auto     hold          Wade                 False   \n",
       "013708850f6d20343f635dc3c0130399b66b0578 Jan   2017 auto     hold          Concord              False   \n",
       "016c1867b7cc7622e183e8d74de2b93ddd9449a3 Jan   2017 heat     auto          Dobson               False   \n",
       "0172751a7d4d785df2216952e1be26ff7861a414 Jan   2017 heat     auto          Greensboro           False   \n",
       "...                                                                                               ...   \n",
       "feabebfe6370877e7c62203ee78d24c1e38f6610 Jan   2017 heat     auto          Huntersville         False   \n",
       "                                                             hold          Huntersville         False   \n",
       "ff53874bcab6f2d086a50ca0766bcadebd61e62e Jan   2017 heat     auto          Matthews             False   \n",
       "ffe96e4925748a4af930a70df17244664778168e Jan   2017 auto     auto          Cary                 False   \n",
       "                                                             hold          Cary                 False   \n",
       "\n",
       "                                                                                         Has_a_Heat_Pump  \n",
       "Identifier                               Month Year HvacMode CalendarEvent City                           \n",
       "00bbbdb5c3b6cb8b9e78191017ecdb97dad87e47 Jan   2017 auto     auto          Jacksonville             True  \n",
       "00e5498cd188bdf09a9a1a902e6084b200cbbdef Jan   2017 auto     hold          Wade                    False  \n",
       "013708850f6d20343f635dc3c0130399b66b0578 Jan   2017 auto     hold          Concord                 False  \n",
       "016c1867b7cc7622e183e8d74de2b93ddd9449a3 Jan   2017 heat     auto          Dobson                   True  \n",
       "0172751a7d4d785df2216952e1be26ff7861a414 Jan   2017 heat     auto          Greensboro              False  \n",
       "...                                                                                                  ...  \n",
       "feabebfe6370877e7c62203ee78d24c1e38f6610 Jan   2017 heat     auto          Huntersville            False  \n",
       "                                                             hold          Huntersville            False  \n",
       "ff53874bcab6f2d086a50ca0766bcadebd61e62e Jan   2017 heat     auto          Matthews                 True  \n",
       "ffe96e4925748a4af930a70df17244664778168e Jan   2017 auto     auto          Cary                    False  \n",
       "                                                             hold          Cary                    False  \n",
       "\n",
       "[941 rows x 7 columns]"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "jan_2017_ave = jan_2017.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "jan_2017_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "jan_2017_ave.to_csv(\"data/day/NC/jan/jan_2017_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2018 January Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "jan_2018 = pd.read_csv(\"../data_large/NC-day/2018-jan-day-NC.csv\")\n",
    "\n",
    "# jan_2018"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>33b36b20c9c2fa019823f18d1dcc07bd351aa9b0</td>\n",
       "      <td>2018-01-30 17:05:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>666</td>\n",
       "      <td>665</td>\n",
       "      <td>665</td>\n",
       "      <td>NC</td>\n",
       "      <td>Greensboro</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>f61cfa0f4be88d5562c709cf4739f9162b259b8b</td>\n",
       "      <td>2018-01-21 12:45:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>722</td>\n",
       "      <td>775</td>\n",
       "      <td>725</td>\n",
       "      <td>NC</td>\n",
       "      <td>High Point</td>\n",
       "      <td>17</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>adc01ca5408d5656af2e3aa0101d6ded5f7ac702</td>\n",
       "      <td>2018-01-26 15:00:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>665</td>\n",
       "      <td>694</td>\n",
       "      <td>694</td>\n",
       "      <td>NC</td>\n",
       "      <td>Durham</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3e4bfd3306a20c34031fee16fed65095a54825ae</td>\n",
       "      <td>2018-01-02 18:40:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>760</td>\n",
       "      <td>795</td>\n",
       "      <td>745</td>\n",
       "      <td>NC</td>\n",
       "      <td>Raleigh</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>741378eb5d4c12b2beef4f22df3cfca96e826b76</td>\n",
       "      <td>2018-01-02 13:40:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>728</td>\n",
       "      <td>728</td>\n",
       "      <td>728</td>\n",
       "      <td>NC</td>\n",
       "      <td>Cary</td>\n",
       "      <td>20</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1313272</th>\n",
       "      <td>33008b03223e678c72f11f6501d1ec4889a3c804</td>\n",
       "      <td>2018-01-22 13:30:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>727</td>\n",
       "      <td>765</td>\n",
       "      <td>715</td>\n",
       "      <td>NC</td>\n",
       "      <td>Oak Ridge</td>\n",
       "      <td>20</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1313273</th>\n",
       "      <td>5abf7828789fa4adbf80999680757dc3023ca061</td>\n",
       "      <td>2018-01-06 15:25:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>723</td>\n",
       "      <td>765</td>\n",
       "      <td>715</td>\n",
       "      <td>NC</td>\n",
       "      <td>Charlotte</td>\n",
       "      <td>15</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1313274</th>\n",
       "      <td>31e864a282eea5243339482ba0296c4c6396bc41</td>\n",
       "      <td>2018-01-26 13:30:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>698</td>\n",
       "      <td>765</td>\n",
       "      <td>695</td>\n",
       "      <td>NC</td>\n",
       "      <td>Summerfield</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1313275</th>\n",
       "      <td>33008b03223e678c72f11f6501d1ec4889a3c804</td>\n",
       "      <td>2018-01-02 14:10:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>707</td>\n",
       "      <td>765</td>\n",
       "      <td>715</td>\n",
       "      <td>NC</td>\n",
       "      <td>Oak Ridge</td>\n",
       "      <td>20</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1313276</th>\n",
       "      <td>2d8ef2a298f368c522c875ac0dde959a48c8b0ba</td>\n",
       "      <td>2018-01-17 13:50:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>712</td>\n",
       "      <td>765</td>\n",
       "      <td>715</td>\n",
       "      <td>NC</td>\n",
       "      <td>Cary</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1249399 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                       Identifier                date_time  \\\n",
       "0        33b36b20c9c2fa019823f18d1dcc07bd351aa9b0  2018-01-30 17:05:00 UTC   \n",
       "1        f61cfa0f4be88d5562c709cf4739f9162b259b8b  2018-01-21 12:45:00 UTC   \n",
       "2        adc01ca5408d5656af2e3aa0101d6ded5f7ac702  2018-01-26 15:00:00 UTC   \n",
       "3        3e4bfd3306a20c34031fee16fed65095a54825ae  2018-01-02 18:40:00 UTC   \n",
       "4        741378eb5d4c12b2beef4f22df3cfca96e826b76  2018-01-02 13:40:00 UTC   \n",
       "...                                           ...                      ...   \n",
       "1313272  33008b03223e678c72f11f6501d1ec4889a3c804  2018-01-22 13:30:00 UTC   \n",
       "1313273  5abf7828789fa4adbf80999680757dc3023ca061  2018-01-06 15:25:00 UTC   \n",
       "1313274  31e864a282eea5243339482ba0296c4c6396bc41  2018-01-26 13:30:00 UTC   \n",
       "1313275  33008b03223e678c72f11f6501d1ec4889a3c804  2018-01-02 14:10:00 UTC   \n",
       "1313276  2d8ef2a298f368c522c875ac0dde959a48c8b0ba  2018-01-17 13:50:00 UTC   \n",
       "\n",
       "        HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0           heat          hold               666                      665   \n",
       "1           auto          auto               722                      775   \n",
       "2           heat          auto               665                      694   \n",
       "3           auto          hold               760                      795   \n",
       "4           heat          hold               728                      728   \n",
       "...          ...           ...               ...                      ...   \n",
       "1313272     heat          hold               727                      765   \n",
       "1313273     auto          auto               723                      765   \n",
       "1313274     auto          hold               698                      765   \n",
       "1313275     auto          hold               707                      765   \n",
       "1313276     auto          auto               712                      765   \n",
       "\n",
       "         TemperatureExpectedHeat ProvinceState         City  \\\n",
       "0                            665            NC   Greensboro   \n",
       "1                            725            NC   High Point   \n",
       "2                            694            NC       Durham   \n",
       "3                            745            NC      Raleigh   \n",
       "4                            728            NC         Cary   \n",
       "...                          ...           ...          ...   \n",
       "1313272                      715            NC    Oak Ridge   \n",
       "1313273                      715            NC    Charlotte   \n",
       "1313274                      695            NC  Summerfield   \n",
       "1313275                      715            NC    Oak Ridge   \n",
       "1313276                      715            NC         Cary   \n",
       "\n",
       "         Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                          0             False         False             True   \n",
       "1                         17             False         False            False   \n",
       "2                          0             False         False            False   \n",
       "3                         10             False         False            False   \n",
       "4                         20             False         False            False   \n",
       "...                      ...               ...           ...              ...   \n",
       "1313272                   20             False         False            False   \n",
       "1313273                   15             False         False             True   \n",
       "1313274                    0             False         False            False   \n",
       "1313275                   20             False         False            False   \n",
       "1313276                    5             False         False            False   \n",
       "\n",
       "        Auxilliary_Heat_Fuel_Type  \n",
       "0                        Electric  \n",
       "1                             Gas  \n",
       "2                             Gas  \n",
       "3                             Gas  \n",
       "4                             Gas  \n",
       "...                           ...  \n",
       "1313272                       Gas  \n",
       "1313273                  Electric  \n",
       "1313274                       Gas  \n",
       "1313275                       Gas  \n",
       "1313276                       Gas  \n",
       "\n",
       "[1249399 rows x 14 columns]"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "jan_2018.drop(jan_2018[jan_2018['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "jan_2018.drop(jan_2018[jan_2018['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "jan_2018.drop(jan_2018[jan_2018['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "jan_2018.drop(jan_2018[jan_2018['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "jan_2018"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "jan_2018[\"Year\"] = \"2018\"\n",
    "jan_2018[\"Month\"] = \"Jan\"\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "jan_2018 = jan_2018.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "jan_2018_ave = jan_2018.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# jan_2018_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "jan_2018_ave.to_csv(\"data/day/NC/jan/jan_2018_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2019 January Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "jan_2019 = pd.read_csv(\"../data_large/NC-day/2019-jan-day-NC.csv\")\n",
    "\n",
    "# jan_2019"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>23bc03c69b69caf4bad18d4e5b770a8684da4630</td>\n",
       "      <td>2019-01-03 19:20:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>717</td>\n",
       "      <td>736</td>\n",
       "      <td>680</td>\n",
       "      <td>NC</td>\n",
       "      <td>Knightdale</td>\n",
       "      <td>5</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2f2d0ebdb47d3eb4ef152f71ad2ed372da29d1e6</td>\n",
       "      <td>2019-01-22 15:55:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>694</td>\n",
       "      <td>755</td>\n",
       "      <td>705</td>\n",
       "      <td>NC</td>\n",
       "      <td>Greenville</td>\n",
       "      <td>40</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>600ed36d2f3adce252cc6329c9d439c200bb2a9e</td>\n",
       "      <td>2019-01-08 16:10:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>727</td>\n",
       "      <td>724</td>\n",
       "      <td>724</td>\n",
       "      <td>NC</td>\n",
       "      <td>Chapel Hill</td>\n",
       "      <td>0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4ea6f03b24963f48a63aa54033f1c1c750b450ec</td>\n",
       "      <td>2019-01-21 13:15:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>686</td>\n",
       "      <td>737</td>\n",
       "      <td>687</td>\n",
       "      <td>NC</td>\n",
       "      <td>Greensboro</td>\n",
       "      <td>80</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>813e28bda0c816913ab8ad57425a58df804e8338</td>\n",
       "      <td>2019-01-03 17:35:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>680</td>\n",
       "      <td>685</td>\n",
       "      <td>685</td>\n",
       "      <td>NC</td>\n",
       "      <td>Asheville</td>\n",
       "      <td>20</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2281846</th>\n",
       "      <td>6c6bf2e0a6d2cdec4131e25ef14878cf885921ac</td>\n",
       "      <td>2019-01-03 11:30:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>673</td>\n",
       "      <td>760</td>\n",
       "      <td>670</td>\n",
       "      <td>NC</td>\n",
       "      <td>Greenville</td>\n",
       "      <td>50</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2281847</th>\n",
       "      <td>a053dd25784c094d0409dd78ea727427b9b37f01</td>\n",
       "      <td>2019-01-27 18:05:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>649</td>\n",
       "      <td>760</td>\n",
       "      <td>650</td>\n",
       "      <td>NC</td>\n",
       "      <td>Greensboro</td>\n",
       "      <td>0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2281848</th>\n",
       "      <td>448e8f94363f77a27105340db7ec707b99c7bb8c</td>\n",
       "      <td>2019-01-30 16:30:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>695</td>\n",
       "      <td>760</td>\n",
       "      <td>680</td>\n",
       "      <td>NC</td>\n",
       "      <td>Raleigh</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2281849</th>\n",
       "      <td>3efdb91d5f20c718e9d9038903e5cf9fcbf7ecca</td>\n",
       "      <td>2019-01-01 19:30:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>754</td>\n",
       "      <td>760</td>\n",
       "      <td>740</td>\n",
       "      <td>NC</td>\n",
       "      <td>Avon</td>\n",
       "      <td>18</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2281850</th>\n",
       "      <td>7ce5aa6160c998b98a93f8c490c117db0bf53c31</td>\n",
       "      <td>2019-01-10 12:25:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>683</td>\n",
       "      <td>760</td>\n",
       "      <td>670</td>\n",
       "      <td>NC</td>\n",
       "      <td>CHARLOTTE</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2183447 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                       Identifier                date_time  \\\n",
       "0        23bc03c69b69caf4bad18d4e5b770a8684da4630  2019-01-03 19:20:00 UTC   \n",
       "1        2f2d0ebdb47d3eb4ef152f71ad2ed372da29d1e6  2019-01-22 15:55:00 UTC   \n",
       "2        600ed36d2f3adce252cc6329c9d439c200bb2a9e  2019-01-08 16:10:00 UTC   \n",
       "3        4ea6f03b24963f48a63aa54033f1c1c750b450ec  2019-01-21 13:15:00 UTC   \n",
       "4        813e28bda0c816913ab8ad57425a58df804e8338  2019-01-03 17:35:00 UTC   \n",
       "...                                           ...                      ...   \n",
       "2281846  6c6bf2e0a6d2cdec4131e25ef14878cf885921ac  2019-01-03 11:30:00 UTC   \n",
       "2281847  a053dd25784c094d0409dd78ea727427b9b37f01  2019-01-27 18:05:00 UTC   \n",
       "2281848  448e8f94363f77a27105340db7ec707b99c7bb8c  2019-01-30 16:30:00 UTC   \n",
       "2281849  3efdb91d5f20c718e9d9038903e5cf9fcbf7ecca  2019-01-01 19:30:00 UTC   \n",
       "2281850  7ce5aa6160c998b98a93f8c490c117db0bf53c31  2019-01-10 12:25:00 UTC   \n",
       "\n",
       "        HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0           auto          hold               717                      736   \n",
       "1           auto          hold               694                      755   \n",
       "2           heat          hold               727                      724   \n",
       "3           auto          hold               686                      737   \n",
       "4           heat          hold               680                      685   \n",
       "...          ...           ...               ...                      ...   \n",
       "2281846     auto          hold               673                      760   \n",
       "2281847     auto          auto               649                      760   \n",
       "2281848     heat          auto               695                      760   \n",
       "2281849     auto          hold               754                      760   \n",
       "2281850     heat          auto               683                      760   \n",
       "\n",
       "         TemperatureExpectedHeat ProvinceState         City  \\\n",
       "0                            680            NC   Knightdale   \n",
       "1                            705            NC   Greenville   \n",
       "2                            724            NC  Chapel Hill   \n",
       "3                            687            NC   Greensboro   \n",
       "4                            685            NC    Asheville   \n",
       "...                          ...           ...          ...   \n",
       "2281846                      670            NC   Greenville   \n",
       "2281847                      650            NC   Greensboro   \n",
       "2281848                      680            NC      Raleigh   \n",
       "2281849                      740            NC         Avon   \n",
       "2281850                      670            NC    CHARLOTTE   \n",
       "\n",
       "         Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                          5              True         False            False   \n",
       "1                         40             False         False            False   \n",
       "2                          0              True         False             True   \n",
       "3                         80             False         False            False   \n",
       "4                         20              True         False             True   \n",
       "...                      ...               ...           ...              ...   \n",
       "2281846                   50             False         False            False   \n",
       "2281847                    0              True         False             True   \n",
       "2281848                    5             False         False            False   \n",
       "2281849                   18              True         False             True   \n",
       "2281850                    0             False         False            False   \n",
       "\n",
       "        Auxilliary_Heat_Fuel_Type  \n",
       "0                             Gas  \n",
       "1                             Gas  \n",
       "2                        Electric  \n",
       "3                             Gas  \n",
       "4                        Electric  \n",
       "...                           ...  \n",
       "2281846                       Gas  \n",
       "2281847                  Electric  \n",
       "2281848                       Gas  \n",
       "2281849                  Electric  \n",
       "2281850                       Gas  \n",
       "\n",
       "[2183447 rows x 14 columns]"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "jan_2019.drop(jan_2019[jan_2019['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "jan_2019.drop(jan_2019[jan_2019['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "jan_2019.drop(jan_2019[jan_2019['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "jan_2019.drop(jan_2019[jan_2019['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "jan_2019"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "jan_2019[\"Year\"] = \"2019\"\n",
    "jan_2019[\"Month\"] = \"Jan\"\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "jan_2019 = jan_2019.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "jan_2019_ave = jan_2019.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# jan_2019_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "jan_2019_ave.to_csv(\"data/day/NC/jan/jan_2019_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2020 January Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "jan_2020 = pd.read_csv(\"../data_large/NC-day/2020-jan-day-NC.csv\")\n",
    "\n",
    "# jan_2020"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>f27e2f012391952361c2a035eb81cd6e2eb926cd</td>\n",
       "      <td>2020-01-05 17:05:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>655</td>\n",
       "      <td>712</td>\n",
       "      <td>659</td>\n",
       "      <td>NC</td>\n",
       "      <td>Gastonia</td>\n",
       "      <td>35</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>9ea980db970426a400501e395079024f1dbce316</td>\n",
       "      <td>2020-01-29 18:20:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>661</td>\n",
       "      <td>665</td>\n",
       "      <td>665</td>\n",
       "      <td>NC</td>\n",
       "      <td>Winston salem</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>6ba7f07494cb93fd588038124c62e6e1977f315a</td>\n",
       "      <td>2020-01-12 19:50:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>723</td>\n",
       "      <td>707</td>\n",
       "      <td>707</td>\n",
       "      <td>NC</td>\n",
       "      <td>Troutman</td>\n",
       "      <td>10</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>a3c71856e280b64edc61bcd382ac3543b7754c83</td>\n",
       "      <td>2020-01-30 15:45:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>721</td>\n",
       "      <td>776</td>\n",
       "      <td>726</td>\n",
       "      <td>NC</td>\n",
       "      <td>Rolesville</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>9a6bf0321a3c79285b78c99fa0a47e5ef43ae464</td>\n",
       "      <td>2020-01-01 07:45:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>677</td>\n",
       "      <td>785</td>\n",
       "      <td>675</td>\n",
       "      <td>NC</td>\n",
       "      <td>Shallotte</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2548600</th>\n",
       "      <td>7316f5707304380513e0013533b15fe19be36ac6</td>\n",
       "      <td>2020-01-18 17:15:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>696</td>\n",
       "      <td>765</td>\n",
       "      <td>715</td>\n",
       "      <td>NC</td>\n",
       "      <td>Raleigh</td>\n",
       "      <td>0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2548601</th>\n",
       "      <td>6edc166b16b045dd12b3ac2419e7088ec4423b3f</td>\n",
       "      <td>2020-01-19 16:45:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>720</td>\n",
       "      <td>765</td>\n",
       "      <td>715</td>\n",
       "      <td>NC</td>\n",
       "      <td>Raleigh</td>\n",
       "      <td>16</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2548602</th>\n",
       "      <td>788369e6a13855797ace59f45d3b9471a1c6afc7</td>\n",
       "      <td>2020-01-19 19:25:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>760</td>\n",
       "      <td>765</td>\n",
       "      <td>765</td>\n",
       "      <td>NC</td>\n",
       "      <td>Hope Mills</td>\n",
       "      <td>0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2548603</th>\n",
       "      <td>7316f5707304380513e0013533b15fe19be36ac6</td>\n",
       "      <td>2020-01-03 17:45:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>697</td>\n",
       "      <td>765</td>\n",
       "      <td>715</td>\n",
       "      <td>NC</td>\n",
       "      <td>Raleigh</td>\n",
       "      <td>0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2548604</th>\n",
       "      <td>fe3b5890669606d65262cdb7eb5ff7f0d5ef3ae5</td>\n",
       "      <td>2020-01-02 13:20:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>729</td>\n",
       "      <td>765</td>\n",
       "      <td>675</td>\n",
       "      <td>NC</td>\n",
       "      <td>Durham</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2430448 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                       Identifier                date_time  \\\n",
       "0        f27e2f012391952361c2a035eb81cd6e2eb926cd  2020-01-05 17:05:00 UTC   \n",
       "1        9ea980db970426a400501e395079024f1dbce316  2020-01-29 18:20:00 UTC   \n",
       "3        6ba7f07494cb93fd588038124c62e6e1977f315a  2020-01-12 19:50:00 UTC   \n",
       "4        a3c71856e280b64edc61bcd382ac3543b7754c83  2020-01-30 15:45:00 UTC   \n",
       "8        9a6bf0321a3c79285b78c99fa0a47e5ef43ae464  2020-01-01 07:45:00 UTC   \n",
       "...                                           ...                      ...   \n",
       "2548600  7316f5707304380513e0013533b15fe19be36ac6  2020-01-18 17:15:00 UTC   \n",
       "2548601  6edc166b16b045dd12b3ac2419e7088ec4423b3f  2020-01-19 16:45:00 UTC   \n",
       "2548602  788369e6a13855797ace59f45d3b9471a1c6afc7  2020-01-19 19:25:00 UTC   \n",
       "2548603  7316f5707304380513e0013533b15fe19be36ac6  2020-01-03 17:45:00 UTC   \n",
       "2548604  fe3b5890669606d65262cdb7eb5ff7f0d5ef3ae5  2020-01-02 13:20:00 UTC   \n",
       "\n",
       "        HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0           auto          hold               655                      712   \n",
       "1           heat          hold               661                      665   \n",
       "3           heat          hold               723                      707   \n",
       "4           auto          hold               721                      776   \n",
       "8           auto          hold               677                      785   \n",
       "...          ...           ...               ...                      ...   \n",
       "2548600     auto          auto               696                      765   \n",
       "2548601     auto          hold               720                      765   \n",
       "2548602     heat          hold               760                      765   \n",
       "2548603     auto          auto               697                      765   \n",
       "2548604     auto          hold               729                      765   \n",
       "\n",
       "         TemperatureExpectedHeat ProvinceState           City  \\\n",
       "0                            659            NC       Gastonia   \n",
       "1                            665            NC  Winston salem   \n",
       "3                            707            NC       Troutman   \n",
       "4                            726            NC     Rolesville   \n",
       "8                            675            NC      Shallotte   \n",
       "...                          ...           ...            ...   \n",
       "2548600                      715            NC        Raleigh   \n",
       "2548601                      715            NC        Raleigh   \n",
       "2548602                      765            NC     Hope Mills   \n",
       "2548603                      715            NC        Raleigh   \n",
       "2548604                      675            NC         Durham   \n",
       "\n",
       "         Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                         35              True         False             True   \n",
       "1                          0             False         False            False   \n",
       "3                         10              True         False             True   \n",
       "4                          5             False         False            False   \n",
       "8                          0             False         False             True   \n",
       "...                      ...               ...           ...              ...   \n",
       "2548600                    0              True         False            False   \n",
       "2548601                   16             False         False            False   \n",
       "2548602                    0              True         False             True   \n",
       "2548603                    0              True         False            False   \n",
       "2548604                   10             False         False            False   \n",
       "\n",
       "        Auxilliary_Heat_Fuel_Type  \n",
       "0                        Electric  \n",
       "1                             Gas  \n",
       "3                        Electric  \n",
       "4                             Gas  \n",
       "8                        Electric  \n",
       "...                           ...  \n",
       "2548600                       Gas  \n",
       "2548601                       Gas  \n",
       "2548602                  Electric  \n",
       "2548603                       Gas  \n",
       "2548604                       Gas  \n",
       "\n",
       "[2430448 rows x 14 columns]"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "jan_2020.drop(jan_2020[jan_2020['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "jan_2020.drop(jan_2020[jan_2020['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "jan_2020.drop(jan_2020[jan_2020['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "jan_2020.drop(jan_2020[jan_2020['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "jan_2020"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "jan_2020[\"Year\"] = \"2020\"\n",
    "jan_2020[\"Month\"] = \"Jan\"\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "jan_2020 = jan_2020.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "jan_2020_ave = jan_2020.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# jan_2020_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "jan_2020_ave.to_csv(\"data/day/NC/jan/jan_2020_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2021 January Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "jan_2021 = pd.read_csv(\"../data_large/NC-day/2021-jan-day-NC.csv\")\n",
    "\n",
    "# jan_2021"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>a4b953edab7fd5bade6228c091c0ed7f596566e2</td>\n",
       "      <td>2021-01-11 17:00:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>698</td>\n",
       "      <td>753</td>\n",
       "      <td>703</td>\n",
       "      <td>NC</td>\n",
       "      <td>Clemmons</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1c0aba5d81a3923946fa516d3e333da72471aa19</td>\n",
       "      <td>2021-01-07 17:20:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>682</td>\n",
       "      <td>766</td>\n",
       "      <td>716</td>\n",
       "      <td>NC</td>\n",
       "      <td>Greensboro</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>a95fc8f324c6fca34372f86a2f7c31015351cf68</td>\n",
       "      <td>2021-01-01 07:35:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>721</td>\n",
       "      <td>726</td>\n",
       "      <td>726</td>\n",
       "      <td>NC</td>\n",
       "      <td>Cary</td>\n",
       "      <td>40</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>ee30fd2ea842c847d0286a17f479a9c14d6cc659</td>\n",
       "      <td>2021-01-29 14:15:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>611</td>\n",
       "      <td>620</td>\n",
       "      <td>620</td>\n",
       "      <td>NC</td>\n",
       "      <td>Raleigh</td>\n",
       "      <td>10</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4d3475306d51510440bb03298c5f2f3a2a2d9c88</td>\n",
       "      <td>2021-01-17 15:00:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>637</td>\n",
       "      <td>687</td>\n",
       "      <td>637</td>\n",
       "      <td>NC</td>\n",
       "      <td>Nags Head</td>\n",
       "      <td>15</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1608759</th>\n",
       "      <td>2fd293b3b28a292cf11dabd21d98da1942fbb10b</td>\n",
       "      <td>2021-01-11 17:20:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>650</td>\n",
       "      <td>765</td>\n",
       "      <td>695</td>\n",
       "      <td>NC</td>\n",
       "      <td>Hillsborough</td>\n",
       "      <td>70</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1608760</th>\n",
       "      <td>e99b5862e2007bf0eea22724d4ec755d93a68617</td>\n",
       "      <td>2021-01-03 19:20:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>700</td>\n",
       "      <td>765</td>\n",
       "      <td>705</td>\n",
       "      <td>NC</td>\n",
       "      <td>Raleigh</td>\n",
       "      <td>15</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1608761</th>\n",
       "      <td>2fd293b3b28a292cf11dabd21d98da1942fbb10b</td>\n",
       "      <td>2021-01-13 16:55:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>694</td>\n",
       "      <td>765</td>\n",
       "      <td>695</td>\n",
       "      <td>NC</td>\n",
       "      <td>Hillsborough</td>\n",
       "      <td>70</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1608762</th>\n",
       "      <td>2fd293b3b28a292cf11dabd21d98da1942fbb10b</td>\n",
       "      <td>2021-01-25 19:10:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>688</td>\n",
       "      <td>765</td>\n",
       "      <td>695</td>\n",
       "      <td>NC</td>\n",
       "      <td>Hillsborough</td>\n",
       "      <td>70</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1608763</th>\n",
       "      <td>6669546365d14453b4f84a3db45ae9b26a9b2c39</td>\n",
       "      <td>2021-01-18 15:55:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>705</td>\n",
       "      <td>765</td>\n",
       "      <td>715</td>\n",
       "      <td>NC</td>\n",
       "      <td>Hubert</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1548927 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                       Identifier                date_time  \\\n",
       "0        a4b953edab7fd5bade6228c091c0ed7f596566e2  2021-01-11 17:00:00 UTC   \n",
       "1        1c0aba5d81a3923946fa516d3e333da72471aa19  2021-01-07 17:20:00 UTC   \n",
       "2        a95fc8f324c6fca34372f86a2f7c31015351cf68  2021-01-01 07:35:00 UTC   \n",
       "3        ee30fd2ea842c847d0286a17f479a9c14d6cc659  2021-01-29 14:15:00 UTC   \n",
       "4        4d3475306d51510440bb03298c5f2f3a2a2d9c88  2021-01-17 15:00:00 UTC   \n",
       "...                                           ...                      ...   \n",
       "1608759  2fd293b3b28a292cf11dabd21d98da1942fbb10b  2021-01-11 17:20:00 UTC   \n",
       "1608760  e99b5862e2007bf0eea22724d4ec755d93a68617  2021-01-03 19:20:00 UTC   \n",
       "1608761  2fd293b3b28a292cf11dabd21d98da1942fbb10b  2021-01-13 16:55:00 UTC   \n",
       "1608762  2fd293b3b28a292cf11dabd21d98da1942fbb10b  2021-01-25 19:10:00 UTC   \n",
       "1608763  6669546365d14453b4f84a3db45ae9b26a9b2c39  2021-01-18 15:55:00 UTC   \n",
       "\n",
       "        HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0           auto          hold               698                      753   \n",
       "1           auto          hold               682                      766   \n",
       "2           heat          hold               721                      726   \n",
       "3           heat          hold               611                      620   \n",
       "4           auto          hold               637                      687   \n",
       "...          ...           ...               ...                      ...   \n",
       "1608759     auto          hold               650                      765   \n",
       "1608760     auto          hold               700                      765   \n",
       "1608761     auto          hold               694                      765   \n",
       "1608762     auto          hold               688                      765   \n",
       "1608763     auto          hold               705                      765   \n",
       "\n",
       "         TemperatureExpectedHeat ProvinceState          City  \\\n",
       "0                            703            NC      Clemmons   \n",
       "1                            716            NC    Greensboro   \n",
       "2                            726            NC          Cary   \n",
       "3                            620            NC       Raleigh   \n",
       "4                            637            NC     Nags Head   \n",
       "...                          ...           ...           ...   \n",
       "1608759                      695            NC  Hillsborough   \n",
       "1608760                      705            NC       Raleigh   \n",
       "1608761                      695            NC  Hillsborough   \n",
       "1608762                      695            NC  Hillsborough   \n",
       "1608763                      715            NC        Hubert   \n",
       "\n",
       "         Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                         10             False         False            False   \n",
       "1                          0             False         False            False   \n",
       "2                         40              True         False             True   \n",
       "3                         10              True         False            False   \n",
       "4                         15              True         False             True   \n",
       "...                      ...               ...           ...              ...   \n",
       "1608759                   70             False         False            False   \n",
       "1608760                   15             False         False            False   \n",
       "1608761                   70             False         False            False   \n",
       "1608762                   70             False         False            False   \n",
       "1608763                    5             False         False             True   \n",
       "\n",
       "        Auxilliary_Heat_Fuel_Type  \n",
       "0                             Gas  \n",
       "1                             Gas  \n",
       "2                        Electric  \n",
       "3                             Gas  \n",
       "4                        Electric  \n",
       "...                           ...  \n",
       "1608759                       Gas  \n",
       "1608760                       Gas  \n",
       "1608761                       Gas  \n",
       "1608762                       Gas  \n",
       "1608763                  Electric  \n",
       "\n",
       "[1548927 rows x 14 columns]"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "jan_2021.drop(jan_2021[jan_2021['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "jan_2021.drop(jan_2021[jan_2021['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "jan_2021.drop(jan_2021[jan_2021['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "jan_2021.drop(jan_2021[jan_2021['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "jan_2021"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "jan_2021[\"Year\"] = \"2021\"\n",
    "jan_2021[\"Month\"] = \"Jan\"\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "jan_2021 = jan_2021.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "jan_2021_ave = jan_2021.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# jan_2021_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "jan_2021_ave.to_csv(\"data/day/NC/jan/jan_2021_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "\n",
    "### Combine month CSV Files \n",
    "1. Read in files in folders for each state\n",
    "2. Export as combined CSV"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create variable for files in directory\n",
    "files = [f for f in os.listdir(\"data/day/NC/jan/\") if f.endswith(\".csv\")]\n",
    "\n",
    "# files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>Month</th>\n",
       "      <th>Year</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>City</th>\n",
       "      <th>Temperature_ctrl_ave</th>\n",
       "      <th>TemperatureExpectedCool_ave</th>\n",
       "      <th>TemperatureExpectedHeat_ave</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>00bbbdb5c3b6cb8b9e78191017ecdb97dad87e47</td>\n",
       "      <td>Jan</td>\n",
       "      <td>2017</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>Jacksonville</td>\n",
       "      <td>689.997824</td>\n",
       "      <td>762.501632</td>\n",
       "      <td>687.490751</td>\n",
       "      <td>5.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>00e5498cd188bdf09a9a1a902e6084b200cbbdef</td>\n",
       "      <td>Jan</td>\n",
       "      <td>2017</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>Wade</td>\n",
       "      <td>686.206897</td>\n",
       "      <td>751.310345</td>\n",
       "      <td>688.275862</td>\n",
       "      <td>55.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>013708850f6d20343f635dc3c0130399b66b0578</td>\n",
       "      <td>Jan</td>\n",
       "      <td>2017</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>Concord</td>\n",
       "      <td>690.990385</td>\n",
       "      <td>750.528846</td>\n",
       "      <td>690.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>016c1867b7cc7622e183e8d74de2b93ddd9449a3</td>\n",
       "      <td>Jan</td>\n",
       "      <td>2017</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>Dobson</td>\n",
       "      <td>720.290960</td>\n",
       "      <td>710.104520</td>\n",
       "      <td>709.816384</td>\n",
       "      <td>15.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0172751a7d4d785df2216952e1be26ff7861a414</td>\n",
       "      <td>Jan</td>\n",
       "      <td>2017</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>Greensboro</td>\n",
       "      <td>708.029650</td>\n",
       "      <td>707.734501</td>\n",
       "      <td>705.371968</td>\n",
       "      <td>65.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1872</th>\n",
       "      <td>fec2afca2bade938aaf514027a1d575b8a7084c8</td>\n",
       "      <td>Jan</td>\n",
       "      <td>2021</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>Raleigh</td>\n",
       "      <td>682.482185</td>\n",
       "      <td>736.783056</td>\n",
       "      <td>683.902613</td>\n",
       "      <td>28.0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1873</th>\n",
       "      <td>fefc39b30cc29ec1a80954d26e0550bf864455f7</td>\n",
       "      <td>Jan</td>\n",
       "      <td>2021</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>Raleigh</td>\n",
       "      <td>687.260119</td>\n",
       "      <td>695.516558</td>\n",
       "      <td>695.489952</td>\n",
       "      <td>9.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1874</th>\n",
       "      <td>ff14f7a9fbb7df32781e13d339c125fd7d6b8d4d</td>\n",
       "      <td>Jan</td>\n",
       "      <td>2021</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>Charlotte</td>\n",
       "      <td>710.594595</td>\n",
       "      <td>710.000000</td>\n",
       "      <td>710.000000</td>\n",
       "      <td>50.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1875</th>\n",
       "      <td>ff62cf17a37a458b09c22b255b1317cf85257110</td>\n",
       "      <td>Jan</td>\n",
       "      <td>2021</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>Wake Forest</td>\n",
       "      <td>705.074074</td>\n",
       "      <td>710.000000</td>\n",
       "      <td>710.000000</td>\n",
       "      <td>5.0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1876</th>\n",
       "      <td>ffc40b303849f7f5af77972db456dc63e2eebf34</td>\n",
       "      <td>Jan</td>\n",
       "      <td>2021</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>Charlotte</td>\n",
       "      <td>748.916667</td>\n",
       "      <td>748.114583</td>\n",
       "      <td>747.135417</td>\n",
       "      <td>17.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>12247 rows × 13 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                    Identifier Month  Year HvacMode  \\\n",
       "0     00bbbdb5c3b6cb8b9e78191017ecdb97dad87e47   Jan  2017     auto   \n",
       "1     00e5498cd188bdf09a9a1a902e6084b200cbbdef   Jan  2017     auto   \n",
       "2     013708850f6d20343f635dc3c0130399b66b0578   Jan  2017     auto   \n",
       "3     016c1867b7cc7622e183e8d74de2b93ddd9449a3   Jan  2017     heat   \n",
       "4     0172751a7d4d785df2216952e1be26ff7861a414   Jan  2017     heat   \n",
       "...                                        ...   ...   ...      ...   \n",
       "1872  fec2afca2bade938aaf514027a1d575b8a7084c8   Jan  2021     auto   \n",
       "1873  fefc39b30cc29ec1a80954d26e0550bf864455f7   Jan  2021     heat   \n",
       "1874  ff14f7a9fbb7df32781e13d339c125fd7d6b8d4d   Jan  2021     heat   \n",
       "1875  ff62cf17a37a458b09c22b255b1317cf85257110   Jan  2021     heat   \n",
       "1876  ffc40b303849f7f5af77972db456dc63e2eebf34   Jan  2021     heat   \n",
       "\n",
       "     CalendarEvent          City  Temperature_ctrl_ave  \\\n",
       "0             auto  Jacksonville            689.997824   \n",
       "1             hold          Wade            686.206897   \n",
       "2             hold       Concord            690.990385   \n",
       "3             auto        Dobson            720.290960   \n",
       "4             auto    Greensboro            708.029650   \n",
       "...            ...           ...                   ...   \n",
       "1872          hold       Raleigh            682.482185   \n",
       "1873          hold       Raleigh            687.260119   \n",
       "1874          hold     Charlotte            710.594595   \n",
       "1875          hold   Wake Forest            705.074074   \n",
       "1876          hold     Charlotte            748.916667   \n",
       "\n",
       "      TemperatureExpectedCool_ave  TemperatureExpectedHeat_ave  \\\n",
       "0                      762.501632                   687.490751   \n",
       "1                      751.310345                   688.275862   \n",
       "2                      750.528846                   690.000000   \n",
       "3                      710.104520                   709.816384   \n",
       "4                      707.734501                   705.371968   \n",
       "...                           ...                          ...   \n",
       "1872                   736.783056                   683.902613   \n",
       "1873                   695.516558                   695.489952   \n",
       "1874                   710.000000                   710.000000   \n",
       "1875                   710.000000                   710.000000   \n",
       "1876                   748.114583                   747.135417   \n",
       "\n",
       "      Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \n",
       "0                     5.0             False         False             True  \n",
       "1                    55.0             False         False            False  \n",
       "2                     0.0             False         False            False  \n",
       "3                    15.0             False         False             True  \n",
       "4                    65.0             False         False            False  \n",
       "...                   ...               ...           ...              ...  \n",
       "1872                 28.0              True         False            False  \n",
       "1873                  9.0             False         False            False  \n",
       "1874                 50.0             False         False             True  \n",
       "1875                  5.0              True         False             True  \n",
       "1876                 17.0             False         False            False  \n",
       "\n",
       "[12247 rows x 13 columns]"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# https://stackoverflow.com/questions/63886787/how-to-create-a-dataframe-from-multiple-csv-files\n",
    "# Initialize an empty dataframe\n",
    "NC_jan = pd.DataFrame()\n",
    "\n",
    "# Iterate through files and contents, then concatenate the data into the dataframe\n",
    "for file in files:\n",
    "    df = pd.read_csv(\"data/day/NC/jan/\" + file)\n",
    "    NC_jan = pd.concat([NC_jan, df])\n",
    "    \n",
    "NC_jan"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "NC_jan.to_csv(\"Scraper_Output/State_Month_Day/NC/NC_jan.csv\", header=True, index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "\n",
    "## February"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2017 February Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "feb_2017 = pd.read_csv(\"../data_large/NC-day/2017-feb-day-NC.csv\")\n",
    "\n",
    "# feb_2017"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>3a6c99e2c5bdb4afcff1db623695263b0728305c</td>\n",
       "      <td>2017-02-16 19:05:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>631</td>\n",
       "      <td>650</td>\n",
       "      <td>620</td>\n",
       "      <td>NC</td>\n",
       "      <td>Frisco</td>\n",
       "      <td>25</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2c83b1db2dc80b271e78b4755698b6b1070de3bf</td>\n",
       "      <td>2017-02-05 17:40:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>687</td>\n",
       "      <td>680</td>\n",
       "      <td>680</td>\n",
       "      <td>NC</td>\n",
       "      <td>Unionville</td>\n",
       "      <td>15</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>b71bf02769b40d4b9eebc157aede968ea529ad5e</td>\n",
       "      <td>2017-02-26 17:45:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>694</td>\n",
       "      <td>740</td>\n",
       "      <td>680</td>\n",
       "      <td>NC</td>\n",
       "      <td>winnabow</td>\n",
       "      <td>5</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>b0e6c0de512a2688c45bea23aa37e6bfc6aa700e</td>\n",
       "      <td>2017-02-05 19:00:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>682</td>\n",
       "      <td>680</td>\n",
       "      <td>680</td>\n",
       "      <td>NC</td>\n",
       "      <td>Greenville</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2d6ceac6219d1c8c0c0141f036a3d6867b15169a</td>\n",
       "      <td>2017-02-09 19:40:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>646</td>\n",
       "      <td>650</td>\n",
       "      <td>610</td>\n",
       "      <td>NC</td>\n",
       "      <td>Saluda</td>\n",
       "      <td>0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>424633</th>\n",
       "      <td>ffe96e4925748a4af930a70df17244664778168e</td>\n",
       "      <td>2017-02-04 13:25:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>733</td>\n",
       "      <td>780</td>\n",
       "      <td>730</td>\n",
       "      <td>NC</td>\n",
       "      <td>Cary</td>\n",
       "      <td>6</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>424634</th>\n",
       "      <td>ffe96e4925748a4af930a70df17244664778168e</td>\n",
       "      <td>2017-02-19 19:50:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>759</td>\n",
       "      <td>790</td>\n",
       "      <td>730</td>\n",
       "      <td>NC</td>\n",
       "      <td>Cary</td>\n",
       "      <td>6</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>424635</th>\n",
       "      <td>ffe96e4925748a4af930a70df17244664778168e</td>\n",
       "      <td>2017-02-20 13:00:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>728</td>\n",
       "      <td>790</td>\n",
       "      <td>730</td>\n",
       "      <td>NC</td>\n",
       "      <td>Cary</td>\n",
       "      <td>6</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>424636</th>\n",
       "      <td>ffe96e4925748a4af930a70df17244664778168e</td>\n",
       "      <td>2017-02-27 13:40:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>730</td>\n",
       "      <td>790</td>\n",
       "      <td>730</td>\n",
       "      <td>NC</td>\n",
       "      <td>Cary</td>\n",
       "      <td>6</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>424637</th>\n",
       "      <td>ffe96e4925748a4af930a70df17244664778168e</td>\n",
       "      <td>2017-02-20 15:25:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>729</td>\n",
       "      <td>790</td>\n",
       "      <td>730</td>\n",
       "      <td>NC</td>\n",
       "      <td>Cary</td>\n",
       "      <td>6</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>387987 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                      Identifier                date_time  \\\n",
       "0       3a6c99e2c5bdb4afcff1db623695263b0728305c  2017-02-16 19:05:00 UTC   \n",
       "1       2c83b1db2dc80b271e78b4755698b6b1070de3bf  2017-02-05 17:40:00 UTC   \n",
       "2       b71bf02769b40d4b9eebc157aede968ea529ad5e  2017-02-26 17:45:00 UTC   \n",
       "3       b0e6c0de512a2688c45bea23aa37e6bfc6aa700e  2017-02-05 19:00:00 UTC   \n",
       "4       2d6ceac6219d1c8c0c0141f036a3d6867b15169a  2017-02-09 19:40:00 UTC   \n",
       "...                                          ...                      ...   \n",
       "424633  ffe96e4925748a4af930a70df17244664778168e  2017-02-04 13:25:00 UTC   \n",
       "424634  ffe96e4925748a4af930a70df17244664778168e  2017-02-19 19:50:00 UTC   \n",
       "424635  ffe96e4925748a4af930a70df17244664778168e  2017-02-20 13:00:00 UTC   \n",
       "424636  ffe96e4925748a4af930a70df17244664778168e  2017-02-27 13:40:00 UTC   \n",
       "424637  ffe96e4925748a4af930a70df17244664778168e  2017-02-20 15:25:00 UTC   \n",
       "\n",
       "       HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0          heat          auto               631                      650   \n",
       "1          heat          hold               687                      680   \n",
       "2          auto          hold               694                      740   \n",
       "3          heat          auto               682                      680   \n",
       "4          heat          auto               646                      650   \n",
       "...         ...           ...               ...                      ...   \n",
       "424633     auto          hold               733                      780   \n",
       "424634     auto          hold               759                      790   \n",
       "424635     auto          hold               728                      790   \n",
       "424636     auto          hold               730                      790   \n",
       "424637     auto          hold               729                      790   \n",
       "\n",
       "        TemperatureExpectedHeat ProvinceState        City  \\\n",
       "0                           620            NC      Frisco   \n",
       "1                           680            NC  Unionville   \n",
       "2                           680            NC    winnabow   \n",
       "3                           680            NC  Greenville   \n",
       "4                           610            NC      Saluda   \n",
       "...                         ...           ...         ...   \n",
       "424633                      730            NC        Cary   \n",
       "424634                      730            NC        Cary   \n",
       "424635                      730            NC        Cary   \n",
       "424636                      730            NC        Cary   \n",
       "424637                      730            NC        Cary   \n",
       "\n",
       "        Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                        25             False         False            False   \n",
       "1                        15             False         False             True   \n",
       "2                         5              True         False             True   \n",
       "3                         5             False         False             True   \n",
       "4                         0              True         False             True   \n",
       "...                     ...               ...           ...              ...   \n",
       "424633                    6             False         False            False   \n",
       "424634                    6             False         False            False   \n",
       "424635                    6             False         False            False   \n",
       "424636                    6             False         False            False   \n",
       "424637                    6             False         False            False   \n",
       "\n",
       "       Auxilliary_Heat_Fuel_Type  \n",
       "0                            Gas  \n",
       "1                       Electric  \n",
       "2                       Electric  \n",
       "3                       Electric  \n",
       "4                       Electric  \n",
       "...                          ...  \n",
       "424633                       Gas  \n",
       "424634                       Gas  \n",
       "424635                       Gas  \n",
       "424636                       Gas  \n",
       "424637                       Gas  \n",
       "\n",
       "[387987 rows x 14 columns]"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "feb_2017.drop(feb_2017[feb_2017['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "feb_2017.drop(feb_2017[feb_2017['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "feb_2017.drop(feb_2017[feb_2017['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "feb_2017.drop(feb_2017[feb_2017['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "feb_2017"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "feb_2017[\"Year\"] = \"2017\"\n",
    "feb_2017[\"Month\"] = \"feb\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "feb_2017 = feb_2017.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "feb_2017_ave = feb_2017.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# feb_2017_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "feb_2017_ave.to_csv(\"data/day/NC/feb/feb_2017_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2018 February Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "feb_2018 = pd.read_csv(\"../data_large/NC-day/2018-feb-day-NC.csv\")\n",
    "\n",
    "# feb_2018"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1150388e8e2224e22c3c597f2623f05ad773d2bb</td>\n",
       "      <td>2018-02-19 12:10:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>660</td>\n",
       "      <td>660</td>\n",
       "      <td>660</td>\n",
       "      <td>NC</td>\n",
       "      <td>huntersville</td>\n",
       "      <td>15</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>17dca6affc300fcb0b3fd3496a7c4b29c4bcae67</td>\n",
       "      <td>2018-02-04 16:50:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>678</td>\n",
       "      <td>759</td>\n",
       "      <td>680</td>\n",
       "      <td>NC</td>\n",
       "      <td>Greenville</td>\n",
       "      <td>40</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>988cd1eeca74d66b7928569397bb9284e75cb86e</td>\n",
       "      <td>2018-02-17 10:45:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>713</td>\n",
       "      <td>730</td>\n",
       "      <td>680</td>\n",
       "      <td>NC</td>\n",
       "      <td>Salisbury</td>\n",
       "      <td>15</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>031ebcd358c12af7ca6302b5ccbcac9af3142e89</td>\n",
       "      <td>2018-02-28 17:30:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>698</td>\n",
       "      <td>700</td>\n",
       "      <td>700</td>\n",
       "      <td>NC</td>\n",
       "      <td>Fletcher</td>\n",
       "      <td>20</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>b71bf02769b40d4b9eebc157aede968ea529ad5e</td>\n",
       "      <td>2018-02-11 18:45:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>701</td>\n",
       "      <td>700</td>\n",
       "      <td>650</td>\n",
       "      <td>NC</td>\n",
       "      <td>winnabow</td>\n",
       "      <td>5</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1214049</th>\n",
       "      <td>60c2af9302a78bb5398ffdc36eb0d645e8357070</td>\n",
       "      <td>2018-02-03 16:05:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>675</td>\n",
       "      <td>670</td>\n",
       "      <td>670</td>\n",
       "      <td>NC</td>\n",
       "      <td>Ocean Isle Beach</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1214050</th>\n",
       "      <td>43999a6fc6c24361d81931a316cf5a96ce14b2b5</td>\n",
       "      <td>2018-02-10 15:00:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>656</td>\n",
       "      <td>650</td>\n",
       "      <td>650</td>\n",
       "      <td>NC</td>\n",
       "      <td>Ocean Isle Beach</td>\n",
       "      <td>0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1214051</th>\n",
       "      <td>60c2af9302a78bb5398ffdc36eb0d645e8357070</td>\n",
       "      <td>2018-02-03 18:30:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>672</td>\n",
       "      <td>670</td>\n",
       "      <td>670</td>\n",
       "      <td>NC</td>\n",
       "      <td>Ocean Isle Beach</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1214052</th>\n",
       "      <td>4f60b83cc51d90e13839e112d06909cbb5ba4273</td>\n",
       "      <td>2018-02-28 12:50:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>677</td>\n",
       "      <td>650</td>\n",
       "      <td>650</td>\n",
       "      <td>NC</td>\n",
       "      <td>Ocean Isle Beach</td>\n",
       "      <td>15</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1214053</th>\n",
       "      <td>4a26911ab35b12b6da6a3307fce27898497a221d</td>\n",
       "      <td>2018-02-10 16:00:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>680</td>\n",
       "      <td>680</td>\n",
       "      <td>680</td>\n",
       "      <td>NC</td>\n",
       "      <td>Ocean Isle Beach</td>\n",
       "      <td>15</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1150430 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                       Identifier                date_time  \\\n",
       "0        1150388e8e2224e22c3c597f2623f05ad773d2bb  2018-02-19 12:10:00 UTC   \n",
       "1        17dca6affc300fcb0b3fd3496a7c4b29c4bcae67  2018-02-04 16:50:00 UTC   \n",
       "2        988cd1eeca74d66b7928569397bb9284e75cb86e  2018-02-17 10:45:00 UTC   \n",
       "4        031ebcd358c12af7ca6302b5ccbcac9af3142e89  2018-02-28 17:30:00 UTC   \n",
       "5        b71bf02769b40d4b9eebc157aede968ea529ad5e  2018-02-11 18:45:00 UTC   \n",
       "...                                           ...                      ...   \n",
       "1214049  60c2af9302a78bb5398ffdc36eb0d645e8357070  2018-02-03 16:05:00 UTC   \n",
       "1214050  43999a6fc6c24361d81931a316cf5a96ce14b2b5  2018-02-10 15:00:00 UTC   \n",
       "1214051  60c2af9302a78bb5398ffdc36eb0d645e8357070  2018-02-03 18:30:00 UTC   \n",
       "1214052  4f60b83cc51d90e13839e112d06909cbb5ba4273  2018-02-28 12:50:00 UTC   \n",
       "1214053  4a26911ab35b12b6da6a3307fce27898497a221d  2018-02-10 16:00:00 UTC   \n",
       "\n",
       "        HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0           heat          hold               660                      660   \n",
       "1           auto          hold               678                      759   \n",
       "2           auto          auto               713                      730   \n",
       "4           heat          hold               698                      700   \n",
       "5           auto          hold               701                      700   \n",
       "...          ...           ...               ...                      ...   \n",
       "1214049     heat          hold               675                      670   \n",
       "1214050     heat          auto               656                      650   \n",
       "1214051     heat          hold               672                      670   \n",
       "1214052     heat          auto               677                      650   \n",
       "1214053     heat          hold               680                      680   \n",
       "\n",
       "         TemperatureExpectedHeat ProvinceState              City  \\\n",
       "0                            660            NC      huntersville   \n",
       "1                            680            NC        Greenville   \n",
       "2                            680            NC         Salisbury   \n",
       "4                            700            NC          Fletcher   \n",
       "5                            650            NC          winnabow   \n",
       "...                          ...           ...               ...   \n",
       "1214049                      670            NC  Ocean Isle Beach   \n",
       "1214050                      650            NC  Ocean Isle Beach   \n",
       "1214051                      670            NC  Ocean Isle Beach   \n",
       "1214052                      650            NC  Ocean Isle Beach   \n",
       "1214053                      680            NC  Ocean Isle Beach   \n",
       "\n",
       "         Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                         15             False         False            False   \n",
       "1                         40             False         False            False   \n",
       "2                         15             False         False            False   \n",
       "4                         20              True         False             True   \n",
       "5                          5              True         False             True   \n",
       "...                      ...               ...           ...              ...   \n",
       "1214049                   10             False         False             True   \n",
       "1214050                    0              True         False             True   \n",
       "1214051                   10             False         False             True   \n",
       "1214052                   15              True         False             True   \n",
       "1214053                   15             False         False            False   \n",
       "\n",
       "        Auxilliary_Heat_Fuel_Type  \n",
       "0                             Gas  \n",
       "1                             Gas  \n",
       "2                             Gas  \n",
       "4                        Electric  \n",
       "5                        Electric  \n",
       "...                           ...  \n",
       "1214049                  Electric  \n",
       "1214050                  Electric  \n",
       "1214051                  Electric  \n",
       "1214052                  Electric  \n",
       "1214053                       Gas  \n",
       "\n",
       "[1150430 rows x 14 columns]"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "feb_2018.drop(feb_2018[feb_2018['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "feb_2018.drop(feb_2018[feb_2018['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "feb_2018.drop(feb_2018[feb_2018['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "feb_2018.drop(feb_2018[feb_2018['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "feb_2018"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "feb_2018[\"Year\"] = \"2018\"\n",
    "feb_2018[\"Month\"] = \"feb\"\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "feb_2018 = feb_2018.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "feb_2018_ave = feb_2018.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# feb_2018_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "feb_2018_ave.to_csv(\"data/day/NC/feb/feb_2018_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2019 February Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "feb_2019 = pd.read_csv(\"../data_large/NC-day/2019-feb-day-NC.csv\")\n",
    "\n",
    "# feb_2019"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>a33a41c4d4916487c5ce1b021c8dbb323a5cd65e</td>\n",
       "      <td>2019-02-02 12:30:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>668</td>\n",
       "      <td>672</td>\n",
       "      <td>672</td>\n",
       "      <td>NC</td>\n",
       "      <td>Huntersville</td>\n",
       "      <td>20</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>82b236372e7787e7397e32cd1c5940c76e83db4b</td>\n",
       "      <td>2019-02-18 17:35:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>701</td>\n",
       "      <td>745</td>\n",
       "      <td>695</td>\n",
       "      <td>NC</td>\n",
       "      <td>Durham</td>\n",
       "      <td>0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>40379567a6ded963ebeef5eaf6492ecbe6638fda</td>\n",
       "      <td>2019-02-13 15:50:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>683</td>\n",
       "      <td>712</td>\n",
       "      <td>690</td>\n",
       "      <td>NC</td>\n",
       "      <td>Davidson</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>2d6ceac6219d1c8c0c0141f036a3d6867b15169a</td>\n",
       "      <td>2019-02-17 12:00:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>739</td>\n",
       "      <td>737</td>\n",
       "      <td>737</td>\n",
       "      <td>NC</td>\n",
       "      <td>Saluda</td>\n",
       "      <td>0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>f78c4dccacad53365b6c7f5a0e3c506ad85f0d1d</td>\n",
       "      <td>2019-02-24 18:20:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>714</td>\n",
       "      <td>749</td>\n",
       "      <td>699</td>\n",
       "      <td>NC</td>\n",
       "      <td>Hickory</td>\n",
       "      <td>30</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1549376</th>\n",
       "      <td>04a9eef44531edead29bceed262f48fe970ba604</td>\n",
       "      <td>2019-02-14 19:00:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>705</td>\n",
       "      <td>760</td>\n",
       "      <td>710</td>\n",
       "      <td>NC</td>\n",
       "      <td>Clemmons</td>\n",
       "      <td>25</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1549377</th>\n",
       "      <td>c9365a10564f6bc105a972ad99fd71faa9e8bcbf</td>\n",
       "      <td>2019-02-21 12:30:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>684</td>\n",
       "      <td>760</td>\n",
       "      <td>690</td>\n",
       "      <td>NC</td>\n",
       "      <td>Denver</td>\n",
       "      <td>15</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1549379</th>\n",
       "      <td>3efdb91d5f20c718e9d9038903e5cf9fcbf7ecca</td>\n",
       "      <td>2019-02-10 11:40:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>739</td>\n",
       "      <td>760</td>\n",
       "      <td>740</td>\n",
       "      <td>NC</td>\n",
       "      <td>Avon</td>\n",
       "      <td>18</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1549380</th>\n",
       "      <td>53b726d36d16370680f826c56e6ced913b2e38f2</td>\n",
       "      <td>2019-02-06 17:25:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>707</td>\n",
       "      <td>760</td>\n",
       "      <td>710</td>\n",
       "      <td>NC</td>\n",
       "      <td>Hickory</td>\n",
       "      <td>30</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1549381</th>\n",
       "      <td>4701a84c767ca3e57720053a386adfb0116f08e5</td>\n",
       "      <td>2019-02-15 17:10:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>664</td>\n",
       "      <td>760</td>\n",
       "      <td>640</td>\n",
       "      <td>NC</td>\n",
       "      <td>High Point</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1467163 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                       Identifier                date_time  \\\n",
       "0        a33a41c4d4916487c5ce1b021c8dbb323a5cd65e  2019-02-02 12:30:00 UTC   \n",
       "1        82b236372e7787e7397e32cd1c5940c76e83db4b  2019-02-18 17:35:00 UTC   \n",
       "5        40379567a6ded963ebeef5eaf6492ecbe6638fda  2019-02-13 15:50:00 UTC   \n",
       "6        2d6ceac6219d1c8c0c0141f036a3d6867b15169a  2019-02-17 12:00:00 UTC   \n",
       "7        f78c4dccacad53365b6c7f5a0e3c506ad85f0d1d  2019-02-24 18:20:00 UTC   \n",
       "...                                           ...                      ...   \n",
       "1549376  04a9eef44531edead29bceed262f48fe970ba604  2019-02-14 19:00:00 UTC   \n",
       "1549377  c9365a10564f6bc105a972ad99fd71faa9e8bcbf  2019-02-21 12:30:00 UTC   \n",
       "1549379  3efdb91d5f20c718e9d9038903e5cf9fcbf7ecca  2019-02-10 11:40:00 UTC   \n",
       "1549380  53b726d36d16370680f826c56e6ced913b2e38f2  2019-02-06 17:25:00 UTC   \n",
       "1549381  4701a84c767ca3e57720053a386adfb0116f08e5  2019-02-15 17:10:00 UTC   \n",
       "\n",
       "        HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0           heat          hold               668                      672   \n",
       "1           auto          hold               701                      745   \n",
       "5           heat          auto               683                      712   \n",
       "6           heat          hold               739                      737   \n",
       "7           auto          hold               714                      749   \n",
       "...          ...           ...               ...                      ...   \n",
       "1549376     auto          hold               705                      760   \n",
       "1549377     auto          hold               684                      760   \n",
       "1549379     auto          hold               739                      760   \n",
       "1549380     auto          hold               707                      760   \n",
       "1549381     heat          auto               664                      760   \n",
       "\n",
       "         TemperatureExpectedHeat ProvinceState          City  \\\n",
       "0                            672            NC  Huntersville   \n",
       "1                            695            NC        Durham   \n",
       "5                            690            NC      Davidson   \n",
       "6                            737            NC        Saluda   \n",
       "7                            699            NC       Hickory   \n",
       "...                          ...           ...           ...   \n",
       "1549376                      710            NC      Clemmons   \n",
       "1549377                      690            NC        Denver   \n",
       "1549379                      740            NC          Avon   \n",
       "1549380                      710            NC       Hickory   \n",
       "1549381                      640            NC    High Point   \n",
       "\n",
       "         Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                         20              True         False            False   \n",
       "1                          0              True         False             True   \n",
       "5                          0             False         False            False   \n",
       "6                          0              True         False             True   \n",
       "7                         30             False         False             True   \n",
       "...                      ...               ...           ...              ...   \n",
       "1549376                   25             False         False            False   \n",
       "1549377                   15              True         False             True   \n",
       "1549379                   18              True         False             True   \n",
       "1549380                   30             False         False            False   \n",
       "1549381                    0             False         False            False   \n",
       "\n",
       "        Auxilliary_Heat_Fuel_Type  \n",
       "0                             Gas  \n",
       "1                        Electric  \n",
       "5                             Gas  \n",
       "6                        Electric  \n",
       "7                        Electric  \n",
       "...                           ...  \n",
       "1549376                       Gas  \n",
       "1549377                  Electric  \n",
       "1549379                  Electric  \n",
       "1549380                       Gas  \n",
       "1549381                       Gas  \n",
       "\n",
       "[1467163 rows x 14 columns]"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "feb_2019.drop(feb_2019[feb_2019['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "feb_2019.drop(feb_2019[feb_2019['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "feb_2019.drop(feb_2019[feb_2019['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "feb_2019.drop(feb_2019[feb_2019['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "feb_2019"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "feb_2019[\"Year\"] = \"2019\"\n",
    "feb_2019[\"Month\"] = \"feb\"\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "feb_2019 = feb_2019.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "feb_2019_ave = feb_2019.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# feb_2019_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "feb_2019_ave.to_csv(\"data/day/NC/feb/feb_2019_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2020 February Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "feb_2020 = pd.read_csv(\"../data_large/NC-day/2020-feb-day-NC.csv\")\n",
    "\n",
    "# feb_2020"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>92f07dddb78aaf112931f694759d81aa249d5a82</td>\n",
       "      <td>2020-02-22 16:35:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>690</td>\n",
       "      <td>689</td>\n",
       "      <td>689</td>\n",
       "      <td>NC</td>\n",
       "      <td>Wake Forest</td>\n",
       "      <td>5</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>e6fd4a1081c3c2e05147fb1f5ce2e8c7d9ac366d</td>\n",
       "      <td>2020-02-24 13:30:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>683</td>\n",
       "      <td>756</td>\n",
       "      <td>686</td>\n",
       "      <td>NC</td>\n",
       "      <td>Raleigh</td>\n",
       "      <td>40</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>87fb9c0cbfadf8306e8a0fbdd3f6dfeeb1d7d895</td>\n",
       "      <td>2020-02-28 10:30:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>689</td>\n",
       "      <td>695</td>\n",
       "      <td>695</td>\n",
       "      <td>NC</td>\n",
       "      <td>Waxhaw</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>87fb9c0cbfadf8306e8a0fbdd3f6dfeeb1d7d895</td>\n",
       "      <td>2020-02-21 14:40:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>675</td>\n",
       "      <td>675</td>\n",
       "      <td>675</td>\n",
       "      <td>NC</td>\n",
       "      <td>Waxhaw</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>080a06a3d22de098f71ee307c9fd975721b63f3e</td>\n",
       "      <td>2020-02-05 14:35:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>736</td>\n",
       "      <td>741</td>\n",
       "      <td>730</td>\n",
       "      <td>NC</td>\n",
       "      <td>Charlotte</td>\n",
       "      <td>50</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2322267</th>\n",
       "      <td>ddb70c031e35407581ccb7567a7afdc06e991345</td>\n",
       "      <td>2020-02-11 13:55:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>716</td>\n",
       "      <td>765</td>\n",
       "      <td>715</td>\n",
       "      <td>NC</td>\n",
       "      <td>St James</td>\n",
       "      <td>5</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2322268</th>\n",
       "      <td>55a761d0276ab1bd1cb1de02fd17b27030bbbf6a</td>\n",
       "      <td>2020-02-24 17:10:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>717</td>\n",
       "      <td>765</td>\n",
       "      <td>715</td>\n",
       "      <td>NC</td>\n",
       "      <td>Chapel Hill</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2322269</th>\n",
       "      <td>55a761d0276ab1bd1cb1de02fd17b27030bbbf6a</td>\n",
       "      <td>2020-02-11 15:15:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>715</td>\n",
       "      <td>765</td>\n",
       "      <td>715</td>\n",
       "      <td>NC</td>\n",
       "      <td>Chapel Hill</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2322270</th>\n",
       "      <td>8b647121b63936d90962aa84f677b96fecab13e2</td>\n",
       "      <td>2020-02-02 12:05:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>760</td>\n",
       "      <td>765</td>\n",
       "      <td>765</td>\n",
       "      <td>NC</td>\n",
       "      <td>Raleigh</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2322271</th>\n",
       "      <td>6edc166b16b045dd12b3ac2419e7088ec4423b3f</td>\n",
       "      <td>2020-02-04 13:05:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>703</td>\n",
       "      <td>765</td>\n",
       "      <td>705</td>\n",
       "      <td>NC</td>\n",
       "      <td>Raleigh</td>\n",
       "      <td>16</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2224847 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                       Identifier                date_time  \\\n",
       "0        92f07dddb78aaf112931f694759d81aa249d5a82  2020-02-22 16:35:00 UTC   \n",
       "1        e6fd4a1081c3c2e05147fb1f5ce2e8c7d9ac366d  2020-02-24 13:30:00 UTC   \n",
       "3        87fb9c0cbfadf8306e8a0fbdd3f6dfeeb1d7d895  2020-02-28 10:30:00 UTC   \n",
       "4        87fb9c0cbfadf8306e8a0fbdd3f6dfeeb1d7d895  2020-02-21 14:40:00 UTC   \n",
       "6        080a06a3d22de098f71ee307c9fd975721b63f3e  2020-02-05 14:35:00 UTC   \n",
       "...                                           ...                      ...   \n",
       "2322267  ddb70c031e35407581ccb7567a7afdc06e991345  2020-02-11 13:55:00 UTC   \n",
       "2322268  55a761d0276ab1bd1cb1de02fd17b27030bbbf6a  2020-02-24 17:10:00 UTC   \n",
       "2322269  55a761d0276ab1bd1cb1de02fd17b27030bbbf6a  2020-02-11 15:15:00 UTC   \n",
       "2322270  8b647121b63936d90962aa84f677b96fecab13e2  2020-02-02 12:05:00 UTC   \n",
       "2322271  6edc166b16b045dd12b3ac2419e7088ec4423b3f  2020-02-04 13:05:00 UTC   \n",
       "\n",
       "        HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0           heat          hold               690                      689   \n",
       "1           auto          hold               683                      756   \n",
       "3           heat          hold               689                      695   \n",
       "4           heat          hold               675                      675   \n",
       "6           heat          hold               736                      741   \n",
       "...          ...           ...               ...                      ...   \n",
       "2322267     auto          hold               716                      765   \n",
       "2322268     auto          hold               717                      765   \n",
       "2322269     auto          hold               715                      765   \n",
       "2322270     heat          hold               760                      765   \n",
       "2322271     auto          hold               703                      765   \n",
       "\n",
       "         TemperatureExpectedHeat ProvinceState         City  \\\n",
       "0                            689            NC  Wake Forest   \n",
       "1                            686            NC      Raleigh   \n",
       "3                            695            NC       Waxhaw   \n",
       "4                            675            NC       Waxhaw   \n",
       "6                            730            NC    Charlotte   \n",
       "...                          ...           ...          ...   \n",
       "2322267                      715            NC     St James   \n",
       "2322268                      715            NC  Chapel Hill   \n",
       "2322269                      715            NC  Chapel Hill   \n",
       "2322270                      765            NC      Raleigh   \n",
       "2322271                      705            NC      Raleigh   \n",
       "\n",
       "         Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                          5              True         False            False   \n",
       "1                         40              True         False             True   \n",
       "3                          0             False         False            False   \n",
       "4                          0             False         False            False   \n",
       "6                         50             False         False            False   \n",
       "...                      ...               ...           ...              ...   \n",
       "2322267                    5              True         False             True   \n",
       "2322268                    5             False         False            False   \n",
       "2322269                    5             False         False            False   \n",
       "2322270                    0             False         False            False   \n",
       "2322271                   16             False         False            False   \n",
       "\n",
       "        Auxilliary_Heat_Fuel_Type  \n",
       "0                             Gas  \n",
       "1                        Electric  \n",
       "3                             Gas  \n",
       "4                             Gas  \n",
       "6                             Gas  \n",
       "...                           ...  \n",
       "2322267                  Electric  \n",
       "2322268                       Gas  \n",
       "2322269                       Gas  \n",
       "2322270                       Gas  \n",
       "2322271                       Gas  \n",
       "\n",
       "[2224847 rows x 14 columns]"
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "feb_2020.drop(feb_2020[feb_2020['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "feb_2020.drop(feb_2020[feb_2020['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "feb_2020.drop(feb_2020[feb_2020['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "feb_2020.drop(feb_2020[feb_2020['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "feb_2020"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "feb_2020[\"Year\"] = \"2020\"\n",
    "feb_2020[\"Month\"] = \"feb\"\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "feb_2020 = feb_2020.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "feb_2020_ave = feb_2020.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# feb_2020_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "feb_2020_ave.to_csv(\"data/day/NC/feb/feb_2020_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2021 February Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "feb_2021 = pd.read_csv(\"../data_large/NC-day/2021-feb-day-NC.csv\")\n",
    "\n",
    "# feb_2021"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>a6b00ec27c2fc8f156c0aa32dc77ee4be0c34af3</td>\n",
       "      <td>2021-02-25 13:15:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>671</td>\n",
       "      <td>743</td>\n",
       "      <td>671</td>\n",
       "      <td>NC</td>\n",
       "      <td>Hickory</td>\n",
       "      <td>40</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>4d0bce9a61343455f43688bf8edd19cb50891d65</td>\n",
       "      <td>2021-02-11 18:25:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>724</td>\n",
       "      <td>779</td>\n",
       "      <td>729</td>\n",
       "      <td>NC</td>\n",
       "      <td>Fuquay Varina</td>\n",
       "      <td>9</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>d6266abc21b086809b80bb5c3a1ea1f63d45ec82</td>\n",
       "      <td>2021-02-20 15:05:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>703</td>\n",
       "      <td>756</td>\n",
       "      <td>706</td>\n",
       "      <td>NC</td>\n",
       "      <td>Charlotte</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>60926d4d8ab741b8db2e203ed821449e3d9add02</td>\n",
       "      <td>2021-02-02 19:10:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>700</td>\n",
       "      <td>698</td>\n",
       "      <td>698</td>\n",
       "      <td>NC</td>\n",
       "      <td>Monroe</td>\n",
       "      <td>15</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>55afdd934915ce4a0b67c30403ebec9ce4ee01e5</td>\n",
       "      <td>2021-02-20 19:25:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>687</td>\n",
       "      <td>840</td>\n",
       "      <td>690</td>\n",
       "      <td>NC</td>\n",
       "      <td>Chapel Hill</td>\n",
       "      <td>17</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1352818</th>\n",
       "      <td>86f6cc63ed6dc3eaca3dfc650b9feec4847047cf</td>\n",
       "      <td>2021-02-20 08:45:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>747</td>\n",
       "      <td>765</td>\n",
       "      <td>765</td>\n",
       "      <td>NC</td>\n",
       "      <td>Carrboro</td>\n",
       "      <td>30</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1352819</th>\n",
       "      <td>85a31abec17dbcb4a3170146f5b3fde6932470b9</td>\n",
       "      <td>2021-02-13 12:45:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>698</td>\n",
       "      <td>765</td>\n",
       "      <td>715</td>\n",
       "      <td>NC</td>\n",
       "      <td>Rosman</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1352820</th>\n",
       "      <td>86f6cc63ed6dc3eaca3dfc650b9feec4847047cf</td>\n",
       "      <td>2021-02-27 10:35:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>762</td>\n",
       "      <td>765</td>\n",
       "      <td>765</td>\n",
       "      <td>NC</td>\n",
       "      <td>Carrboro</td>\n",
       "      <td>30</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1352821</th>\n",
       "      <td>06b4455e8024315effd9704360795475b0314ec7</td>\n",
       "      <td>2021-02-13 11:40:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>669</td>\n",
       "      <td>765</td>\n",
       "      <td>665</td>\n",
       "      <td>NC</td>\n",
       "      <td>Cary</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1352822</th>\n",
       "      <td>e9a49b5133d10a502464383769dfc11d26b4999b</td>\n",
       "      <td>2021-02-02 12:50:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>715</td>\n",
       "      <td>765</td>\n",
       "      <td>715</td>\n",
       "      <td>NC</td>\n",
       "      <td>Matthews</td>\n",
       "      <td>15</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1291054 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                       Identifier                date_time  \\\n",
       "0        a6b00ec27c2fc8f156c0aa32dc77ee4be0c34af3  2021-02-25 13:15:00 UTC   \n",
       "2        4d0bce9a61343455f43688bf8edd19cb50891d65  2021-02-11 18:25:00 UTC   \n",
       "3        d6266abc21b086809b80bb5c3a1ea1f63d45ec82  2021-02-20 15:05:00 UTC   \n",
       "5        60926d4d8ab741b8db2e203ed821449e3d9add02  2021-02-02 19:10:00 UTC   \n",
       "6        55afdd934915ce4a0b67c30403ebec9ce4ee01e5  2021-02-20 19:25:00 UTC   \n",
       "...                                           ...                      ...   \n",
       "1352818  86f6cc63ed6dc3eaca3dfc650b9feec4847047cf  2021-02-20 08:45:00 UTC   \n",
       "1352819  85a31abec17dbcb4a3170146f5b3fde6932470b9  2021-02-13 12:45:00 UTC   \n",
       "1352820  86f6cc63ed6dc3eaca3dfc650b9feec4847047cf  2021-02-27 10:35:00 UTC   \n",
       "1352821  06b4455e8024315effd9704360795475b0314ec7  2021-02-13 11:40:00 UTC   \n",
       "1352822  e9a49b5133d10a502464383769dfc11d26b4999b  2021-02-02 12:50:00 UTC   \n",
       "\n",
       "        HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0           heat          hold               671                      743   \n",
       "2           auto          hold               724                      779   \n",
       "3           auto          hold               703                      756   \n",
       "5           heat          hold               700                      698   \n",
       "6           auto          hold               687                      840   \n",
       "...          ...           ...               ...                      ...   \n",
       "1352818     heat          hold               747                      765   \n",
       "1352819     auto          hold               698                      765   \n",
       "1352820     heat          hold               762                      765   \n",
       "1352821     auto          hold               669                      765   \n",
       "1352822     auto          hold               715                      765   \n",
       "\n",
       "         TemperatureExpectedHeat ProvinceState           City  \\\n",
       "0                            671            NC        Hickory   \n",
       "2                            729            NC  Fuquay Varina   \n",
       "3                            706            NC      Charlotte   \n",
       "5                            698            NC         Monroe   \n",
       "6                            690            NC    Chapel Hill   \n",
       "...                          ...           ...            ...   \n",
       "1352818                      765            NC       Carrboro   \n",
       "1352819                      715            NC         Rosman   \n",
       "1352820                      765            NC       Carrboro   \n",
       "1352821                      665            NC           Cary   \n",
       "1352822                      715            NC       Matthews   \n",
       "\n",
       "         Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                         40              True         False             True   \n",
       "2                          9              True         False             True   \n",
       "3                          0             False         False            False   \n",
       "5                         15             False         False            False   \n",
       "6                         17             False         False            False   \n",
       "...                      ...               ...           ...              ...   \n",
       "1352818                   30              True         False             True   \n",
       "1352819                    0             False         False             True   \n",
       "1352820                   30              True         False             True   \n",
       "1352821                    5             False         False             True   \n",
       "1352822                   15             False         False            False   \n",
       "\n",
       "        Auxilliary_Heat_Fuel_Type  \n",
       "0                        Electric  \n",
       "2                        Electric  \n",
       "3                             Gas  \n",
       "5                             Gas  \n",
       "6                             Gas  \n",
       "...                           ...  \n",
       "1352818                  Electric  \n",
       "1352819                  Electric  \n",
       "1352820                  Electric  \n",
       "1352821                  Electric  \n",
       "1352822                       Gas  \n",
       "\n",
       "[1291054 rows x 14 columns]"
      ]
     },
     "execution_count": 60,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "feb_2021.drop(feb_2021[feb_2021['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "feb_2021.drop(feb_2021[feb_2021['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "feb_2021.drop(feb_2021[feb_2021['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "feb_2021.drop(feb_2021[feb_2021['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "feb_2021"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "feb_2021[\"Year\"] = \"2021\"\n",
    "feb_2021[\"Month\"] = \"feb\"\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "feb_2021 = feb_2021.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "feb_2021_ave = feb_2021.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# feb_2021_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "feb_2021_ave.to_csv(\"data/day/NC/feb/feb_2021_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "\n",
    "### Combine month CSV Files \n",
    "1. Read in files in folders for each state\n",
    "2. Export as combined CSV"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create variable for files in directory\n",
    "files = [f for f in os.listdir(\"data/day/NC/feb/\") if f.endswith(\".csv\")]\n",
    "\n",
    "# files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>Month</th>\n",
       "      <th>Year</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>City</th>\n",
       "      <th>Temperature_ctrl_ave</th>\n",
       "      <th>TemperatureExpectedCool_ave</th>\n",
       "      <th>TemperatureExpectedHeat_ave</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>00bbbdb5c3b6cb8b9e78191017ecdb97dad87e47</td>\n",
       "      <td>feb</td>\n",
       "      <td>2017</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>Jacksonville</td>\n",
       "      <td>704.408854</td>\n",
       "      <td>766.585938</td>\n",
       "      <td>683.395833</td>\n",
       "      <td>5.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>00e5498cd188bdf09a9a1a902e6084b200cbbdef</td>\n",
       "      <td>feb</td>\n",
       "      <td>2017</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>Wade</td>\n",
       "      <td>668.212121</td>\n",
       "      <td>750.000000</td>\n",
       "      <td>670.000000</td>\n",
       "      <td>55.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>016c1867b7cc7622e183e8d74de2b93ddd9449a3</td>\n",
       "      <td>feb</td>\n",
       "      <td>2017</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>Dobson</td>\n",
       "      <td>678.413174</td>\n",
       "      <td>665.101796</td>\n",
       "      <td>660.119760</td>\n",
       "      <td>15.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>016c1867b7cc7622e183e8d74de2b93ddd9449a3</td>\n",
       "      <td>feb</td>\n",
       "      <td>2017</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>Dobson</td>\n",
       "      <td>671.799043</td>\n",
       "      <td>651.466507</td>\n",
       "      <td>646.074163</td>\n",
       "      <td>15.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0172751a7d4d785df2216952e1be26ff7861a414</td>\n",
       "      <td>feb</td>\n",
       "      <td>2017</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>Greensboro</td>\n",
       "      <td>715.666667</td>\n",
       "      <td>745.833333</td>\n",
       "      <td>714.666667</td>\n",
       "      <td>65.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1789</th>\n",
       "      <td>fec63da2d22356a5415bb21fc808aa2359d43333</td>\n",
       "      <td>feb</td>\n",
       "      <td>2021</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>High Point</td>\n",
       "      <td>695.341176</td>\n",
       "      <td>749.964706</td>\n",
       "      <td>699.858824</td>\n",
       "      <td>10.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1790</th>\n",
       "      <td>fefc39b30cc29ec1a80954d26e0550bf864455f7</td>\n",
       "      <td>feb</td>\n",
       "      <td>2021</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>Raleigh</td>\n",
       "      <td>680.634615</td>\n",
       "      <td>694.442308</td>\n",
       "      <td>694.442308</td>\n",
       "      <td>9.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1791</th>\n",
       "      <td>fefc39b30cc29ec1a80954d26e0550bf864455f7</td>\n",
       "      <td>feb</td>\n",
       "      <td>2021</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>Raleigh</td>\n",
       "      <td>680.950523</td>\n",
       "      <td>689.486760</td>\n",
       "      <td>689.479094</td>\n",
       "      <td>9.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1792</th>\n",
       "      <td>ff53874bcab6f2d086a50ca0766bcadebd61e62e</td>\n",
       "      <td>feb</td>\n",
       "      <td>2021</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>Matthews</td>\n",
       "      <td>687.875000</td>\n",
       "      <td>696.291667</td>\n",
       "      <td>694.458333</td>\n",
       "      <td>40.0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1793</th>\n",
       "      <td>ffc40b303849f7f5af77972db456dc63e2eebf34</td>\n",
       "      <td>feb</td>\n",
       "      <td>2021</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>Charlotte</td>\n",
       "      <td>729.204082</td>\n",
       "      <td>728.714286</td>\n",
       "      <td>727.071429</td>\n",
       "      <td>17.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>11850 rows × 13 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                    Identifier Month  Year HvacMode  \\\n",
       "0     00bbbdb5c3b6cb8b9e78191017ecdb97dad87e47   feb  2017     auto   \n",
       "1     00e5498cd188bdf09a9a1a902e6084b200cbbdef   feb  2017     auto   \n",
       "2     016c1867b7cc7622e183e8d74de2b93ddd9449a3   feb  2017     heat   \n",
       "3     016c1867b7cc7622e183e8d74de2b93ddd9449a3   feb  2017     heat   \n",
       "4     0172751a7d4d785df2216952e1be26ff7861a414   feb  2017     auto   \n",
       "...                                        ...   ...   ...      ...   \n",
       "1789  fec63da2d22356a5415bb21fc808aa2359d43333   feb  2021     auto   \n",
       "1790  fefc39b30cc29ec1a80954d26e0550bf864455f7   feb  2021     cool   \n",
       "1791  fefc39b30cc29ec1a80954d26e0550bf864455f7   feb  2021     heat   \n",
       "1792  ff53874bcab6f2d086a50ca0766bcadebd61e62e   feb  2021     heat   \n",
       "1793  ffc40b303849f7f5af77972db456dc63e2eebf34   feb  2021     heat   \n",
       "\n",
       "     CalendarEvent          City  Temperature_ctrl_ave  \\\n",
       "0             auto  Jacksonville            704.408854   \n",
       "1             hold          Wade            668.212121   \n",
       "2             auto        Dobson            678.413174   \n",
       "3             hold        Dobson            671.799043   \n",
       "4             auto    Greensboro            715.666667   \n",
       "...            ...           ...                   ...   \n",
       "1789          hold    High Point            695.341176   \n",
       "1790          hold       Raleigh            680.634615   \n",
       "1791          hold       Raleigh            680.950523   \n",
       "1792          hold      Matthews            687.875000   \n",
       "1793          hold     Charlotte            729.204082   \n",
       "\n",
       "      TemperatureExpectedCool_ave  TemperatureExpectedHeat_ave  \\\n",
       "0                      766.585938                   683.395833   \n",
       "1                      750.000000                   670.000000   \n",
       "2                      665.101796                   660.119760   \n",
       "3                      651.466507                   646.074163   \n",
       "4                      745.833333                   714.666667   \n",
       "...                           ...                          ...   \n",
       "1789                   749.964706                   699.858824   \n",
       "1790                   694.442308                   694.442308   \n",
       "1791                   689.486760                   689.479094   \n",
       "1792                   696.291667                   694.458333   \n",
       "1793                   728.714286                   727.071429   \n",
       "\n",
       "      Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \n",
       "0                     5.0             False         False             True  \n",
       "1                    55.0             False         False            False  \n",
       "2                    15.0             False         False             True  \n",
       "3                    15.0             False         False             True  \n",
       "4                    65.0             False         False            False  \n",
       "...                   ...               ...           ...              ...  \n",
       "1789                 10.0             False         False            False  \n",
       "1790                  9.0             False         False            False  \n",
       "1791                  9.0             False         False            False  \n",
       "1792                 40.0              True         False             True  \n",
       "1793                 17.0             False         False            False  \n",
       "\n",
       "[11850 rows x 13 columns]"
      ]
     },
     "execution_count": 66,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# https://stackoverflow.com/questions/63886787/how-to-create-a-dataframe-from-multiple-csv-files\n",
    "# Initialize an empty dataframe\n",
    "NC_feb = pd.DataFrame()\n",
    "\n",
    "# Iterate through files and contents, then concatenate the data into the dataframe\n",
    "for file in files:\n",
    "    df = pd.read_csv(\"data/day/NC/feb/\" + file)\n",
    "    NC_feb = pd.concat([NC_feb, df])\n",
    "    \n",
    "NC_feb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [],
   "source": [
    "NC_feb.to_csv(\"Scraper_Output/State_Month_Day/NC/NC_feb.csv\", header=True, index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "\n",
    "## March"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "\n",
    "## April"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "\n",
    "## May"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "\n",
    "## June"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2017 June Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "jun_2017 = pd.read_csv(\"../data_large/NC-day/2017-jun-day-NC.csv\")\n",
    "\n",
    "# jun_2017"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0728880afd6266fdf2d3460f2117d83883eaad3f</td>\n",
       "      <td>2017-06-25 18:50:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>713</td>\n",
       "      <td>720</td>\n",
       "      <td>720</td>\n",
       "      <td>NC</td>\n",
       "      <td>fuquay varina</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>9603c05f578f754545822e2e30a39e3a0c94885b</td>\n",
       "      <td>2017-06-08 15:50:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>692</td>\n",
       "      <td>690</td>\n",
       "      <td>640</td>\n",
       "      <td>NC</td>\n",
       "      <td>China grove</td>\n",
       "      <td>15</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>7d45db4b4f18c0950c490b7ef6297af2f876fd08</td>\n",
       "      <td>2017-06-05 13:55:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>728</td>\n",
       "      <td>730</td>\n",
       "      <td>680</td>\n",
       "      <td>NC</td>\n",
       "      <td>Leland</td>\n",
       "      <td>15</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>cc229476854c73b582f675278138537e884a2e9c</td>\n",
       "      <td>2017-06-03 14:15:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>717</td>\n",
       "      <td>720</td>\n",
       "      <td>650</td>\n",
       "      <td>NC</td>\n",
       "      <td>Otto</td>\n",
       "      <td>45</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>87d2561ec5cfa0cceb2f2b65c449b7d23d2df03b</td>\n",
       "      <td>2017-06-24 14:25:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>718</td>\n",
       "      <td>720</td>\n",
       "      <td>730</td>\n",
       "      <td>NC</td>\n",
       "      <td>Statesville</td>\n",
       "      <td>55</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>881817</th>\n",
       "      <td>4f60b83cc51d90e13839e112d06909cbb5ba4273</td>\n",
       "      <td>2017-06-21 17:25:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>719</td>\n",
       "      <td>720</td>\n",
       "      <td>720</td>\n",
       "      <td>NC</td>\n",
       "      <td>Ocean Isle Beach</td>\n",
       "      <td>15</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>881818</th>\n",
       "      <td>92de2058ee016c38e6488ba9086868768d604156</td>\n",
       "      <td>2017-06-19 15:15:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>758</td>\n",
       "      <td>740</td>\n",
       "      <td>740</td>\n",
       "      <td>NC</td>\n",
       "      <td>Ocean Isle Beach</td>\n",
       "      <td>10</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>881819</th>\n",
       "      <td>4f60b83cc51d90e13839e112d06909cbb5ba4273</td>\n",
       "      <td>2017-06-06 11:45:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>742</td>\n",
       "      <td>740</td>\n",
       "      <td>740</td>\n",
       "      <td>NC</td>\n",
       "      <td>Ocean Isle Beach</td>\n",
       "      <td>15</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>881820</th>\n",
       "      <td>4f60b83cc51d90e13839e112d06909cbb5ba4273</td>\n",
       "      <td>2017-06-01 19:45:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>743</td>\n",
       "      <td>740</td>\n",
       "      <td>740</td>\n",
       "      <td>NC</td>\n",
       "      <td>Ocean Isle Beach</td>\n",
       "      <td>15</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>881821</th>\n",
       "      <td>4f60b83cc51d90e13839e112d06909cbb5ba4273</td>\n",
       "      <td>2017-06-23 17:10:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>724</td>\n",
       "      <td>720</td>\n",
       "      <td>720</td>\n",
       "      <td>NC</td>\n",
       "      <td>Ocean Isle Beach</td>\n",
       "      <td>15</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>852296 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                      Identifier                date_time  \\\n",
       "0       0728880afd6266fdf2d3460f2117d83883eaad3f  2017-06-25 18:50:00 UTC   \n",
       "1       9603c05f578f754545822e2e30a39e3a0c94885b  2017-06-08 15:50:00 UTC   \n",
       "2       7d45db4b4f18c0950c490b7ef6297af2f876fd08  2017-06-05 13:55:00 UTC   \n",
       "3       cc229476854c73b582f675278138537e884a2e9c  2017-06-03 14:15:00 UTC   \n",
       "4       87d2561ec5cfa0cceb2f2b65c449b7d23d2df03b  2017-06-24 14:25:00 UTC   \n",
       "...                                          ...                      ...   \n",
       "881817  4f60b83cc51d90e13839e112d06909cbb5ba4273  2017-06-21 17:25:00 UTC   \n",
       "881818  92de2058ee016c38e6488ba9086868768d604156  2017-06-19 15:15:00 UTC   \n",
       "881819  4f60b83cc51d90e13839e112d06909cbb5ba4273  2017-06-06 11:45:00 UTC   \n",
       "881820  4f60b83cc51d90e13839e112d06909cbb5ba4273  2017-06-01 19:45:00 UTC   \n",
       "881821  4f60b83cc51d90e13839e112d06909cbb5ba4273  2017-06-23 17:10:00 UTC   \n",
       "\n",
       "       HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0          cool          hold               713                      720   \n",
       "1          auto          hold               692                      690   \n",
       "2          auto          hold               728                      730   \n",
       "3          auto          hold               717                      720   \n",
       "4          cool          auto               718                      720   \n",
       "...         ...           ...               ...                      ...   \n",
       "881817     cool          auto               719                      720   \n",
       "881818     cool          hold               758                      740   \n",
       "881819     cool          hold               742                      740   \n",
       "881820     cool          hold               743                      740   \n",
       "881821     cool          hold               724                      720   \n",
       "\n",
       "        TemperatureExpectedHeat ProvinceState              City  \\\n",
       "0                           720            NC     fuquay varina   \n",
       "1                           640            NC       China grove   \n",
       "2                           680            NC            Leland   \n",
       "3                           650            NC              Otto   \n",
       "4                           730            NC       Statesville   \n",
       "...                         ...           ...               ...   \n",
       "881817                      720            NC  Ocean Isle Beach   \n",
       "881818                      740            NC  Ocean Isle Beach   \n",
       "881819                      740            NC  Ocean Isle Beach   \n",
       "881820                      740            NC  Ocean Isle Beach   \n",
       "881821                      720            NC  Ocean Isle Beach   \n",
       "\n",
       "        Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                         5             False         False            False   \n",
       "1                        15              True         False             True   \n",
       "2                        15             False         False             True   \n",
       "3                        45             False         False             True   \n",
       "4                        55             False         False            False   \n",
       "...                     ...               ...           ...              ...   \n",
       "881817                   15              True         False             True   \n",
       "881818                   10              True         False             True   \n",
       "881819                   15              True         False             True   \n",
       "881820                   15              True         False             True   \n",
       "881821                   15              True         False             True   \n",
       "\n",
       "       Auxilliary_Heat_Fuel_Type  \n",
       "0                            Gas  \n",
       "1                       Electric  \n",
       "2                       Electric  \n",
       "3                       Electric  \n",
       "4                            Gas  \n",
       "...                          ...  \n",
       "881817                  Electric  \n",
       "881818                  Electric  \n",
       "881819                  Electric  \n",
       "881820                  Electric  \n",
       "881821                  Electric  \n",
       "\n",
       "[852296 rows x 14 columns]"
      ]
     },
     "execution_count": 69,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "jun_2017.drop(jun_2017[jun_2017['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "jun_2017.drop(jun_2017[jun_2017['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "jun_2017.drop(jun_2017[jun_2017['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "jun_2017.drop(jun_2017[jun_2017['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "jun_2017"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "jun_2017[\"Year\"] = \"2017\"\n",
    "jun_2017[\"Month\"] = \"jun\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "jun_2017 = jun_2017.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "jun_2017_ave = jun_2017.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# jun_2017_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "jun_2017_ave.to_csv(\"data/day/NC/jun/jun_2017_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2018 June Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "jun_2018 = pd.read_csv(\"../data_large/NC-day/2018-jun-day-NC.csv\")\n",
    "\n",
    "# jun_2018"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0084e95b39ca4e03056e3d534b32f75d40dd6d9c</td>\n",
       "      <td>2018-06-13 18:30:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>707</td>\n",
       "      <td>705</td>\n",
       "      <td>655</td>\n",
       "      <td>NC</td>\n",
       "      <td>Holly Springs</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>b8be492bf0cc720170fb1148587f54015e9ee80e</td>\n",
       "      <td>2018-06-01 12:25:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>717</td>\n",
       "      <td>716</td>\n",
       "      <td>716</td>\n",
       "      <td>NC</td>\n",
       "      <td>Raleigh</td>\n",
       "      <td>20</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>74bfd0a47e69f1488341f189f00d74f1334272d8</td>\n",
       "      <td>2018-06-20 18:45:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>738</td>\n",
       "      <td>732</td>\n",
       "      <td>732</td>\n",
       "      <td>NC</td>\n",
       "      <td>Kernersville</td>\n",
       "      <td>20</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>c446a0fe5bbee92382817e5562cbd754aeb7f46c</td>\n",
       "      <td>2018-06-28 19:05:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>725</td>\n",
       "      <td>695</td>\n",
       "      <td>695</td>\n",
       "      <td>NC</td>\n",
       "      <td>Bolivia</td>\n",
       "      <td>10</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1c693b10da7dc9702c23d465b8338e2e1177508c</td>\n",
       "      <td>2018-06-27 17:35:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>718</td>\n",
       "      <td>725</td>\n",
       "      <td>725</td>\n",
       "      <td>NC</td>\n",
       "      <td>Asheville</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1965559</th>\n",
       "      <td>ec6f00e73c1d00f4a4d9d94c16cd97354ae0da2d</td>\n",
       "      <td>2018-06-30 17:15:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>753</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>NC</td>\n",
       "      <td>Waxhaw</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1965560</th>\n",
       "      <td>82f5f5424be6a8cc4453828e476b3a9f235c2d8a</td>\n",
       "      <td>2018-06-27 14:35:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>NC</td>\n",
       "      <td>Raeford</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1965561</th>\n",
       "      <td>5c9250f19f9dac7e44323e9cc38e08d6b300eafd</td>\n",
       "      <td>2018-06-27 18:40:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>767</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>NC</td>\n",
       "      <td>Calabash</td>\n",
       "      <td>0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1965562</th>\n",
       "      <td>0f145da50f523037b4b8e170e563840a7ba0739a</td>\n",
       "      <td>2018-06-24 18:55:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>788</td>\n",
       "      <td>785</td>\n",
       "      <td>760</td>\n",
       "      <td>NC</td>\n",
       "      <td>Wake Forest</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1965563</th>\n",
       "      <td>92de2058ee016c38e6488ba9086868768d604156</td>\n",
       "      <td>2018-06-03 18:05:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>766</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>NC</td>\n",
       "      <td>Ocean Isle Beach</td>\n",
       "      <td>10</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1880670 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                       Identifier                date_time  \\\n",
       "0        0084e95b39ca4e03056e3d534b32f75d40dd6d9c  2018-06-13 18:30:00 UTC   \n",
       "1        b8be492bf0cc720170fb1148587f54015e9ee80e  2018-06-01 12:25:00 UTC   \n",
       "2        74bfd0a47e69f1488341f189f00d74f1334272d8  2018-06-20 18:45:00 UTC   \n",
       "3        c446a0fe5bbee92382817e5562cbd754aeb7f46c  2018-06-28 19:05:00 UTC   \n",
       "4        1c693b10da7dc9702c23d465b8338e2e1177508c  2018-06-27 17:35:00 UTC   \n",
       "...                                           ...                      ...   \n",
       "1965559  ec6f00e73c1d00f4a4d9d94c16cd97354ae0da2d  2018-06-30 17:15:00 UTC   \n",
       "1965560  82f5f5424be6a8cc4453828e476b3a9f235c2d8a  2018-06-27 14:35:00 UTC   \n",
       "1965561  5c9250f19f9dac7e44323e9cc38e08d6b300eafd  2018-06-27 18:40:00 UTC   \n",
       "1965562  0f145da50f523037b4b8e170e563840a7ba0739a  2018-06-24 18:55:00 UTC   \n",
       "1965563  92de2058ee016c38e6488ba9086868768d604156  2018-06-03 18:05:00 UTC   \n",
       "\n",
       "        HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0           auto          hold               707                      705   \n",
       "1           cool          hold               717                      716   \n",
       "2           cool          hold               738                      732   \n",
       "3           cool          hold               725                      695   \n",
       "4           cool          hold               718                      725   \n",
       "...          ...           ...               ...                      ...   \n",
       "1965559     cool          auto               753                      760   \n",
       "1965560     cool          hold               760                      760   \n",
       "1965561     cool          hold               767                      760   \n",
       "1965562     cool          hold               788                      785   \n",
       "1965563     cool          hold               766                      760   \n",
       "\n",
       "         TemperatureExpectedHeat ProvinceState              City  \\\n",
       "0                            655            NC     Holly Springs   \n",
       "1                            716            NC           Raleigh   \n",
       "2                            732            NC      Kernersville   \n",
       "3                            695            NC           Bolivia   \n",
       "4                            725            NC         Asheville   \n",
       "...                          ...           ...               ...   \n",
       "1965559                      760            NC            Waxhaw   \n",
       "1965560                      760            NC           Raeford   \n",
       "1965561                      760            NC          Calabash   \n",
       "1965562                      760            NC       Wake Forest   \n",
       "1965563                      760            NC  Ocean Isle Beach   \n",
       "\n",
       "         Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                         10             False         False            False   \n",
       "1                         20             False         False            False   \n",
       "2                         20             False         False            False   \n",
       "3                         10              True         False             True   \n",
       "4                          0             False         False            False   \n",
       "...                      ...               ...           ...              ...   \n",
       "1965559                    0             False         False            False   \n",
       "1965560                    0             False         False             True   \n",
       "1965561                    0              True         False             True   \n",
       "1965562                    5             False         False            False   \n",
       "1965563                   10              True         False             True   \n",
       "\n",
       "        Auxilliary_Heat_Fuel_Type  \n",
       "0                             Gas  \n",
       "1                             Gas  \n",
       "2                             Gas  \n",
       "3                        Electric  \n",
       "4                             Gas  \n",
       "...                           ...  \n",
       "1965559                       Gas  \n",
       "1965560                  Electric  \n",
       "1965561                  Electric  \n",
       "1965562                       Gas  \n",
       "1965563                  Electric  \n",
       "\n",
       "[1880670 rows x 14 columns]"
      ]
     },
     "execution_count": 75,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "jun_2018.drop(jun_2018[jun_2018['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "jun_2018.drop(jun_2018[jun_2018['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "jun_2018.drop(jun_2018[jun_2018['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "jun_2018.drop(jun_2018[jun_2018['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "jun_2018"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "jun_2018[\"Year\"] = \"2018\"\n",
    "jun_2018[\"Month\"] = \"jun\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "jun_2018 = jun_2018.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "jun_2018_ave = jun_2018.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# jun_2018_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "jun_2018_ave.to_csv(\"data/day/NC/jun/jun_2018_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2019 June Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "jun_2019 = pd.read_csv(\"../data_large/NC-day/2019-jun-day-NC.csv\")\n",
    "\n",
    "# jun_2019"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>066bafc50eec2538b881bbf59abdcf81c9165d00</td>\n",
       "      <td>2019-06-20 18:20:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>782</td>\n",
       "      <td>785</td>\n",
       "      <td>735</td>\n",
       "      <td>NC</td>\n",
       "      <td>Morrisville</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>589d9ccec1f293d98e6228a58441cbe92f1cdd27</td>\n",
       "      <td>2019-06-16 12:40:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>680</td>\n",
       "      <td>735</td>\n",
       "      <td>735</td>\n",
       "      <td>NC</td>\n",
       "      <td>Oxford</td>\n",
       "      <td>70</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>6543db6cb136267d6927a8c11e07be95b468b1c9</td>\n",
       "      <td>2019-06-01 16:40:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>724</td>\n",
       "      <td>749</td>\n",
       "      <td>635</td>\n",
       "      <td>NC</td>\n",
       "      <td>Winston Salem</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>182728ad7e077c2e61ced2f5c6c40c988679803d</td>\n",
       "      <td>2019-06-09 13:50:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>729</td>\n",
       "      <td>725</td>\n",
       "      <td>675</td>\n",
       "      <td>NC</td>\n",
       "      <td>Durham</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>c29ab5fbaa67337111cdf527ba67ce10fc29d708</td>\n",
       "      <td>2019-06-22 13:20:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>723</td>\n",
       "      <td>720</td>\n",
       "      <td>610</td>\n",
       "      <td>NC</td>\n",
       "      <td>Matthews</td>\n",
       "      <td>17</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2939733</th>\n",
       "      <td>8b4d3f18accb8430c3943d0ee59134c3cd0a01c7</td>\n",
       "      <td>2019-06-06 15:20:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>766</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>NC</td>\n",
       "      <td>Charlotte</td>\n",
       "      <td>8</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2939734</th>\n",
       "      <td>3d7afe0b7003787c72b48a686b444355c251141a</td>\n",
       "      <td>2019-06-24 14:50:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>734</td>\n",
       "      <td>730</td>\n",
       "      <td>760</td>\n",
       "      <td>NC</td>\n",
       "      <td>Morganton</td>\n",
       "      <td>5</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2939735</th>\n",
       "      <td>c072bfdd0711f81e0dbeee389f577cb6d3fa04b9</td>\n",
       "      <td>2019-06-17 12:50:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>740</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>NC</td>\n",
       "      <td>Cary</td>\n",
       "      <td>27</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2939736</th>\n",
       "      <td>b339a917fe80c8c20571231413ae40578b82557d</td>\n",
       "      <td>2019-06-07 11:50:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>748</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>NC</td>\n",
       "      <td>Cary</td>\n",
       "      <td>20</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2939737</th>\n",
       "      <td>ebbe4fc38dcb386a76eda2b0a42f615e440ce341</td>\n",
       "      <td>2019-06-08 14:30:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>741</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>NC</td>\n",
       "      <td>Raleigh</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2830545 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                       Identifier                date_time  \\\n",
       "0        066bafc50eec2538b881bbf59abdcf81c9165d00  2019-06-20 18:20:00 UTC   \n",
       "1        589d9ccec1f293d98e6228a58441cbe92f1cdd27  2019-06-16 12:40:00 UTC   \n",
       "2        6543db6cb136267d6927a8c11e07be95b468b1c9  2019-06-01 16:40:00 UTC   \n",
       "4        182728ad7e077c2e61ced2f5c6c40c988679803d  2019-06-09 13:50:00 UTC   \n",
       "5        c29ab5fbaa67337111cdf527ba67ce10fc29d708  2019-06-22 13:20:00 UTC   \n",
       "...                                           ...                      ...   \n",
       "2939733  8b4d3f18accb8430c3943d0ee59134c3cd0a01c7  2019-06-06 15:20:00 UTC   \n",
       "2939734  3d7afe0b7003787c72b48a686b444355c251141a  2019-06-24 14:50:00 UTC   \n",
       "2939735  c072bfdd0711f81e0dbeee389f577cb6d3fa04b9  2019-06-17 12:50:00 UTC   \n",
       "2939736  b339a917fe80c8c20571231413ae40578b82557d  2019-06-07 11:50:00 UTC   \n",
       "2939737  ebbe4fc38dcb386a76eda2b0a42f615e440ce341  2019-06-08 14:30:00 UTC   \n",
       "\n",
       "        HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0           auto          hold               782                      785   \n",
       "1           cool          hold               680                      735   \n",
       "2           auto          hold               724                      749   \n",
       "4           auto          hold               729                      725   \n",
       "5           auto          hold               723                      720   \n",
       "...          ...           ...               ...                      ...   \n",
       "2939733     cool          hold               766                      760   \n",
       "2939734     cool          auto               734                      730   \n",
       "2939735     cool          hold               740                      760   \n",
       "2939736     cool          hold               748                      760   \n",
       "2939737     cool          hold               741                      760   \n",
       "\n",
       "         TemperatureExpectedHeat ProvinceState           City  \\\n",
       "0                            735            NC    Morrisville   \n",
       "1                            735            NC         Oxford   \n",
       "2                            635            NC  Winston Salem   \n",
       "4                            675            NC         Durham   \n",
       "5                            610            NC       Matthews   \n",
       "...                          ...           ...            ...   \n",
       "2939733                      760            NC      Charlotte   \n",
       "2939734                      760            NC      Morganton   \n",
       "2939735                      760            NC           Cary   \n",
       "2939736                      760            NC           Cary   \n",
       "2939737                      760            NC        Raleigh   \n",
       "\n",
       "         Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                          0             False         False            False   \n",
       "1                         70              True         False             True   \n",
       "2                         10             False         False            False   \n",
       "4                          0             False         False             True   \n",
       "5                         17              True         False             True   \n",
       "...                      ...               ...           ...              ...   \n",
       "2939733                    8             False         False            False   \n",
       "2939734                    5              True         False             True   \n",
       "2939735                   27             False         False            False   \n",
       "2939736                   20             False         False            False   \n",
       "2939737                   10             False         False            False   \n",
       "\n",
       "        Auxilliary_Heat_Fuel_Type  \n",
       "0                             Gas  \n",
       "1                        Electric  \n",
       "2                             Gas  \n",
       "4                        Electric  \n",
       "5                        Electric  \n",
       "...                           ...  \n",
       "2939733                       Gas  \n",
       "2939734                  Electric  \n",
       "2939735                       Gas  \n",
       "2939736                       Gas  \n",
       "2939737                       Gas  \n",
       "\n",
       "[2830545 rows x 14 columns]"
      ]
     },
     "execution_count": 81,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "jun_2019.drop(jun_2019[jun_2019['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "jun_2019.drop(jun_2019[jun_2019['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "jun_2019.drop(jun_2019[jun_2019['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "jun_2019.drop(jun_2019[jun_2019['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "jun_2019"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "jun_2019[\"Year\"] = \"2019\"\n",
    "jun_2019[\"Month\"] = \"jun\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "jun_2019 = jun_2019.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "jun_2019_ave = jun_2019.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# jun_2019_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "jun_2019_ave.to_csv(\"data/day/NC/jun/jun_2019_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2020 June Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "jun_2020 = pd.read_csv(\"../data_large/NC-day/2020-jun-day-NC.csv\")\n",
    "\n",
    "# jun_2020"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>e5c2cf6e3f1c3c70580d437007d360470e77accf</td>\n",
       "      <td>2020-06-21 19:10:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>747</td>\n",
       "      <td>742</td>\n",
       "      <td>692</td>\n",
       "      <td>NC</td>\n",
       "      <td>Charlotte</td>\n",
       "      <td>30</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>e1e77ef09f2de24bfa3e59ae1485f4029e72a452</td>\n",
       "      <td>2020-06-07 15:45:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>730</td>\n",
       "      <td>726</td>\n",
       "      <td>726</td>\n",
       "      <td>NC</td>\n",
       "      <td>Carrboro</td>\n",
       "      <td>20</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>ca8fc81e5e4f567bbe49aad9edc0173056960640</td>\n",
       "      <td>2020-06-07 16:30:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>715</td>\n",
       "      <td>715</td>\n",
       "      <td>655</td>\n",
       "      <td>NC</td>\n",
       "      <td>Cary</td>\n",
       "      <td>39</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>e1e77ef09f2de24bfa3e59ae1485f4029e72a452</td>\n",
       "      <td>2020-06-25 18:20:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>726</td>\n",
       "      <td>726</td>\n",
       "      <td>726</td>\n",
       "      <td>NC</td>\n",
       "      <td>Carrboro</td>\n",
       "      <td>20</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>a3c71856e280b64edc61bcd382ac3543b7754c83</td>\n",
       "      <td>2020-06-16 18:15:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>744</td>\n",
       "      <td>733</td>\n",
       "      <td>683</td>\n",
       "      <td>NC</td>\n",
       "      <td>Rolesville</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2679415</th>\n",
       "      <td>7c222047990d1e0e21086d3229c943c384cdb8ea</td>\n",
       "      <td>2020-06-27 17:15:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>758</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>NC</td>\n",
       "      <td>Greensboro</td>\n",
       "      <td>95</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2679416</th>\n",
       "      <td>6ef60dca7aa6fef9018979fe62829c60a5b80974</td>\n",
       "      <td>2020-06-27 13:50:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>NC</td>\n",
       "      <td>Valdese</td>\n",
       "      <td>0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2679417</th>\n",
       "      <td>520d05b12eb1681ea8a331402cd4184b532426cb</td>\n",
       "      <td>2020-06-14 10:35:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>736</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>NC</td>\n",
       "      <td>Durham</td>\n",
       "      <td>0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2679418</th>\n",
       "      <td>96d485494e0a00c665d84cf06e96bc21d1d651ab</td>\n",
       "      <td>2020-06-24 13:45:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>NC</td>\n",
       "      <td>Kernersville</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2679419</th>\n",
       "      <td>6ef60dca7aa6fef9018979fe62829c60a5b80974</td>\n",
       "      <td>2020-06-10 17:00:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>758</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>NC</td>\n",
       "      <td>Valdese</td>\n",
       "      <td>0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2599802 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                       Identifier                date_time  \\\n",
       "0        e5c2cf6e3f1c3c70580d437007d360470e77accf  2020-06-21 19:10:00 UTC   \n",
       "1        e1e77ef09f2de24bfa3e59ae1485f4029e72a452  2020-06-07 15:45:00 UTC   \n",
       "3        ca8fc81e5e4f567bbe49aad9edc0173056960640  2020-06-07 16:30:00 UTC   \n",
       "4        e1e77ef09f2de24bfa3e59ae1485f4029e72a452  2020-06-25 18:20:00 UTC   \n",
       "5        a3c71856e280b64edc61bcd382ac3543b7754c83  2020-06-16 18:15:00 UTC   \n",
       "...                                           ...                      ...   \n",
       "2679415  7c222047990d1e0e21086d3229c943c384cdb8ea  2020-06-27 17:15:00 UTC   \n",
       "2679416  6ef60dca7aa6fef9018979fe62829c60a5b80974  2020-06-27 13:50:00 UTC   \n",
       "2679417  520d05b12eb1681ea8a331402cd4184b532426cb  2020-06-14 10:35:00 UTC   \n",
       "2679418  96d485494e0a00c665d84cf06e96bc21d1d651ab  2020-06-24 13:45:00 UTC   \n",
       "2679419  6ef60dca7aa6fef9018979fe62829c60a5b80974  2020-06-10 17:00:00 UTC   \n",
       "\n",
       "        HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0           auto          hold               747                      742   \n",
       "1           cool          hold               730                      726   \n",
       "3           auto          hold               715                      715   \n",
       "4           cool          hold               726                      726   \n",
       "5           auto          hold               744                      733   \n",
       "...          ...           ...               ...                      ...   \n",
       "2679415     cool          hold               758                      760   \n",
       "2679416     cool          hold               760                      760   \n",
       "2679417     cool          auto               736                      760   \n",
       "2679418     cool          auto               760                      760   \n",
       "2679419     cool          hold               758                      760   \n",
       "\n",
       "         TemperatureExpectedHeat ProvinceState          City  \\\n",
       "0                            692            NC     Charlotte   \n",
       "1                            726            NC      Carrboro   \n",
       "3                            655            NC          Cary   \n",
       "4                            726            NC      Carrboro   \n",
       "5                            683            NC    Rolesville   \n",
       "...                          ...           ...           ...   \n",
       "2679415                      760            NC    Greensboro   \n",
       "2679416                      760            NC       Valdese   \n",
       "2679417                      760            NC        Durham   \n",
       "2679418                      760            NC  Kernersville   \n",
       "2679419                      760            NC       Valdese   \n",
       "\n",
       "         Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                         30             False         False            False   \n",
       "1                         20             False         False            False   \n",
       "3                         39             False         False            False   \n",
       "4                         20             False         False            False   \n",
       "5                          5             False         False            False   \n",
       "...                      ...               ...           ...              ...   \n",
       "2679415                   95              True         False            False   \n",
       "2679416                    0              True         False            False   \n",
       "2679417                    0              True         False            False   \n",
       "2679418                   10             False         False            False   \n",
       "2679419                    0              True         False            False   \n",
       "\n",
       "        Auxilliary_Heat_Fuel_Type  \n",
       "0                             Gas  \n",
       "1                             Gas  \n",
       "3                             Gas  \n",
       "4                             Gas  \n",
       "5                             Gas  \n",
       "...                           ...  \n",
       "2679415                       Gas  \n",
       "2679416                       Gas  \n",
       "2679417                       Gas  \n",
       "2679418                       Gas  \n",
       "2679419                       Gas  \n",
       "\n",
       "[2599802 rows x 14 columns]"
      ]
     },
     "execution_count": 87,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "jun_2020.drop(jun_2020[jun_2020['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "jun_2020.drop(jun_2020[jun_2020['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "jun_2020.drop(jun_2020[jun_2020['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "jun_2020.drop(jun_2020[jun_2020['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "jun_2020"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "jun_2020[\"Year\"] = \"2020\"\n",
    "jun_2020[\"Month\"] = \"jun\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "jun_2020 = jun_2020.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "jun_2020_ave = jun_2020.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# jun_2020_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "jun_2020_ave.to_csv(\"data/day/NC/jun/jun_2020_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2021 June Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "jun_2021 = pd.read_csv(\"../data_large/NC-day/2021-jun-day-NC.csv\")\n",
    "\n",
    "# jun_2021"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1dad29d4f066ac2bc6feaffe1b8c91760d8be7e3</td>\n",
       "      <td>2021-06-14 16:00:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>778</td>\n",
       "      <td>775</td>\n",
       "      <td>775</td>\n",
       "      <td>NC</td>\n",
       "      <td>Ocean Isle Beach</td>\n",
       "      <td>38</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>6ed72dea1c1113cebbd1e68929be48316aacb9c0</td>\n",
       "      <td>2021-06-11 18:00:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>770</td>\n",
       "      <td>765</td>\n",
       "      <td>765</td>\n",
       "      <td>NC</td>\n",
       "      <td>Fayetteville</td>\n",
       "      <td>20</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1ab7a4af883a5034db39634b035980051b174734</td>\n",
       "      <td>2021-06-10 16:55:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>725</td>\n",
       "      <td>721</td>\n",
       "      <td>647</td>\n",
       "      <td>NC</td>\n",
       "      <td>Colfax</td>\n",
       "      <td>18</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>6ba7f07494cb93fd588038124c62e6e1977f315a</td>\n",
       "      <td>2021-06-29 14:00:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>733</td>\n",
       "      <td>737</td>\n",
       "      <td>737</td>\n",
       "      <td>NC</td>\n",
       "      <td>Troutman</td>\n",
       "      <td>10</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>cdd1ece1a6315fc7f11ade506f5e647aae3762eb</td>\n",
       "      <td>2021-06-05 13:35:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>711</td>\n",
       "      <td>708</td>\n",
       "      <td>648</td>\n",
       "      <td>NC</td>\n",
       "      <td>Wilmington</td>\n",
       "      <td>39</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1653325</th>\n",
       "      <td>90ad92b79cb938e75aa5714e3dc3a9d36cbd70ac</td>\n",
       "      <td>2021-06-20 15:40:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>759</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>NC</td>\n",
       "      <td>Clayton</td>\n",
       "      <td>9</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1653326</th>\n",
       "      <td>69d4cc4af97a0dcd265ef096bfe01956034cddc4</td>\n",
       "      <td>2021-06-23 19:35:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>NC</td>\n",
       "      <td>Apex</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1653327</th>\n",
       "      <td>e5b88888d7887de52548eaa14e345563999c5af1</td>\n",
       "      <td>2021-06-12 14:55:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>757</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>NC</td>\n",
       "      <td>Raleigh</td>\n",
       "      <td>29</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1653328</th>\n",
       "      <td>4099729dd97004b4b8e526c5d4425c0c5c0f3d43</td>\n",
       "      <td>2021-06-15 14:30:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>744</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>NC</td>\n",
       "      <td>Charlotte</td>\n",
       "      <td>7</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1653329</th>\n",
       "      <td>522bdfb385a131b84f09730f8c1e90294bc279ad</td>\n",
       "      <td>2021-06-13 18:40:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>762</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>NC</td>\n",
       "      <td>Holly Springs</td>\n",
       "      <td>10</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1599638 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                       Identifier                date_time  \\\n",
       "0        1dad29d4f066ac2bc6feaffe1b8c91760d8be7e3  2021-06-14 16:00:00 UTC   \n",
       "1        6ed72dea1c1113cebbd1e68929be48316aacb9c0  2021-06-11 18:00:00 UTC   \n",
       "2        1ab7a4af883a5034db39634b035980051b174734  2021-06-10 16:55:00 UTC   \n",
       "3        6ba7f07494cb93fd588038124c62e6e1977f315a  2021-06-29 14:00:00 UTC   \n",
       "5        cdd1ece1a6315fc7f11ade506f5e647aae3762eb  2021-06-05 13:35:00 UTC   \n",
       "...                                           ...                      ...   \n",
       "1653325  90ad92b79cb938e75aa5714e3dc3a9d36cbd70ac  2021-06-20 15:40:00 UTC   \n",
       "1653326  69d4cc4af97a0dcd265ef096bfe01956034cddc4  2021-06-23 19:35:00 UTC   \n",
       "1653327  e5b88888d7887de52548eaa14e345563999c5af1  2021-06-12 14:55:00 UTC   \n",
       "1653328  4099729dd97004b4b8e526c5d4425c0c5c0f3d43  2021-06-15 14:30:00 UTC   \n",
       "1653329  522bdfb385a131b84f09730f8c1e90294bc279ad  2021-06-13 18:40:00 UTC   \n",
       "\n",
       "        HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0           cool          hold               778                      775   \n",
       "1           cool          hold               770                      765   \n",
       "2           auto          hold               725                      721   \n",
       "3           cool          hold               733                      737   \n",
       "5           cool          hold               711                      708   \n",
       "...          ...           ...               ...                      ...   \n",
       "1653325     cool          hold               759                      760   \n",
       "1653326     cool          hold               760                      760   \n",
       "1653327     cool          hold               757                      760   \n",
       "1653328     cool          hold               744                      760   \n",
       "1653329     cool          hold               762                      760   \n",
       "\n",
       "         TemperatureExpectedHeat ProvinceState              City  \\\n",
       "0                            775            NC  Ocean Isle Beach   \n",
       "1                            765            NC      Fayetteville   \n",
       "2                            647            NC            Colfax   \n",
       "3                            737            NC          Troutman   \n",
       "5                            648            NC        Wilmington   \n",
       "...                          ...           ...               ...   \n",
       "1653325                      760            NC           Clayton   \n",
       "1653326                      760            NC              Apex   \n",
       "1653327                      760            NC           Raleigh   \n",
       "1653328                      760            NC         Charlotte   \n",
       "1653329                      760            NC     Holly Springs   \n",
       "\n",
       "         Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                         38              True         False             True   \n",
       "1                         20              True         False             True   \n",
       "2                         18             False         False            False   \n",
       "3                         10              True         False             True   \n",
       "5                         39             False         False             True   \n",
       "...                      ...               ...           ...              ...   \n",
       "1653325                    9              True         False            False   \n",
       "1653326                   10             False         False            False   \n",
       "1653327                   29              True         False            False   \n",
       "1653328                    7             False         False            False   \n",
       "1653329                   10              True         False            False   \n",
       "\n",
       "        Auxilliary_Heat_Fuel_Type  \n",
       "0                        Electric  \n",
       "1                        Electric  \n",
       "2                             Gas  \n",
       "3                        Electric  \n",
       "5                        Electric  \n",
       "...                           ...  \n",
       "1653325                       Gas  \n",
       "1653326                       Gas  \n",
       "1653327                       Gas  \n",
       "1653328                       Gas  \n",
       "1653329                       Gas  \n",
       "\n",
       "[1599638 rows x 14 columns]"
      ]
     },
     "execution_count": 93,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "jun_2021.drop(jun_2021[jun_2021['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "jun_2021.drop(jun_2021[jun_2021['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "jun_2021.drop(jun_2021[jun_2021['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "jun_2021.drop(jun_2021[jun_2021['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "jun_2021"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "jun_2021[\"Year\"] = \"2021\"\n",
    "jun_2021[\"Month\"] = \"jun\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "jun_2021 = jun_2021.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "jun_2021_ave = jun_2021.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# jun_2021_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "jun_2021_ave.to_csv(\"data/day/NC/jun/jun_2021_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "\n",
    "### Combine month CSV Files \n",
    "1. Read in files in folders for each state\n",
    "2. Export as combined CSV"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create variable for files in directory\n",
    "files = [f for f in os.listdir(\"data/day/NC/jun/\") if f.endswith(\".csv\")]\n",
    "\n",
    "# files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>Month</th>\n",
       "      <th>Year</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>City</th>\n",
       "      <th>Temperature_ctrl_ave</th>\n",
       "      <th>TemperatureExpectedCool_ave</th>\n",
       "      <th>TemperatureExpectedHeat_ave</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0010db271149fae01fa527e2c2105074d5771e5f</td>\n",
       "      <td>jun</td>\n",
       "      <td>2017</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>Raleigh</td>\n",
       "      <td>743.740741</td>\n",
       "      <td>760.000000</td>\n",
       "      <td>690.000000</td>\n",
       "      <td>37.0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0010db271149fae01fa527e2c2105074d5771e5f</td>\n",
       "      <td>jun</td>\n",
       "      <td>2017</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>Raleigh</td>\n",
       "      <td>738.705882</td>\n",
       "      <td>740.000000</td>\n",
       "      <td>740.000000</td>\n",
       "      <td>37.0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>004dfca396d696b8a7ce7dd71862b78384f55976</td>\n",
       "      <td>jun</td>\n",
       "      <td>2017</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>Charlotte</td>\n",
       "      <td>785.607143</td>\n",
       "      <td>804.000000</td>\n",
       "      <td>650.392857</td>\n",
       "      <td>85.0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>00bbbdb5c3b6cb8b9e78191017ecdb97dad87e47</td>\n",
       "      <td>jun</td>\n",
       "      <td>2017</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>Jacksonville</td>\n",
       "      <td>749.244755</td>\n",
       "      <td>756.865385</td>\n",
       "      <td>687.638112</td>\n",
       "      <td>5.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>00bbbdb5c3b6cb8b9e78191017ecdb97dad87e47</td>\n",
       "      <td>jun</td>\n",
       "      <td>2017</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>Jacksonville</td>\n",
       "      <td>737.281250</td>\n",
       "      <td>729.632812</td>\n",
       "      <td>677.875000</td>\n",
       "      <td>5.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1903</th>\n",
       "      <td>fefc39b30cc29ec1a80954d26e0550bf864455f7</td>\n",
       "      <td>jun</td>\n",
       "      <td>2021</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>Raleigh</td>\n",
       "      <td>703.003068</td>\n",
       "      <td>700.633473</td>\n",
       "      <td>700.633473</td>\n",
       "      <td>9.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1904</th>\n",
       "      <td>ff14f7a9fbb7df32781e13d339c125fd7d6b8d4d</td>\n",
       "      <td>jun</td>\n",
       "      <td>2021</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>Charlotte</td>\n",
       "      <td>714.161491</td>\n",
       "      <td>705.863354</td>\n",
       "      <td>705.863354</td>\n",
       "      <td>50.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1905</th>\n",
       "      <td>ff53874bcab6f2d086a50ca0766bcadebd61e62e</td>\n",
       "      <td>jun</td>\n",
       "      <td>2021</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>Matthews</td>\n",
       "      <td>728.871429</td>\n",
       "      <td>721.828571</td>\n",
       "      <td>721.157143</td>\n",
       "      <td>40.0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1906</th>\n",
       "      <td>ff62cf17a37a458b09c22b255b1317cf85257110</td>\n",
       "      <td>jun</td>\n",
       "      <td>2021</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>Wake Forest</td>\n",
       "      <td>747.920245</td>\n",
       "      <td>748.024540</td>\n",
       "      <td>747.963190</td>\n",
       "      <td>5.0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1907</th>\n",
       "      <td>ffc40b303849f7f5af77972db456dc63e2eebf34</td>\n",
       "      <td>jun</td>\n",
       "      <td>2021</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>Charlotte</td>\n",
       "      <td>775.270950</td>\n",
       "      <td>776.187151</td>\n",
       "      <td>776.145251</td>\n",
       "      <td>17.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>13251 rows × 13 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                    Identifier Month  Year HvacMode  \\\n",
       "0     0010db271149fae01fa527e2c2105074d5771e5f   jun  2017     cool   \n",
       "1     0010db271149fae01fa527e2c2105074d5771e5f   jun  2017     cool   \n",
       "2     004dfca396d696b8a7ce7dd71862b78384f55976   jun  2017     cool   \n",
       "3     00bbbdb5c3b6cb8b9e78191017ecdb97dad87e47   jun  2017     auto   \n",
       "4     00bbbdb5c3b6cb8b9e78191017ecdb97dad87e47   jun  2017     auto   \n",
       "...                                        ...   ...   ...      ...   \n",
       "1903  fefc39b30cc29ec1a80954d26e0550bf864455f7   jun  2021     cool   \n",
       "1904  ff14f7a9fbb7df32781e13d339c125fd7d6b8d4d   jun  2021     cool   \n",
       "1905  ff53874bcab6f2d086a50ca0766bcadebd61e62e   jun  2021     cool   \n",
       "1906  ff62cf17a37a458b09c22b255b1317cf85257110   jun  2021     cool   \n",
       "1907  ffc40b303849f7f5af77972db456dc63e2eebf34   jun  2021     cool   \n",
       "\n",
       "     CalendarEvent          City  Temperature_ctrl_ave  \\\n",
       "0             auto       Raleigh            743.740741   \n",
       "1             hold       Raleigh            738.705882   \n",
       "2             auto     Charlotte            785.607143   \n",
       "3             auto  Jacksonville            749.244755   \n",
       "4             hold  Jacksonville            737.281250   \n",
       "...            ...           ...                   ...   \n",
       "1903          hold       Raleigh            703.003068   \n",
       "1904          hold     Charlotte            714.161491   \n",
       "1905          hold      Matthews            728.871429   \n",
       "1906          hold   Wake Forest            747.920245   \n",
       "1907          hold     Charlotte            775.270950   \n",
       "\n",
       "      TemperatureExpectedCool_ave  TemperatureExpectedHeat_ave  \\\n",
       "0                      760.000000                   690.000000   \n",
       "1                      740.000000                   740.000000   \n",
       "2                      804.000000                   650.392857   \n",
       "3                      756.865385                   687.638112   \n",
       "4                      729.632812                   677.875000   \n",
       "...                           ...                          ...   \n",
       "1903                   700.633473                   700.633473   \n",
       "1904                   705.863354                   705.863354   \n",
       "1905                   721.828571                   721.157143   \n",
       "1906                   748.024540                   747.963190   \n",
       "1907                   776.187151                   776.145251   \n",
       "\n",
       "      Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \n",
       "0                    37.0              True         False             True  \n",
       "1                    37.0              True         False             True  \n",
       "2                    85.0              True         False             True  \n",
       "3                     5.0             False         False             True  \n",
       "4                     5.0             False         False             True  \n",
       "...                   ...               ...           ...              ...  \n",
       "1903                  9.0             False         False            False  \n",
       "1904                 50.0             False         False             True  \n",
       "1905                 40.0              True         False             True  \n",
       "1906                  5.0              True         False             True  \n",
       "1907                 17.0             False         False            False  \n",
       "\n",
       "[13251 rows x 13 columns]"
      ]
     },
     "execution_count": 99,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# https://stackoverflow.com/questions/63886787/how-to-create-a-dataframe-from-multiple-csv-files\n",
    "# Initialize an empty dataframe\n",
    "NC_jun = pd.DataFrame()\n",
    "\n",
    "# Iterate through files and contents, then concatenate the data into the dataframe\n",
    "for file in files:\n",
    "    df = pd.read_csv(\"data/day/NC/jun/\" + file)\n",
    "    NC_jun = pd.concat([NC_jun, df])\n",
    "    \n",
    "NC_jun"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "metadata": {},
   "outputs": [],
   "source": [
    "NC_jun.to_csv(\"Scraper_Output/State_Month_Day/NC/NC_jun.csv\", header=True, index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "\n",
    "## July"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2017 July Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "jul_2017 = pd.read_csv(\"../data_large/NC-day/2017-jul-day-NC.csv\")\n",
    "\n",
    "# jul_2017"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>377de51444c02381e69ac664b095e71ff80cc9bd</td>\n",
       "      <td>2017-07-16 14:40:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>727</td>\n",
       "      <td>730</td>\n",
       "      <td>680</td>\n",
       "      <td>NC</td>\n",
       "      <td>Statesville</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>bf6bbf1aa5e0e28a5770fe1e246e7ae42e08f6ca</td>\n",
       "      <td>2017-07-06 12:20:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>791</td>\n",
       "      <td>800</td>\n",
       "      <td>650</td>\n",
       "      <td>NC</td>\n",
       "      <td>Thomasville</td>\n",
       "      <td>0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>5de3dd3420b4f41182f7fa754b594df9ac5c1754</td>\n",
       "      <td>2017-07-11 12:40:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>728</td>\n",
       "      <td>730</td>\n",
       "      <td>730</td>\n",
       "      <td>NC</td>\n",
       "      <td>Carolina Shores</td>\n",
       "      <td>0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>acb94b7491948bf4e84c64f34a780e18db02490c</td>\n",
       "      <td>2017-07-08 19:55:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>785</td>\n",
       "      <td>780</td>\n",
       "      <td>724</td>\n",
       "      <td>NC</td>\n",
       "      <td>Kill Devil Hills</td>\n",
       "      <td>50</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>97dab1e68e4b4f8382294010806244f96d3c4b74</td>\n",
       "      <td>2017-07-19 15:10:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>729</td>\n",
       "      <td>735</td>\n",
       "      <td>735</td>\n",
       "      <td>NC</td>\n",
       "      <td>Hampstead</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1050853</th>\n",
       "      <td>4f60b83cc51d90e13839e112d06909cbb5ba4273</td>\n",
       "      <td>2017-07-13 16:15:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>731</td>\n",
       "      <td>730</td>\n",
       "      <td>730</td>\n",
       "      <td>NC</td>\n",
       "      <td>Ocean Isle Beach</td>\n",
       "      <td>15</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1050854</th>\n",
       "      <td>56d7ecc23c9321f7ab2eeaec28e9cc8303992698</td>\n",
       "      <td>2017-07-12 14:55:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>774</td>\n",
       "      <td>770</td>\n",
       "      <td>690</td>\n",
       "      <td>NC</td>\n",
       "      <td>Ocean Isle Beach</td>\n",
       "      <td>16</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1050855</th>\n",
       "      <td>5e8295fd5a96376af08b9f92fd14c1524e0cb70f</td>\n",
       "      <td>2017-07-14 11:05:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>768</td>\n",
       "      <td>760</td>\n",
       "      <td>690</td>\n",
       "      <td>NC</td>\n",
       "      <td>Ocean Isle Beach</td>\n",
       "      <td>16</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1050856</th>\n",
       "      <td>92de2058ee016c38e6488ba9086868768d604156</td>\n",
       "      <td>2017-07-11 18:10:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>753</td>\n",
       "      <td>730</td>\n",
       "      <td>730</td>\n",
       "      <td>NC</td>\n",
       "      <td>Ocean Isle Beach</td>\n",
       "      <td>10</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1050857</th>\n",
       "      <td>56d7ecc23c9321f7ab2eeaec28e9cc8303992698</td>\n",
       "      <td>2017-07-09 14:55:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>761</td>\n",
       "      <td>760</td>\n",
       "      <td>690</td>\n",
       "      <td>NC</td>\n",
       "      <td>Ocean Isle Beach</td>\n",
       "      <td>16</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1011179 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                       Identifier                date_time  \\\n",
       "0        377de51444c02381e69ac664b095e71ff80cc9bd  2017-07-16 14:40:00 UTC   \n",
       "1        bf6bbf1aa5e0e28a5770fe1e246e7ae42e08f6ca  2017-07-06 12:20:00 UTC   \n",
       "2        5de3dd3420b4f41182f7fa754b594df9ac5c1754  2017-07-11 12:40:00 UTC   \n",
       "3        acb94b7491948bf4e84c64f34a780e18db02490c  2017-07-08 19:55:00 UTC   \n",
       "4        97dab1e68e4b4f8382294010806244f96d3c4b74  2017-07-19 15:10:00 UTC   \n",
       "...                                           ...                      ...   \n",
       "1050853  4f60b83cc51d90e13839e112d06909cbb5ba4273  2017-07-13 16:15:00 UTC   \n",
       "1050854  56d7ecc23c9321f7ab2eeaec28e9cc8303992698  2017-07-12 14:55:00 UTC   \n",
       "1050855  5e8295fd5a96376af08b9f92fd14c1524e0cb70f  2017-07-14 11:05:00 UTC   \n",
       "1050856  92de2058ee016c38e6488ba9086868768d604156  2017-07-11 18:10:00 UTC   \n",
       "1050857  56d7ecc23c9321f7ab2eeaec28e9cc8303992698  2017-07-09 14:55:00 UTC   \n",
       "\n",
       "        HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0           auto          auto               727                      730   \n",
       "1           auto          auto               791                      800   \n",
       "2           cool          hold               728                      730   \n",
       "3           cool          hold               785                      780   \n",
       "4           cool          hold               729                      735   \n",
       "...          ...           ...               ...                      ...   \n",
       "1050853     cool          auto               731                      730   \n",
       "1050854     cool          hold               774                      770   \n",
       "1050855     auto          hold               768                      760   \n",
       "1050856     cool          hold               753                      730   \n",
       "1050857     auto          hold               761                      760   \n",
       "\n",
       "         TemperatureExpectedHeat ProvinceState              City  \\\n",
       "0                            680            NC       Statesville   \n",
       "1                            650            NC       Thomasville   \n",
       "2                            730            NC   Carolina Shores   \n",
       "3                            724            NC  Kill Devil Hills   \n",
       "4                            735            NC         Hampstead   \n",
       "...                          ...           ...               ...   \n",
       "1050853                      730            NC  Ocean Isle Beach   \n",
       "1050854                      690            NC  Ocean Isle Beach   \n",
       "1050855                      690            NC  Ocean Isle Beach   \n",
       "1050856                      730            NC  Ocean Isle Beach   \n",
       "1050857                      690            NC  Ocean Isle Beach   \n",
       "\n",
       "         Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                          0             False         False            False   \n",
       "1                          0              True         False             True   \n",
       "2                          0              True         False             True   \n",
       "3                         50              True          True             True   \n",
       "4                          5             False         False             True   \n",
       "...                      ...               ...           ...              ...   \n",
       "1050853                   15              True         False             True   \n",
       "1050854                   16              True         False             True   \n",
       "1050855                   16              True         False             True   \n",
       "1050856                   10              True         False             True   \n",
       "1050857                   16              True         False             True   \n",
       "\n",
       "        Auxilliary_Heat_Fuel_Type  \n",
       "0                             Gas  \n",
       "1                        Electric  \n",
       "2                        Electric  \n",
       "3                        Electric  \n",
       "4                        Electric  \n",
       "...                           ...  \n",
       "1050853                  Electric  \n",
       "1050854                  Electric  \n",
       "1050855                  Electric  \n",
       "1050856                  Electric  \n",
       "1050857                  Electric  \n",
       "\n",
       "[1011179 rows x 14 columns]"
      ]
     },
     "execution_count": 102,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "jul_2017.drop(jul_2017[jul_2017['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "jul_2017.drop(jul_2017[jul_2017['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "jul_2017.drop(jul_2017[jul_2017['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "jul_2017.drop(jul_2017[jul_2017['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "jul_2017"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "jul_2017[\"Year\"] = \"2017\"\n",
    "jul_2017[\"Month\"] = \"jul\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "jul_2017 = jul_2017.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "jul_2017_ave = jul_2017.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# jul_2017_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "jul_2017_ave.to_csv(\"data/day/NC/jul/jul_2017_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2018 July Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "jul_2018 = pd.read_csv(\"../data_large/NC-day/2018-jul-day-NC.csv\")\n",
    "\n",
    "# jul_2018"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 108,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>9d10052cfc1673dc8ff1c530f764d82e0d6347a8</td>\n",
       "      <td>2018-07-02 18:55:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>727</td>\n",
       "      <td>724</td>\n",
       "      <td>716</td>\n",
       "      <td>NC</td>\n",
       "      <td>Durham</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>e915fff9e4bcf1725640b9c28405cedcc73e4396</td>\n",
       "      <td>2018-07-30 14:50:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>702</td>\n",
       "      <td>695</td>\n",
       "      <td>695</td>\n",
       "      <td>NC</td>\n",
       "      <td>Charlotte</td>\n",
       "      <td>50</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>bcd6083ce58d796f10487861799fef4e291f1a74</td>\n",
       "      <td>2018-07-27 10:45:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>704</td>\n",
       "      <td>705</td>\n",
       "      <td>705</td>\n",
       "      <td>NC</td>\n",
       "      <td>Old Fort</td>\n",
       "      <td>15</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>391de5e847cad47bf40a61b582cd1f2c7fa77404</td>\n",
       "      <td>2018-07-14 08:55:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>720</td>\n",
       "      <td>716</td>\n",
       "      <td>716</td>\n",
       "      <td>NC</td>\n",
       "      <td>Charlotte</td>\n",
       "      <td>60</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>16c5fe0fc0ae92e7a5bdaa319a00388524d85888</td>\n",
       "      <td>2018-07-17 11:10:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>756</td>\n",
       "      <td>738</td>\n",
       "      <td>628</td>\n",
       "      <td>NC</td>\n",
       "      <td>Cornelius</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2161411</th>\n",
       "      <td>1983cdbfb4b9ae195604af6eba9ab952f8f31b00</td>\n",
       "      <td>2018-07-06 11:55:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>756</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>NC</td>\n",
       "      <td>Huntersville</td>\n",
       "      <td>15</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2161412</th>\n",
       "      <td>c95ea551d5467443b6ad0c4dd344015ac57c4f5a</td>\n",
       "      <td>2018-07-09 19:20:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>761</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>NC</td>\n",
       "      <td>Wrightsville Beach</td>\n",
       "      <td>0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2161413</th>\n",
       "      <td>fd6616193097062f46d7bd2a972def83a8618a64</td>\n",
       "      <td>2018-07-31 15:55:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>758</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>NC</td>\n",
       "      <td>Durham</td>\n",
       "      <td>10</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2161414</th>\n",
       "      <td>ec6f00e73c1d00f4a4d9d94c16cd97354ae0da2d</td>\n",
       "      <td>2018-07-01 17:40:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>750</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>NC</td>\n",
       "      <td>Waxhaw</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2161415</th>\n",
       "      <td>4eb42daa3315537d9ca6b7b34b86225211189865</td>\n",
       "      <td>2018-07-03 18:40:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>NC</td>\n",
       "      <td>Asheboro</td>\n",
       "      <td>10</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2042388 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                       Identifier                date_time  \\\n",
       "1        9d10052cfc1673dc8ff1c530f764d82e0d6347a8  2018-07-02 18:55:00 UTC   \n",
       "2        e915fff9e4bcf1725640b9c28405cedcc73e4396  2018-07-30 14:50:00 UTC   \n",
       "3        bcd6083ce58d796f10487861799fef4e291f1a74  2018-07-27 10:45:00 UTC   \n",
       "4        391de5e847cad47bf40a61b582cd1f2c7fa77404  2018-07-14 08:55:00 UTC   \n",
       "5        16c5fe0fc0ae92e7a5bdaa319a00388524d85888  2018-07-17 11:10:00 UTC   \n",
       "...                                           ...                      ...   \n",
       "2161411  1983cdbfb4b9ae195604af6eba9ab952f8f31b00  2018-07-06 11:55:00 UTC   \n",
       "2161412  c95ea551d5467443b6ad0c4dd344015ac57c4f5a  2018-07-09 19:20:00 UTC   \n",
       "2161413  fd6616193097062f46d7bd2a972def83a8618a64  2018-07-31 15:55:00 UTC   \n",
       "2161414  ec6f00e73c1d00f4a4d9d94c16cd97354ae0da2d  2018-07-01 17:40:00 UTC   \n",
       "2161415  4eb42daa3315537d9ca6b7b34b86225211189865  2018-07-03 18:40:00 UTC   \n",
       "\n",
       "        HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "1           cool          hold               727                      724   \n",
       "2           cool          hold               702                      695   \n",
       "3           cool          hold               704                      705   \n",
       "4           cool          auto               720                      716   \n",
       "5           cool          auto               756                      738   \n",
       "...          ...           ...               ...                      ...   \n",
       "2161411     cool          hold               756                      760   \n",
       "2161412     cool          auto               761                      760   \n",
       "2161413     cool          auto               758                      760   \n",
       "2161414     cool          auto               750                      760   \n",
       "2161415     cool          hold               760                      760   \n",
       "\n",
       "         TemperatureExpectedHeat ProvinceState                City  \\\n",
       "1                            716            NC              Durham   \n",
       "2                            695            NC           Charlotte   \n",
       "3                            705            NC            Old Fort   \n",
       "4                            716            NC           Charlotte   \n",
       "5                            628            NC           Cornelius   \n",
       "...                          ...           ...                 ...   \n",
       "2161411                      760            NC        Huntersville   \n",
       "2161412                      760            NC  Wrightsville Beach   \n",
       "2161413                      760            NC              Durham   \n",
       "2161414                      760            NC              Waxhaw   \n",
       "2161415                      760            NC            Asheboro   \n",
       "\n",
       "         Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "1                         10             False         False            False   \n",
       "2                         50             False         False            False   \n",
       "3                         15             False         False             True   \n",
       "4                         60              True         False            False   \n",
       "5                          5             False         False            False   \n",
       "...                      ...               ...           ...              ...   \n",
       "2161411                   15             False         False            False   \n",
       "2161412                    0              True         False             True   \n",
       "2161413                   10              True         False             True   \n",
       "2161414                    0             False         False            False   \n",
       "2161415                   10              True         False             True   \n",
       "\n",
       "        Auxilliary_Heat_Fuel_Type  \n",
       "1                             Gas  \n",
       "2                             Gas  \n",
       "3                        Electric  \n",
       "4                             Gas  \n",
       "5                             Gas  \n",
       "...                           ...  \n",
       "2161411                       Gas  \n",
       "2161412                  Electric  \n",
       "2161413                  Electric  \n",
       "2161414                       Gas  \n",
       "2161415                  Electric  \n",
       "\n",
       "[2042388 rows x 14 columns]"
      ]
     },
     "execution_count": 108,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "jul_2018.drop(jul_2018[jul_2018['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "jul_2018.drop(jul_2018[jul_2018['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "jul_2018.drop(jul_2018[jul_2018['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "jul_2018.drop(jul_2018[jul_2018['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "jul_2018"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "jul_2018[\"Year\"] = \"2018\"\n",
    "jul_2018[\"Month\"] = \"jul\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "jul_2018 = jul_2018.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 111,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "jul_2018_ave = jul_2018.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# jul_2018_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 112,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "jul_2018_ave.to_csv(\"data/day/NC/jul/jul_2018_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2019 July Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 113,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "jul_2019 = pd.read_csv(\"../data_large/NC-day/2019-jul-day-NC.csv\")\n",
    "\n",
    "# jul_2019"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 114,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>03258b6f32bb89fc0f8ae00ef05eeaa32cdbaf19</td>\n",
       "      <td>2019-07-02 12:40:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>685</td>\n",
       "      <td>685</td>\n",
       "      <td>685</td>\n",
       "      <td>NC</td>\n",
       "      <td>Charlotte</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>06ba41ea55e519411f46fde6c4eb41ab60eb0387</td>\n",
       "      <td>2019-07-20 19:55:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>759</td>\n",
       "      <td>745</td>\n",
       "      <td>675</td>\n",
       "      <td>NC</td>\n",
       "      <td>Charlotte</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>939a28657261e2936974bb67df922b7b099d926e</td>\n",
       "      <td>2019-07-06 19:45:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>766</td>\n",
       "      <td>750</td>\n",
       "      <td>712</td>\n",
       "      <td>NC</td>\n",
       "      <td>Raleigh</td>\n",
       "      <td>59</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>d9872f9bb4de09707398a70cfd5e7ff87507e994</td>\n",
       "      <td>2019-07-29 11:15:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>729</td>\n",
       "      <td>735</td>\n",
       "      <td>645</td>\n",
       "      <td>NC</td>\n",
       "      <td>Cary</td>\n",
       "      <td>0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>c496438f59de27180d4b6b4b16b41d711a4f0660</td>\n",
       "      <td>2019-07-25 14:20:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>744</td>\n",
       "      <td>745</td>\n",
       "      <td>745</td>\n",
       "      <td>NC</td>\n",
       "      <td>Charlotte</td>\n",
       "      <td>20</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3105942</th>\n",
       "      <td>b339a917fe80c8c20571231413ae40578b82557d</td>\n",
       "      <td>2019-07-07 16:35:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>755</td>\n",
       "      <td>750</td>\n",
       "      <td>760</td>\n",
       "      <td>NC</td>\n",
       "      <td>Cary</td>\n",
       "      <td>20</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3105943</th>\n",
       "      <td>92de2058ee016c38e6488ba9086868768d604156</td>\n",
       "      <td>2019-07-05 19:15:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>759</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>NC</td>\n",
       "      <td>Ocean Isle Beach</td>\n",
       "      <td>10</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3105944</th>\n",
       "      <td>482c86c8ebb84aa6b36cd479ff93da6e634e84a7</td>\n",
       "      <td>2019-07-17 17:25:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>766</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>NC</td>\n",
       "      <td>Hillsborough</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3105945</th>\n",
       "      <td>e9e9e08dfcb7f1c46c8b52b7e7490bd7161ea183</td>\n",
       "      <td>2019-07-11 19:35:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>763</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>NC</td>\n",
       "      <td>Raleigh</td>\n",
       "      <td>30</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3105946</th>\n",
       "      <td>d6a4ef60c7bd4edb7f383da3429cb85a0a5e6af7</td>\n",
       "      <td>2019-07-26 12:25:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>752</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>NC</td>\n",
       "      <td>Leland</td>\n",
       "      <td>0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2996383 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                       Identifier                date_time  \\\n",
       "0        03258b6f32bb89fc0f8ae00ef05eeaa32cdbaf19  2019-07-02 12:40:00 UTC   \n",
       "1        06ba41ea55e519411f46fde6c4eb41ab60eb0387  2019-07-20 19:55:00 UTC   \n",
       "2        939a28657261e2936974bb67df922b7b099d926e  2019-07-06 19:45:00 UTC   \n",
       "3        d9872f9bb4de09707398a70cfd5e7ff87507e994  2019-07-29 11:15:00 UTC   \n",
       "4        c496438f59de27180d4b6b4b16b41d711a4f0660  2019-07-25 14:20:00 UTC   \n",
       "...                                           ...                      ...   \n",
       "3105942  b339a917fe80c8c20571231413ae40578b82557d  2019-07-07 16:35:00 UTC   \n",
       "3105943  92de2058ee016c38e6488ba9086868768d604156  2019-07-05 19:15:00 UTC   \n",
       "3105944  482c86c8ebb84aa6b36cd479ff93da6e634e84a7  2019-07-17 17:25:00 UTC   \n",
       "3105945  e9e9e08dfcb7f1c46c8b52b7e7490bd7161ea183  2019-07-11 19:35:00 UTC   \n",
       "3105946  d6a4ef60c7bd4edb7f383da3429cb85a0a5e6af7  2019-07-26 12:25:00 UTC   \n",
       "\n",
       "        HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0           cool          hold               685                      685   \n",
       "1           auto          hold               759                      745   \n",
       "2           cool          hold               766                      750   \n",
       "3           auto          hold               729                      735   \n",
       "4           cool          hold               744                      745   \n",
       "...          ...           ...               ...                      ...   \n",
       "3105942     cool          auto               755                      750   \n",
       "3105943     cool          hold               759                      760   \n",
       "3105944     cool          hold               766                      760   \n",
       "3105945     cool          hold               763                      760   \n",
       "3105946     cool          hold               752                      760   \n",
       "\n",
       "         TemperatureExpectedHeat ProvinceState              City  \\\n",
       "0                            685            NC         Charlotte   \n",
       "1                            675            NC         Charlotte   \n",
       "2                            712            NC           Raleigh   \n",
       "3                            645            NC              Cary   \n",
       "4                            745            NC         Charlotte   \n",
       "...                          ...           ...               ...   \n",
       "3105942                      760            NC              Cary   \n",
       "3105943                      760            NC  Ocean Isle Beach   \n",
       "3105944                      760            NC      Hillsborough   \n",
       "3105945                      760            NC           Raleigh   \n",
       "3105946                      760            NC            Leland   \n",
       "\n",
       "         Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                          0             False         False            False   \n",
       "1                         10             False         False            False   \n",
       "2                         59             False         False            False   \n",
       "3                          0              True         False             True   \n",
       "4                         20             False         False            False   \n",
       "...                      ...               ...           ...              ...   \n",
       "3105942                   20             False         False            False   \n",
       "3105943                   10              True         False             True   \n",
       "3105944                   10             False         False             True   \n",
       "3105945                   30              True         False             True   \n",
       "3105946                    0              True         False            False   \n",
       "\n",
       "        Auxilliary_Heat_Fuel_Type  \n",
       "0                             Gas  \n",
       "1                             Gas  \n",
       "2                             Gas  \n",
       "3                        Electric  \n",
       "4                             Gas  \n",
       "...                           ...  \n",
       "3105942                       Gas  \n",
       "3105943                  Electric  \n",
       "3105944                  Electric  \n",
       "3105945                  Electric  \n",
       "3105946                       Gas  \n",
       "\n",
       "[2996383 rows x 14 columns]"
      ]
     },
     "execution_count": 114,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "jul_2019.drop(jul_2019[jul_2019['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "jul_2019.drop(jul_2019[jul_2019['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "jul_2019.drop(jul_2019[jul_2019['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "jul_2019.drop(jul_2019[jul_2019['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "jul_2019"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 115,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "jul_2019[\"Year\"] = \"2019\"\n",
    "jul_2019[\"Month\"] = \"jul\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 116,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "jul_2019 = jul_2019.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 117,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "jul_2019_ave = jul_2019.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# jul_2019_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 118,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "jul_2019_ave.to_csv(\"data/day/NC/jul/jul_2019_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2020 July Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 119,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "jul_2020 = pd.read_csv(\"../data_large/NC-day/2020-jul-day-NC.csv\")\n",
    "\n",
    "# jul_2020"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 120,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>212dbe351fff2ba6e4830d8cd248d0f699e51b78</td>\n",
       "      <td>2020-07-06 19:45:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>691</td>\n",
       "      <td>700</td>\n",
       "      <td>645</td>\n",
       "      <td>NC</td>\n",
       "      <td>Apex</td>\n",
       "      <td>9</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>d9f69ed23eee84247126e161b7dda5dca2c8c4f1</td>\n",
       "      <td>2020-07-04 15:30:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>737</td>\n",
       "      <td>749</td>\n",
       "      <td>749</td>\n",
       "      <td>NC</td>\n",
       "      <td>Charlotte</td>\n",
       "      <td>79</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>d10e54491ae2307339d12c5bed9311228171f497</td>\n",
       "      <td>2020-07-25 13:10:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>772</td>\n",
       "      <td>790</td>\n",
       "      <td>722</td>\n",
       "      <td>NC</td>\n",
       "      <td>Winston-Salem</td>\n",
       "      <td>20</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>008b9fe24b54f090ad27a433c0cb950142d3becb</td>\n",
       "      <td>2020-07-15 14:55:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>752</td>\n",
       "      <td>749</td>\n",
       "      <td>748</td>\n",
       "      <td>NC</td>\n",
       "      <td>Avon</td>\n",
       "      <td>30</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>18cca4d150cef5b337b28a7e3180bd0106bbbd70</td>\n",
       "      <td>2020-07-11 13:35:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>742</td>\n",
       "      <td>735</td>\n",
       "      <td>735</td>\n",
       "      <td>NC</td>\n",
       "      <td>Ocracoke</td>\n",
       "      <td>10</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2790209</th>\n",
       "      <td>8bb7cd4f09c4a1ceb8aa8bcf0cc2deb9dbf2a2db</td>\n",
       "      <td>2020-07-29 12:10:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>764</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>NC</td>\n",
       "      <td>Greensboro</td>\n",
       "      <td>95</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2790210</th>\n",
       "      <td>2f977b4cc55bca0198abee75f9a6ed1be35c95ab</td>\n",
       "      <td>2020-07-31 15:05:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>762</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>NC</td>\n",
       "      <td>Garner</td>\n",
       "      <td>9</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2790211</th>\n",
       "      <td>380233f87525cf8d7e87877191e6ffb8ca58d71e</td>\n",
       "      <td>2020-07-03 16:30:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>756</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>NC</td>\n",
       "      <td>Durham</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2790212</th>\n",
       "      <td>4fca1cbd55522476cf7b4c8a325c4c4296e499b9</td>\n",
       "      <td>2020-07-02 13:50:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>746</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>NC</td>\n",
       "      <td>Durham</td>\n",
       "      <td>0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2790213</th>\n",
       "      <td>6fa22a9bce8e72c6a4b58396a30f4b7ee1d3f2cf</td>\n",
       "      <td>2020-07-13 15:45:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>762</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>NC</td>\n",
       "      <td>Charlotte</td>\n",
       "      <td>27</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2712663 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                       Identifier                date_time  \\\n",
       "0        212dbe351fff2ba6e4830d8cd248d0f699e51b78  2020-07-06 19:45:00 UTC   \n",
       "1        d9f69ed23eee84247126e161b7dda5dca2c8c4f1  2020-07-04 15:30:00 UTC   \n",
       "2        d10e54491ae2307339d12c5bed9311228171f497  2020-07-25 13:10:00 UTC   \n",
       "3        008b9fe24b54f090ad27a433c0cb950142d3becb  2020-07-15 14:55:00 UTC   \n",
       "4        18cca4d150cef5b337b28a7e3180bd0106bbbd70  2020-07-11 13:35:00 UTC   \n",
       "...                                           ...                      ...   \n",
       "2790209  8bb7cd4f09c4a1ceb8aa8bcf0cc2deb9dbf2a2db  2020-07-29 12:10:00 UTC   \n",
       "2790210  2f977b4cc55bca0198abee75f9a6ed1be35c95ab  2020-07-31 15:05:00 UTC   \n",
       "2790211  380233f87525cf8d7e87877191e6ffb8ca58d71e  2020-07-03 16:30:00 UTC   \n",
       "2790212  4fca1cbd55522476cf7b4c8a325c4c4296e499b9  2020-07-02 13:50:00 UTC   \n",
       "2790213  6fa22a9bce8e72c6a4b58396a30f4b7ee1d3f2cf  2020-07-13 15:45:00 UTC   \n",
       "\n",
       "        HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0           cool          hold               691                      700   \n",
       "1           cool          hold               737                      749   \n",
       "2           cool          auto               772                      790   \n",
       "3           cool          hold               752                      749   \n",
       "4           cool          hold               742                      735   \n",
       "...          ...           ...               ...                      ...   \n",
       "2790209     cool          hold               764                      760   \n",
       "2790210     cool          hold               762                      760   \n",
       "2790211     cool          hold               756                      760   \n",
       "2790212     cool          hold               746                      760   \n",
       "2790213     cool          hold               762                      760   \n",
       "\n",
       "         TemperatureExpectedHeat ProvinceState           City  \\\n",
       "0                            645            NC           Apex   \n",
       "1                            749            NC      Charlotte   \n",
       "2                            722            NC  Winston-Salem   \n",
       "3                            748            NC           Avon   \n",
       "4                            735            NC       Ocracoke   \n",
       "...                          ...           ...            ...   \n",
       "2790209                      760            NC     Greensboro   \n",
       "2790210                      760            NC         Garner   \n",
       "2790211                      760            NC         Durham   \n",
       "2790212                      760            NC         Durham   \n",
       "2790213                      760            NC      Charlotte   \n",
       "\n",
       "         Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                          9              True         False             True   \n",
       "1                         79             False         False             True   \n",
       "2                         20              True         False             True   \n",
       "3                         30              True         False             True   \n",
       "4                         10              True         False             True   \n",
       "...                      ...               ...           ...              ...   \n",
       "2790209                   95              True         False             True   \n",
       "2790210                    9              True         False             True   \n",
       "2790211                    5             False         False            False   \n",
       "2790212                    0              True         False            False   \n",
       "2790213                   27             False         False            False   \n",
       "\n",
       "        Auxilliary_Heat_Fuel_Type  \n",
       "0                        Electric  \n",
       "1                        Electric  \n",
       "2                        Electric  \n",
       "3                        Electric  \n",
       "4                        Electric  \n",
       "...                           ...  \n",
       "2790209                  Electric  \n",
       "2790210                  Electric  \n",
       "2790211                       Gas  \n",
       "2790212                       Gas  \n",
       "2790213                       Gas  \n",
       "\n",
       "[2712663 rows x 14 columns]"
      ]
     },
     "execution_count": 120,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "jul_2020.drop(jul_2020[jul_2020['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "jul_2020.drop(jul_2020[jul_2020['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "jul_2020.drop(jul_2020[jul_2020['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "jul_2020.drop(jul_2020[jul_2020['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "jul_2020"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 121,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "jul_2020[\"Year\"] = \"2020\"\n",
    "jul_2020[\"Month\"] = \"jul\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 122,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "jul_2020 = jul_2020.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 123,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "jul_2020_ave = jul_2020.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# jul_2020_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 124,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "jul_2020_ave.to_csv(\"data/day/NC/jul/jul_2020_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2021 July Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 125,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "jul_2021 = pd.read_csv(\"../data_large/NC-day/2021-jul-day-NC.csv\")\n",
    "\n",
    "# jul_2021"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 126,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>40fbd05273121d67ba85b36537e21dc9c784950c</td>\n",
       "      <td>2021-07-21 19:15:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>746</td>\n",
       "      <td>742</td>\n",
       "      <td>682</td>\n",
       "      <td>NC</td>\n",
       "      <td>Raleigh</td>\n",
       "      <td>85</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>5d77b51e42a206d452457c9d3a2250fedf5d71ed</td>\n",
       "      <td>2021-07-10 16:50:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>702</td>\n",
       "      <td>686</td>\n",
       "      <td>686</td>\n",
       "      <td>NC</td>\n",
       "      <td>Weaverville</td>\n",
       "      <td>15</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>f8e7a1463a23641ea0f8d7c96bbe6783c4813f37</td>\n",
       "      <td>2021-07-15 15:40:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>752</td>\n",
       "      <td>761</td>\n",
       "      <td>761</td>\n",
       "      <td>NC</td>\n",
       "      <td>Southwest Raleigh</td>\n",
       "      <td>15</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>fefc39b30cc29ec1a80954d26e0550bf864455f7</td>\n",
       "      <td>2021-07-16 16:50:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>709</td>\n",
       "      <td>706</td>\n",
       "      <td>706</td>\n",
       "      <td>NC</td>\n",
       "      <td>Raleigh</td>\n",
       "      <td>9</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1f77c7ffb64f07c2d1ac3e4343dfe69b75dad533</td>\n",
       "      <td>2021-07-02 13:55:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>731</td>\n",
       "      <td>718</td>\n",
       "      <td>718</td>\n",
       "      <td>NC</td>\n",
       "      <td>Indian Trail</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1596744</th>\n",
       "      <td>826ed7efa89ad0412b598b0be33a65de5b986c2a</td>\n",
       "      <td>2021-07-18 19:20:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>745</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>NC</td>\n",
       "      <td>Durham</td>\n",
       "      <td>20</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1596745</th>\n",
       "      <td>77d1ef2552dea3b8512eb92aa8aaacdc9c6c7fbe</td>\n",
       "      <td>2021-07-29 17:30:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>761</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>NC</td>\n",
       "      <td>Raleigh</td>\n",
       "      <td>29</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1596746</th>\n",
       "      <td>4f60b83cc51d90e13839e112d06909cbb5ba4273</td>\n",
       "      <td>2021-07-18 13:35:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>761</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>NC</td>\n",
       "      <td>Ocean Isle Beach</td>\n",
       "      <td>15</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1596747</th>\n",
       "      <td>3c465be0e0cca2841eeae673c4f4876c0301c7bd</td>\n",
       "      <td>2021-07-09 17:35:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>766</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>NC</td>\n",
       "      <td>Durham</td>\n",
       "      <td>0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1596748</th>\n",
       "      <td>83c0e60b6421dde248a9140e52a0a2d1326dbec5</td>\n",
       "      <td>2021-07-19 16:30:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>752</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>NC</td>\n",
       "      <td>Cullowhee</td>\n",
       "      <td>30</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1539405 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                       Identifier                date_time  \\\n",
       "0        40fbd05273121d67ba85b36537e21dc9c784950c  2021-07-21 19:15:00 UTC   \n",
       "1        5d77b51e42a206d452457c9d3a2250fedf5d71ed  2021-07-10 16:50:00 UTC   \n",
       "2        f8e7a1463a23641ea0f8d7c96bbe6783c4813f37  2021-07-15 15:40:00 UTC   \n",
       "3        fefc39b30cc29ec1a80954d26e0550bf864455f7  2021-07-16 16:50:00 UTC   \n",
       "4        1f77c7ffb64f07c2d1ac3e4343dfe69b75dad533  2021-07-02 13:55:00 UTC   \n",
       "...                                           ...                      ...   \n",
       "1596744  826ed7efa89ad0412b598b0be33a65de5b986c2a  2021-07-18 19:20:00 UTC   \n",
       "1596745  77d1ef2552dea3b8512eb92aa8aaacdc9c6c7fbe  2021-07-29 17:30:00 UTC   \n",
       "1596746  4f60b83cc51d90e13839e112d06909cbb5ba4273  2021-07-18 13:35:00 UTC   \n",
       "1596747  3c465be0e0cca2841eeae673c4f4876c0301c7bd  2021-07-09 17:35:00 UTC   \n",
       "1596748  83c0e60b6421dde248a9140e52a0a2d1326dbec5  2021-07-19 16:30:00 UTC   \n",
       "\n",
       "        HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0           auto          hold               746                      742   \n",
       "1           cool          hold               702                      686   \n",
       "2           cool          hold               752                      761   \n",
       "3           cool          hold               709                      706   \n",
       "4           cool          hold               731                      718   \n",
       "...          ...           ...               ...                      ...   \n",
       "1596744     cool          hold               745                      760   \n",
       "1596745     cool          hold               761                      760   \n",
       "1596746     cool          hold               761                      760   \n",
       "1596747     cool          hold               766                      760   \n",
       "1596748     cool          hold               752                      760   \n",
       "\n",
       "         TemperatureExpectedHeat ProvinceState               City  \\\n",
       "0                            682            NC            Raleigh   \n",
       "1                            686            NC        Weaverville   \n",
       "2                            761            NC  Southwest Raleigh   \n",
       "3                            706            NC            Raleigh   \n",
       "4                            718            NC       Indian Trail   \n",
       "...                          ...           ...                ...   \n",
       "1596744                      760            NC             Durham   \n",
       "1596745                      760            NC            Raleigh   \n",
       "1596746                      760            NC   Ocean Isle Beach   \n",
       "1596747                      760            NC             Durham   \n",
       "1596748                      760            NC          Cullowhee   \n",
       "\n",
       "         Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                         85              True         False             True   \n",
       "1                         15             False         False             True   \n",
       "2                         15             False         False             True   \n",
       "3                          9             False         False            False   \n",
       "4                         10             False         False            False   \n",
       "...                      ...               ...           ...              ...   \n",
       "1596744                   20              True         False            False   \n",
       "1596745                   29              True         False            False   \n",
       "1596746                   15              True         False             True   \n",
       "1596747                    0              True         False             True   \n",
       "1596748                   30             False         False             True   \n",
       "\n",
       "        Auxilliary_Heat_Fuel_Type  \n",
       "0                        Electric  \n",
       "1                        Electric  \n",
       "2                        Electric  \n",
       "3                             Gas  \n",
       "4                             Gas  \n",
       "...                           ...  \n",
       "1596744                       Gas  \n",
       "1596745                       Gas  \n",
       "1596746                  Electric  \n",
       "1596747                  Electric  \n",
       "1596748                  Electric  \n",
       "\n",
       "[1539405 rows x 14 columns]"
      ]
     },
     "execution_count": 126,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "jul_2021.drop(jul_2021[jul_2021['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "jul_2021.drop(jul_2021[jul_2021['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "jul_2021.drop(jul_2021[jul_2021['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "jul_2021.drop(jul_2021[jul_2021['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "\n",
    "jul_2021"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 127,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "jul_2021[\"Year\"] = \"2021\"\n",
    "jul_2021[\"Month\"] = \"jul\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 128,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "jul_2021 = jul_2021.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 129,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "jul_2021_ave = jul_2021.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# jul_2021_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 130,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "jul_2021_ave.to_csv(\"data/day/NC/jul/jul_2021_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "\n",
    "### Combine month CSV Files \n",
    "1. Read in files in folders for each state\n",
    "2. Export as combined CSV"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 131,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create variable for files in directory\n",
    "files = [f for f in os.listdir(\"data/day/NC/jul/\") if f.endswith(\".csv\")]\n",
    "\n",
    "# files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 132,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>Month</th>\n",
       "      <th>Year</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>City</th>\n",
       "      <th>Temperature_ctrl_ave</th>\n",
       "      <th>TemperatureExpectedCool_ave</th>\n",
       "      <th>TemperatureExpectedHeat_ave</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0010db271149fae01fa527e2c2105074d5771e5f</td>\n",
       "      <td>jul</td>\n",
       "      <td>2017</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>Raleigh</td>\n",
       "      <td>799.650746</td>\n",
       "      <td>794.328358</td>\n",
       "      <td>640.000000</td>\n",
       "      <td>37.0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>004dfca396d696b8a7ce7dd71862b78384f55976</td>\n",
       "      <td>jul</td>\n",
       "      <td>2017</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>Charlotte</td>\n",
       "      <td>767.261848</td>\n",
       "      <td>765.283175</td>\n",
       "      <td>764.231043</td>\n",
       "      <td>85.0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>00bbbdb5c3b6cb8b9e78191017ecdb97dad87e47</td>\n",
       "      <td>jul</td>\n",
       "      <td>2017</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>Jacksonville</td>\n",
       "      <td>753.135664</td>\n",
       "      <td>760.220979</td>\n",
       "      <td>681.479720</td>\n",
       "      <td>5.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>00e5498cd188bdf09a9a1a902e6084b200cbbdef</td>\n",
       "      <td>jul</td>\n",
       "      <td>2017</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>Wade</td>\n",
       "      <td>760.682540</td>\n",
       "      <td>762.888889</td>\n",
       "      <td>677.015873</td>\n",
       "      <td>55.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>00eb3069b0d62a9be8ed0bcf0b7ca35860a7b716</td>\n",
       "      <td>jul</td>\n",
       "      <td>2017</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>Mint Hill</td>\n",
       "      <td>730.242574</td>\n",
       "      <td>730.309406</td>\n",
       "      <td>675.846535</td>\n",
       "      <td>40.0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1763</th>\n",
       "      <td>ff47512194799f463f497803bc1f74dd74cb9be0</td>\n",
       "      <td>jul</td>\n",
       "      <td>2021</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>Cary</td>\n",
       "      <td>723.465116</td>\n",
       "      <td>710.000000</td>\n",
       "      <td>710.000000</td>\n",
       "      <td>10.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1764</th>\n",
       "      <td>ff4d61e262c7ef1309d1a5d3fd4e41d0c33b9468</td>\n",
       "      <td>jul</td>\n",
       "      <td>2021</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>Charlotte</td>\n",
       "      <td>749.500000</td>\n",
       "      <td>750.000000</td>\n",
       "      <td>670.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1765</th>\n",
       "      <td>ff53874bcab6f2d086a50ca0766bcadebd61e62e</td>\n",
       "      <td>jul</td>\n",
       "      <td>2021</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>Matthews</td>\n",
       "      <td>726.535032</td>\n",
       "      <td>724.382166</td>\n",
       "      <td>723.777070</td>\n",
       "      <td>40.0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1766</th>\n",
       "      <td>ff62cf17a37a458b09c22b255b1317cf85257110</td>\n",
       "      <td>jul</td>\n",
       "      <td>2021</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>Wake Forest</td>\n",
       "      <td>745.621827</td>\n",
       "      <td>748.652284</td>\n",
       "      <td>748.545685</td>\n",
       "      <td>5.0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1767</th>\n",
       "      <td>ffc40b303849f7f5af77972db456dc63e2eebf34</td>\n",
       "      <td>jul</td>\n",
       "      <td>2021</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>Charlotte</td>\n",
       "      <td>777.968085</td>\n",
       "      <td>777.329787</td>\n",
       "      <td>777.327128</td>\n",
       "      <td>17.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>13112 rows × 13 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                    Identifier Month  Year HvacMode  \\\n",
       "0     0010db271149fae01fa527e2c2105074d5771e5f   jul  2017     cool   \n",
       "1     004dfca396d696b8a7ce7dd71862b78384f55976   jul  2017     cool   \n",
       "2     00bbbdb5c3b6cb8b9e78191017ecdb97dad87e47   jul  2017     auto   \n",
       "3     00e5498cd188bdf09a9a1a902e6084b200cbbdef   jul  2017     auto   \n",
       "4     00eb3069b0d62a9be8ed0bcf0b7ca35860a7b716   jul  2017     auto   \n",
       "...                                        ...   ...   ...      ...   \n",
       "1763  ff47512194799f463f497803bc1f74dd74cb9be0   jul  2021     cool   \n",
       "1764  ff4d61e262c7ef1309d1a5d3fd4e41d0c33b9468   jul  2021     auto   \n",
       "1765  ff53874bcab6f2d086a50ca0766bcadebd61e62e   jul  2021     cool   \n",
       "1766  ff62cf17a37a458b09c22b255b1317cf85257110   jul  2021     cool   \n",
       "1767  ffc40b303849f7f5af77972db456dc63e2eebf34   jul  2021     cool   \n",
       "\n",
       "     CalendarEvent          City  Temperature_ctrl_ave  \\\n",
       "0             auto       Raleigh            799.650746   \n",
       "1             hold     Charlotte            767.261848   \n",
       "2             auto  Jacksonville            753.135664   \n",
       "3             hold          Wade            760.682540   \n",
       "4             auto     Mint Hill            730.242574   \n",
       "...            ...           ...                   ...   \n",
       "1763          hold          Cary            723.465116   \n",
       "1764          hold     Charlotte            749.500000   \n",
       "1765          hold      Matthews            726.535032   \n",
       "1766          hold   Wake Forest            745.621827   \n",
       "1767          hold     Charlotte            777.968085   \n",
       "\n",
       "      TemperatureExpectedCool_ave  TemperatureExpectedHeat_ave  \\\n",
       "0                      794.328358                   640.000000   \n",
       "1                      765.283175                   764.231043   \n",
       "2                      760.220979                   681.479720   \n",
       "3                      762.888889                   677.015873   \n",
       "4                      730.309406                   675.846535   \n",
       "...                           ...                          ...   \n",
       "1763                   710.000000                   710.000000   \n",
       "1764                   750.000000                   670.000000   \n",
       "1765                   724.382166                   723.777070   \n",
       "1766                   748.652284                   748.545685   \n",
       "1767                   777.329787                   777.327128   \n",
       "\n",
       "      Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \n",
       "0                    37.0              True         False             True  \n",
       "1                    85.0              True         False             True  \n",
       "2                     5.0             False         False             True  \n",
       "3                    55.0             False         False            False  \n",
       "4                    40.0              True         False             True  \n",
       "...                   ...               ...           ...              ...  \n",
       "1763                 10.0             False         False             True  \n",
       "1764                  0.0             False         False            False  \n",
       "1765                 40.0              True         False             True  \n",
       "1766                  5.0              True         False             True  \n",
       "1767                 17.0             False         False            False  \n",
       "\n",
       "[13112 rows x 13 columns]"
      ]
     },
     "execution_count": 132,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# https://stackoverflow.com/questions/63886787/how-to-create-a-dataframe-from-multiple-csv-files\n",
    "# Initialize an empty dataframe\n",
    "NC_jul = pd.DataFrame()\n",
    "\n",
    "# Iterate through files and contents, then concatenate the data into the dataframe\n",
    "for file in files:\n",
    "    df = pd.read_csv(\"data/day/NC/jul/\" + file)\n",
    "    NC_jul = pd.concat([NC_jul, df])\n",
    "    \n",
    "NC_jul"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 133,
   "metadata": {},
   "outputs": [],
   "source": [
    "NC_jul.to_csv(\"Scraper_Output/State_Month_Day/NC/NC_jul.csv\", header=True, index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "\n",
    "## August"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2017 August Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 134,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "aug_2017 = pd.read_csv(\"../data_large/NC-day/2017-aug-day-NC.csv\")\n",
    "\n",
    "# aug_2017"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 135,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>a8ad34a1fe061900b1e877f10ede00b29cf6c7aa</td>\n",
       "      <td>2017-08-07 19:30:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>702</td>\n",
       "      <td>700</td>\n",
       "      <td>700</td>\n",
       "      <td>NC</td>\n",
       "      <td>Maiden</td>\n",
       "      <td>5</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>618bd961e1e14c4242ea19aab7166b2def4fc0bc</td>\n",
       "      <td>2017-08-19 13:50:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>780</td>\n",
       "      <td>780</td>\n",
       "      <td>740</td>\n",
       "      <td>NC</td>\n",
       "      <td>Fayetteville</td>\n",
       "      <td>15</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1e4e8ed207839cb1c216c6d6c145d9b2f895ae2f</td>\n",
       "      <td>2017-08-09 11:20:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>745</td>\n",
       "      <td>780</td>\n",
       "      <td>780</td>\n",
       "      <td>NC</td>\n",
       "      <td>Iron Station</td>\n",
       "      <td>0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>d8cb0bf67accca7399caefe566b1a73d86cc5964</td>\n",
       "      <td>2017-08-01 19:05:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>739</td>\n",
       "      <td>740</td>\n",
       "      <td>740</td>\n",
       "      <td>NC</td>\n",
       "      <td>New Bern</td>\n",
       "      <td>15</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>bcb4c4251c7b5a0a1dd895a0adbfefc6a9dea053</td>\n",
       "      <td>2017-08-19 16:45:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>738</td>\n",
       "      <td>740</td>\n",
       "      <td>740</td>\n",
       "      <td>NC</td>\n",
       "      <td>Garner</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1051576</th>\n",
       "      <td>92de2058ee016c38e6488ba9086868768d604156</td>\n",
       "      <td>2017-08-05 18:25:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>784</td>\n",
       "      <td>780</td>\n",
       "      <td>780</td>\n",
       "      <td>NC</td>\n",
       "      <td>Ocean Isle Beach</td>\n",
       "      <td>10</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1051577</th>\n",
       "      <td>4d2964003b2fdf62b47946853a4d5026b4b3dff9</td>\n",
       "      <td>2017-08-12 10:45:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>771</td>\n",
       "      <td>770</td>\n",
       "      <td>680</td>\n",
       "      <td>NC</td>\n",
       "      <td>Ocean Isle Beach</td>\n",
       "      <td>20</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1051578</th>\n",
       "      <td>92de2058ee016c38e6488ba9086868768d604156</td>\n",
       "      <td>2017-08-22 12:00:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>765</td>\n",
       "      <td>770</td>\n",
       "      <td>770</td>\n",
       "      <td>NC</td>\n",
       "      <td>Ocean Isle Beach</td>\n",
       "      <td>10</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1051579</th>\n",
       "      <td>5e8295fd5a96376af08b9f92fd14c1524e0cb70f</td>\n",
       "      <td>2017-08-15 11:10:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>753</td>\n",
       "      <td>750</td>\n",
       "      <td>690</td>\n",
       "      <td>NC</td>\n",
       "      <td>Ocean Isle Beach</td>\n",
       "      <td>16</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1051580</th>\n",
       "      <td>4d2964003b2fdf62b47946853a4d5026b4b3dff9</td>\n",
       "      <td>2017-08-24 19:30:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>781</td>\n",
       "      <td>780</td>\n",
       "      <td>680</td>\n",
       "      <td>NC</td>\n",
       "      <td>Ocean Isle Beach</td>\n",
       "      <td>20</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1004599 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                       Identifier                date_time  \\\n",
       "0        a8ad34a1fe061900b1e877f10ede00b29cf6c7aa  2017-08-07 19:30:00 UTC   \n",
       "1        618bd961e1e14c4242ea19aab7166b2def4fc0bc  2017-08-19 13:50:00 UTC   \n",
       "2        1e4e8ed207839cb1c216c6d6c145d9b2f895ae2f  2017-08-09 11:20:00 UTC   \n",
       "3        d8cb0bf67accca7399caefe566b1a73d86cc5964  2017-08-01 19:05:00 UTC   \n",
       "4        bcb4c4251c7b5a0a1dd895a0adbfefc6a9dea053  2017-08-19 16:45:00 UTC   \n",
       "...                                           ...                      ...   \n",
       "1051576  92de2058ee016c38e6488ba9086868768d604156  2017-08-05 18:25:00 UTC   \n",
       "1051577  4d2964003b2fdf62b47946853a4d5026b4b3dff9  2017-08-12 10:45:00 UTC   \n",
       "1051578  92de2058ee016c38e6488ba9086868768d604156  2017-08-22 12:00:00 UTC   \n",
       "1051579  5e8295fd5a96376af08b9f92fd14c1524e0cb70f  2017-08-15 11:10:00 UTC   \n",
       "1051580  4d2964003b2fdf62b47946853a4d5026b4b3dff9  2017-08-24 19:30:00 UTC   \n",
       "\n",
       "        HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0           cool          auto               702                      700   \n",
       "1           cool          hold               780                      780   \n",
       "2           cool          hold               745                      780   \n",
       "3           cool          hold               739                      740   \n",
       "4           cool          hold               738                      740   \n",
       "...          ...           ...               ...                      ...   \n",
       "1051576     cool          hold               784                      780   \n",
       "1051577     auto          hold               771                      770   \n",
       "1051578     cool          hold               765                      770   \n",
       "1051579     auto          auto               753                      750   \n",
       "1051580     auto          hold               781                      780   \n",
       "\n",
       "         TemperatureExpectedHeat ProvinceState              City  \\\n",
       "0                            700            NC            Maiden   \n",
       "1                            740            NC      Fayetteville   \n",
       "2                            780            NC      Iron Station   \n",
       "3                            740            NC          New Bern   \n",
       "4                            740            NC            Garner   \n",
       "...                          ...           ...               ...   \n",
       "1051576                      780            NC  Ocean Isle Beach   \n",
       "1051577                      680            NC  Ocean Isle Beach   \n",
       "1051578                      770            NC  Ocean Isle Beach   \n",
       "1051579                      690            NC  Ocean Isle Beach   \n",
       "1051580                      680            NC  Ocean Isle Beach   \n",
       "\n",
       "         Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                          5              True         False             True   \n",
       "1                         15             False         False             True   \n",
       "2                          0              True         False             True   \n",
       "3                         15             False         False            False   \n",
       "4                          0             False         False             True   \n",
       "...                      ...               ...           ...              ...   \n",
       "1051576                   10              True         False             True   \n",
       "1051577                   20              True         False             True   \n",
       "1051578                   10              True         False             True   \n",
       "1051579                   16              True         False             True   \n",
       "1051580                   20              True         False             True   \n",
       "\n",
       "        Auxilliary_Heat_Fuel_Type  \n",
       "0                        Electric  \n",
       "1                        Electric  \n",
       "2                        Electric  \n",
       "3                             Gas  \n",
       "4                        Electric  \n",
       "...                           ...  \n",
       "1051576                  Electric  \n",
       "1051577                  Electric  \n",
       "1051578                  Electric  \n",
       "1051579                  Electric  \n",
       "1051580                  Electric  \n",
       "\n",
       "[1004599 rows x 14 columns]"
      ]
     },
     "execution_count": 135,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "aug_2017.drop(aug_2017[aug_2017['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "aug_2017.drop(aug_2017[aug_2017['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "aug_2017.drop(aug_2017[aug_2017['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "aug_2017.drop(aug_2017[aug_2017['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "aug_2017"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 136,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "aug_2017[\"Year\"] = \"2017\"\n",
    "aug_2017[\"Month\"] = \"aug\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 137,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "aug_2017 = aug_2017.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 138,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "aug_2017_ave = aug_2017.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# aug_2017_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 139,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "aug_2017_ave.to_csv(\"data/day/NC/aug/aug_2017_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2018 August Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 140,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "aug_2018 = pd.read_csv(\"../data_large/NC-day/2018-aug-day-NC.csv\")\n",
    "\n",
    "# aug_2018"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 141,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>fd0c2cb66b8eef42560a6d0f475a176f17b147db</td>\n",
       "      <td>2018-08-13 18:30:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>697</td>\n",
       "      <td>675</td>\n",
       "      <td>675</td>\n",
       "      <td>NC</td>\n",
       "      <td>Bolivia</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>6af3255fb19f78bfd4607b3f0c6ab52ee51d0318</td>\n",
       "      <td>2018-08-10 14:00:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>709</td>\n",
       "      <td>735</td>\n",
       "      <td>735</td>\n",
       "      <td>NC</td>\n",
       "      <td>Browns Summit</td>\n",
       "      <td>10</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>5021ad71c92d60727bc1f9e9127133fb107829e9</td>\n",
       "      <td>2018-08-15 11:30:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>714</td>\n",
       "      <td>725</td>\n",
       "      <td>675</td>\n",
       "      <td>NC</td>\n",
       "      <td>Clayton</td>\n",
       "      <td>10</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>5bf3eaf1528843d5fb7ef37aaa1b904b4e991d8b</td>\n",
       "      <td>2018-08-01 16:35:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>745</td>\n",
       "      <td>741</td>\n",
       "      <td>741</td>\n",
       "      <td>NC</td>\n",
       "      <td>Cary</td>\n",
       "      <td>20</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4748f0357a85b54219b0405dc5a9b756a9590eea</td>\n",
       "      <td>2018-08-15 11:40:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>756</td>\n",
       "      <td>755</td>\n",
       "      <td>675</td>\n",
       "      <td>NC</td>\n",
       "      <td>Raleigh</td>\n",
       "      <td>30</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2206046</th>\n",
       "      <td>452a31b4a2e661b2b9ba9f840f50adec0ab0a3af</td>\n",
       "      <td>2018-08-13 12:40:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>765</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>NC</td>\n",
       "      <td>Raeford</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2206047</th>\n",
       "      <td>c6ecbf30915d02bed80af37ada1a4475c6164a45</td>\n",
       "      <td>2018-08-11 13:05:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>753</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>NC</td>\n",
       "      <td>Winston-Salem</td>\n",
       "      <td>25</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2206048</th>\n",
       "      <td>ccfe6b4d039fc678a37e8aaed9afdb1daa875a82</td>\n",
       "      <td>2018-08-28 13:15:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>769</td>\n",
       "      <td>780</td>\n",
       "      <td>760</td>\n",
       "      <td>NC</td>\n",
       "      <td>Durham</td>\n",
       "      <td>45</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2206049</th>\n",
       "      <td>0f145da50f523037b4b8e170e563840a7ba0739a</td>\n",
       "      <td>2018-08-04 11:00:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>769</td>\n",
       "      <td>775</td>\n",
       "      <td>760</td>\n",
       "      <td>NC</td>\n",
       "      <td>Wake Forest</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2206050</th>\n",
       "      <td>70e93e613a2e69c50a13137e42306aab0666b25c</td>\n",
       "      <td>2018-08-26 17:55:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>762</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>NC</td>\n",
       "      <td>Burlington</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2104484 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                       Identifier                date_time  \\\n",
       "0        fd0c2cb66b8eef42560a6d0f475a176f17b147db  2018-08-13 18:30:00 UTC   \n",
       "1        6af3255fb19f78bfd4607b3f0c6ab52ee51d0318  2018-08-10 14:00:00 UTC   \n",
       "2        5021ad71c92d60727bc1f9e9127133fb107829e9  2018-08-15 11:30:00 UTC   \n",
       "3        5bf3eaf1528843d5fb7ef37aaa1b904b4e991d8b  2018-08-01 16:35:00 UTC   \n",
       "4        4748f0357a85b54219b0405dc5a9b756a9590eea  2018-08-15 11:40:00 UTC   \n",
       "...                                           ...                      ...   \n",
       "2206046  452a31b4a2e661b2b9ba9f840f50adec0ab0a3af  2018-08-13 12:40:00 UTC   \n",
       "2206047  c6ecbf30915d02bed80af37ada1a4475c6164a45  2018-08-11 13:05:00 UTC   \n",
       "2206048  ccfe6b4d039fc678a37e8aaed9afdb1daa875a82  2018-08-28 13:15:00 UTC   \n",
       "2206049  0f145da50f523037b4b8e170e563840a7ba0739a  2018-08-04 11:00:00 UTC   \n",
       "2206050  70e93e613a2e69c50a13137e42306aab0666b25c  2018-08-26 17:55:00 UTC   \n",
       "\n",
       "        HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0           cool          hold               697                      675   \n",
       "1           cool          hold               709                      735   \n",
       "2           auto          hold               714                      725   \n",
       "3           cool          hold               745                      741   \n",
       "4           auto          hold               756                      755   \n",
       "...          ...           ...               ...                      ...   \n",
       "2206046     cool          hold               765                      760   \n",
       "2206047     cool          hold               753                      760   \n",
       "2206048     cool          hold               769                      780   \n",
       "2206049     cool          hold               769                      775   \n",
       "2206050     cool          hold               762                      760   \n",
       "\n",
       "         TemperatureExpectedHeat ProvinceState           City  \\\n",
       "0                            675            NC        Bolivia   \n",
       "1                            735            NC  Browns Summit   \n",
       "2                            675            NC        Clayton   \n",
       "3                            741            NC           Cary   \n",
       "4                            675            NC        Raleigh   \n",
       "...                          ...           ...            ...   \n",
       "2206046                      760            NC        Raeford   \n",
       "2206047                      760            NC  Winston-Salem   \n",
       "2206048                      760            NC         Durham   \n",
       "2206049                      760            NC    Wake Forest   \n",
       "2206050                      760            NC     Burlington   \n",
       "\n",
       "         Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                         10             False         False             True   \n",
       "1                         10              True         False             True   \n",
       "2                         10              True         False             True   \n",
       "3                         20             False         False            False   \n",
       "4                         30             False         False            False   \n",
       "...                      ...               ...           ...              ...   \n",
       "2206046                    0             False         False             True   \n",
       "2206047                   25              True         False             True   \n",
       "2206048                   45             False         False            False   \n",
       "2206049                    5             False         False            False   \n",
       "2206050                    0             False         False            False   \n",
       "\n",
       "        Auxilliary_Heat_Fuel_Type  \n",
       "0                        Electric  \n",
       "1                        Electric  \n",
       "2                        Electric  \n",
       "3                             Gas  \n",
       "4                             Gas  \n",
       "...                           ...  \n",
       "2206046                  Electric  \n",
       "2206047                  Electric  \n",
       "2206048                       Gas  \n",
       "2206049                       Gas  \n",
       "2206050                       Gas  \n",
       "\n",
       "[2104484 rows x 14 columns]"
      ]
     },
     "execution_count": 141,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "aug_2018.drop(aug_2018[aug_2018['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "aug_2018.drop(aug_2018[aug_2018['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "aug_2018.drop(aug_2018[aug_2018['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "aug_2018.drop(aug_2018[aug_2018['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "aug_2018"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 142,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "aug_2018[\"Year\"] = \"2018\"\n",
    "aug_2018[\"Month\"] = \"aug\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 143,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "aug_2018 = aug_2018.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 144,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "aug_2018_ave = aug_2018.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# aug_2018_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 145,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "aug_2018_ave.to_csv(\"data/day/NC/aug/aug_2018_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2019 August Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 146,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "aug_2019 = pd.read_csv(\"../data_large/NC-day/2019-aug-day-NC.csv\")\n",
    "\n",
    "# aug_2019"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 147,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>f74ea8f6183abe0a5c87399bb5011b7515b25c7c</td>\n",
       "      <td>2019-08-21 19:10:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>756</td>\n",
       "      <td>755</td>\n",
       "      <td>755</td>\n",
       "      <td>NC</td>\n",
       "      <td>Holly Springs</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2c72aa7a70a7c189e1a142344736aa43624421ef</td>\n",
       "      <td>2019-08-16 12:40:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>698</td>\n",
       "      <td>697</td>\n",
       "      <td>697</td>\n",
       "      <td>NC</td>\n",
       "      <td>North Topsail Beach</td>\n",
       "      <td>10</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>844c79cfaa05770ee857e73cd94ae128cc9e5fe0</td>\n",
       "      <td>2019-08-28 14:25:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>719</td>\n",
       "      <td>717</td>\n",
       "      <td>667</td>\n",
       "      <td>NC</td>\n",
       "      <td>Charlotte</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5be2298c91a26939927b83debb460cf4b83dd554</td>\n",
       "      <td>2019-08-23 13:50:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>773</td>\n",
       "      <td>778</td>\n",
       "      <td>778</td>\n",
       "      <td>NC</td>\n",
       "      <td>Apex</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>7510ea7c3f8a94bc84b0274d86792d5350917dce</td>\n",
       "      <td>2019-08-17 17:30:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>719</td>\n",
       "      <td>720</td>\n",
       "      <td>685</td>\n",
       "      <td>NC</td>\n",
       "      <td>Mebane</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3099466</th>\n",
       "      <td>a95fc8f324c6fca34372f86a2f7c31015351cf68</td>\n",
       "      <td>2019-08-15 08:20:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>753</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>NC</td>\n",
       "      <td>Cary</td>\n",
       "      <td>40</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3099467</th>\n",
       "      <td>4a3a1a6142aa36458838a2bd3aac79c78c81a562</td>\n",
       "      <td>2019-08-08 11:10:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>763</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>NC</td>\n",
       "      <td>Southport</td>\n",
       "      <td>90</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3099468</th>\n",
       "      <td>e10123fad0343c0f5cb665e1fe67cba4f56a20d7</td>\n",
       "      <td>2019-08-15 12:00:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>759</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>NC</td>\n",
       "      <td>fuquay varina</td>\n",
       "      <td>0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3099469</th>\n",
       "      <td>3d925a1c9a798bfa7fd88023a2ebbf3ea08b96f3</td>\n",
       "      <td>2019-08-13 19:20:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>763</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>NC</td>\n",
       "      <td>Weaverville</td>\n",
       "      <td>30</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3099470</th>\n",
       "      <td>e10123fad0343c0f5cb665e1fe67cba4f56a20d7</td>\n",
       "      <td>2019-08-13 16:30:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>765</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>NC</td>\n",
       "      <td>fuquay varina</td>\n",
       "      <td>0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2982882 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                       Identifier                date_time  \\\n",
       "0        f74ea8f6183abe0a5c87399bb5011b7515b25c7c  2019-08-21 19:10:00 UTC   \n",
       "1        2c72aa7a70a7c189e1a142344736aa43624421ef  2019-08-16 12:40:00 UTC   \n",
       "2        844c79cfaa05770ee857e73cd94ae128cc9e5fe0  2019-08-28 14:25:00 UTC   \n",
       "4        5be2298c91a26939927b83debb460cf4b83dd554  2019-08-23 13:50:00 UTC   \n",
       "5        7510ea7c3f8a94bc84b0274d86792d5350917dce  2019-08-17 17:30:00 UTC   \n",
       "...                                           ...                      ...   \n",
       "3099466  a95fc8f324c6fca34372f86a2f7c31015351cf68  2019-08-15 08:20:00 UTC   \n",
       "3099467  4a3a1a6142aa36458838a2bd3aac79c78c81a562  2019-08-08 11:10:00 UTC   \n",
       "3099468  e10123fad0343c0f5cb665e1fe67cba4f56a20d7  2019-08-15 12:00:00 UTC   \n",
       "3099469  3d925a1c9a798bfa7fd88023a2ebbf3ea08b96f3  2019-08-13 19:20:00 UTC   \n",
       "3099470  e10123fad0343c0f5cb665e1fe67cba4f56a20d7  2019-08-13 16:30:00 UTC   \n",
       "\n",
       "        HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0           cool          hold               756                      755   \n",
       "1           cool          hold               698                      697   \n",
       "2           cool          hold               719                      717   \n",
       "4           cool          hold               773                      778   \n",
       "5           cool          auto               719                      720   \n",
       "...          ...           ...               ...                      ...   \n",
       "3099466     cool          hold               753                      760   \n",
       "3099467     cool          hold               763                      760   \n",
       "3099468     cool          auto               759                      760   \n",
       "3099469     cool          hold               763                      760   \n",
       "3099470     cool          auto               765                      760   \n",
       "\n",
       "         TemperatureExpectedHeat ProvinceState                 City  \\\n",
       "0                            755            NC        Holly Springs   \n",
       "1                            697            NC  North Topsail Beach   \n",
       "2                            667            NC            Charlotte   \n",
       "4                            778            NC                 Apex   \n",
       "5                            685            NC               Mebane   \n",
       "...                          ...           ...                  ...   \n",
       "3099466                      760            NC                 Cary   \n",
       "3099467                      760            NC            Southport   \n",
       "3099468                      760            NC        fuquay varina   \n",
       "3099469                      760            NC          Weaverville   \n",
       "3099470                      760            NC        fuquay varina   \n",
       "\n",
       "         Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                          0             False         False            False   \n",
       "1                         10              True          True             True   \n",
       "2                         10             False         False             True   \n",
       "4                         10             False         False            False   \n",
       "5                          5             False         False            False   \n",
       "...                      ...               ...           ...              ...   \n",
       "3099466                   40              True         False             True   \n",
       "3099467                   90              True         False             True   \n",
       "3099468                    0              True         False             True   \n",
       "3099469                   30              True         False             True   \n",
       "3099470                    0              True         False             True   \n",
       "\n",
       "        Auxilliary_Heat_Fuel_Type  \n",
       "0                             Gas  \n",
       "1                        Electric  \n",
       "2                        Electric  \n",
       "4                             Gas  \n",
       "5                             Gas  \n",
       "...                           ...  \n",
       "3099466                  Electric  \n",
       "3099467                  Electric  \n",
       "3099468                  Electric  \n",
       "3099469                  Electric  \n",
       "3099470                  Electric  \n",
       "\n",
       "[2982882 rows x 14 columns]"
      ]
     },
     "execution_count": 147,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "aug_2019.drop(aug_2019[aug_2019['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "aug_2019.drop(aug_2019[aug_2019['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "aug_2019.drop(aug_2019[aug_2019['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "aug_2019.drop(aug_2019[aug_2019['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "aug_2019"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 148,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "aug_2019[\"Year\"] = \"2019\"\n",
    "aug_2019[\"Month\"] = \"aug\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 149,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "aug_2019 = aug_2019.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 150,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "aug_2019_ave = aug_2019.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# aug_2019_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 151,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "aug_2019_ave.to_csv(\"data/day/NC/aug/aug_2019_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2020 August Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 152,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "aug_2020 = pd.read_csv(\"../data_large/NC-day/2020-aug-day-NC.csv\")\n",
    "\n",
    "# aug_2020"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 153,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>984c069d6163513d15be9600db6927a731d45c8e</td>\n",
       "      <td>2020-08-03 15:20:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>735</td>\n",
       "      <td>739</td>\n",
       "      <td>739</td>\n",
       "      <td>NC</td>\n",
       "      <td>Clayton</td>\n",
       "      <td>9</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>6405153f15aff7e6ecdbbddf0096efd745551845</td>\n",
       "      <td>2020-08-02 15:45:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>741</td>\n",
       "      <td>733</td>\n",
       "      <td>683</td>\n",
       "      <td>NC</td>\n",
       "      <td>Southeast Raleigh</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>8b647121b63936d90962aa84f677b96fecab13e2</td>\n",
       "      <td>2020-08-16 15:10:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>765</td>\n",
       "      <td>765</td>\n",
       "      <td>685</td>\n",
       "      <td>NC</td>\n",
       "      <td>Raleigh</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>460049168eac64f3fc535b11d36e02cf5a116d40</td>\n",
       "      <td>2020-08-29 18:05:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>722</td>\n",
       "      <td>738</td>\n",
       "      <td>738</td>\n",
       "      <td>NC</td>\n",
       "      <td>Concord</td>\n",
       "      <td>5</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>3ee21b0b46f8b26618d8a20e2dc3492d95045fd7</td>\n",
       "      <td>2020-08-04 12:55:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>724</td>\n",
       "      <td>715</td>\n",
       "      <td>665</td>\n",
       "      <td>NC</td>\n",
       "      <td>Charlotte</td>\n",
       "      <td>90</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2823647</th>\n",
       "      <td>33461703e2acefd43cf7647919cd81014a653f9d</td>\n",
       "      <td>2020-08-18 16:25:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>768</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>NC</td>\n",
       "      <td>Leland</td>\n",
       "      <td>5</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2823648</th>\n",
       "      <td>eee7c16c9b4aaaa83d9d9ff3ab2fff2d13be4a95</td>\n",
       "      <td>2020-08-03 19:30:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>765</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>NC</td>\n",
       "      <td>Avon</td>\n",
       "      <td>28</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2823649</th>\n",
       "      <td>961816d3a091a132ae7b415041aa65ff922c75f5</td>\n",
       "      <td>2020-08-21 12:45:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>762</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>NC</td>\n",
       "      <td>Wake Forest</td>\n",
       "      <td>19</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2823650</th>\n",
       "      <td>d9f69ed23eee84247126e161b7dda5dca2c8c4f1</td>\n",
       "      <td>2020-08-31 10:50:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>NC</td>\n",
       "      <td>Charlotte</td>\n",
       "      <td>79</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2823651</th>\n",
       "      <td>b387e4f16ef5139f2ed2d54684d469362ff1c60f</td>\n",
       "      <td>2020-08-04 18:55:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>745</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>NC</td>\n",
       "      <td>Charlotte</td>\n",
       "      <td>20</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2731714 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                       Identifier                date_time  \\\n",
       "0        984c069d6163513d15be9600db6927a731d45c8e  2020-08-03 15:20:00 UTC   \n",
       "1        6405153f15aff7e6ecdbbddf0096efd745551845  2020-08-02 15:45:00 UTC   \n",
       "2        8b647121b63936d90962aa84f677b96fecab13e2  2020-08-16 15:10:00 UTC   \n",
       "3        460049168eac64f3fc535b11d36e02cf5a116d40  2020-08-29 18:05:00 UTC   \n",
       "4        3ee21b0b46f8b26618d8a20e2dc3492d95045fd7  2020-08-04 12:55:00 UTC   \n",
       "...                                           ...                      ...   \n",
       "2823647  33461703e2acefd43cf7647919cd81014a653f9d  2020-08-18 16:25:00 UTC   \n",
       "2823648  eee7c16c9b4aaaa83d9d9ff3ab2fff2d13be4a95  2020-08-03 19:30:00 UTC   \n",
       "2823649  961816d3a091a132ae7b415041aa65ff922c75f5  2020-08-21 12:45:00 UTC   \n",
       "2823650  d9f69ed23eee84247126e161b7dda5dca2c8c4f1  2020-08-31 10:50:00 UTC   \n",
       "2823651  b387e4f16ef5139f2ed2d54684d469362ff1c60f  2020-08-04 18:55:00 UTC   \n",
       "\n",
       "        HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0           cool          hold               735                      739   \n",
       "1           auto          auto               741                      733   \n",
       "2           auto          hold               765                      765   \n",
       "3           cool          hold               722                      738   \n",
       "4           auto          hold               724                      715   \n",
       "...          ...           ...               ...                      ...   \n",
       "2823647     cool          hold               768                      760   \n",
       "2823648     cool          hold               765                      760   \n",
       "2823649     cool          hold               762                      760   \n",
       "2823650     cool          hold               760                      760   \n",
       "2823651     cool          hold               745                      760   \n",
       "\n",
       "         TemperatureExpectedHeat ProvinceState               City  \\\n",
       "0                            739            NC            Clayton   \n",
       "1                            683            NC  Southeast Raleigh   \n",
       "2                            685            NC            Raleigh   \n",
       "3                            738            NC            Concord   \n",
       "4                            665            NC          Charlotte   \n",
       "...                          ...           ...                ...   \n",
       "2823647                      760            NC             Leland   \n",
       "2823648                      760            NC               Avon   \n",
       "2823649                      760            NC        Wake Forest   \n",
       "2823650                      760            NC          Charlotte   \n",
       "2823651                      760            NC          Charlotte   \n",
       "\n",
       "         Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                          9              True         False             True   \n",
       "1                          0             False         False            False   \n",
       "2                          0             False         False            False   \n",
       "3                          5              True         False             True   \n",
       "4                         90             False         False            False   \n",
       "...                      ...               ...           ...              ...   \n",
       "2823647                    5              True         False             True   \n",
       "2823648                   28              True         False             True   \n",
       "2823649                   19             False         False            False   \n",
       "2823650                   79             False         False             True   \n",
       "2823651                   20             False         False            False   \n",
       "\n",
       "        Auxilliary_Heat_Fuel_Type  \n",
       "0                        Electric  \n",
       "1                             Gas  \n",
       "2                             Gas  \n",
       "3                        Electric  \n",
       "4                             Gas  \n",
       "...                           ...  \n",
       "2823647                  Electric  \n",
       "2823648                  Electric  \n",
       "2823649                       Gas  \n",
       "2823650                  Electric  \n",
       "2823651                       Gas  \n",
       "\n",
       "[2731714 rows x 14 columns]"
      ]
     },
     "execution_count": 153,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "aug_2020.drop(aug_2020[aug_2020['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "aug_2020.drop(aug_2020[aug_2020['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "aug_2020.drop(aug_2020[aug_2020['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "aug_2020.drop(aug_2020[aug_2020['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "aug_2020"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 154,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "aug_2020[\"Year\"] = \"2020\"\n",
    "aug_2020[\"Month\"] = \"aug\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 155,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "aug_2020 = aug_2020.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 156,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "aug_2020_ave = aug_2020.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# aug_2020_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 157,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "aug_2020_ave.to_csv(\"data/day/NC/aug/aug_2020_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "scrolled": true
   },
   "source": [
    "---\n",
    "\n",
    "### Combine month CSV Files \n",
    "1. Read in files in folders for each state\n",
    "2. Export as combined CSV"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 158,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create variable for files in directory\n",
    "files = [f for f in os.listdir(\"data/day/NC/aug/\") if f.endswith(\".csv\")]\n",
    "\n",
    "# files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 159,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>Month</th>\n",
       "      <th>Year</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>City</th>\n",
       "      <th>Temperature_ctrl_ave</th>\n",
       "      <th>TemperatureExpectedCool_ave</th>\n",
       "      <th>TemperatureExpectedHeat_ave</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>004dfca396d696b8a7ce7dd71862b78384f55976</td>\n",
       "      <td>aug</td>\n",
       "      <td>2017</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>Charlotte</td>\n",
       "      <td>772.839416</td>\n",
       "      <td>778.583942</td>\n",
       "      <td>773.751825</td>\n",
       "      <td>85.0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>00bbbdb5c3b6cb8b9e78191017ecdb97dad87e47</td>\n",
       "      <td>aug</td>\n",
       "      <td>2017</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>Jacksonville</td>\n",
       "      <td>753.210191</td>\n",
       "      <td>756.821656</td>\n",
       "      <td>693.175159</td>\n",
       "      <td>5.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>00e5498cd188bdf09a9a1a902e6084b200cbbdef</td>\n",
       "      <td>aug</td>\n",
       "      <td>2017</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>Wade</td>\n",
       "      <td>779.000000</td>\n",
       "      <td>785.000000</td>\n",
       "      <td>640.000000</td>\n",
       "      <td>55.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>00e5498cd188bdf09a9a1a902e6084b200cbbdef</td>\n",
       "      <td>aug</td>\n",
       "      <td>2017</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>Wade</td>\n",
       "      <td>757.897436</td>\n",
       "      <td>760.000000</td>\n",
       "      <td>680.000000</td>\n",
       "      <td>55.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>00eb3069b0d62a9be8ed0bcf0b7ca35860a7b716</td>\n",
       "      <td>aug</td>\n",
       "      <td>2017</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>Mint Hill</td>\n",
       "      <td>740.389105</td>\n",
       "      <td>740.280156</td>\n",
       "      <td>679.719844</td>\n",
       "      <td>40.0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3236</th>\n",
       "      <td>ff62cf17a37a458b09c22b255b1317cf85257110</td>\n",
       "      <td>aug</td>\n",
       "      <td>2020</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>Wake Forest</td>\n",
       "      <td>750.974895</td>\n",
       "      <td>750.297071</td>\n",
       "      <td>750.138075</td>\n",
       "      <td>5.0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3237</th>\n",
       "      <td>ff662f2d3b89e4b539e7917a75b87fc605074bcd</td>\n",
       "      <td>aug</td>\n",
       "      <td>2020</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>Cary</td>\n",
       "      <td>744.613158</td>\n",
       "      <td>741.868421</td>\n",
       "      <td>739.700000</td>\n",
       "      <td>10.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3238</th>\n",
       "      <td>ff6a096f122e1d780dbd7448eeaf4b87810ddb33</td>\n",
       "      <td>aug</td>\n",
       "      <td>2020</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>Whitsett</td>\n",
       "      <td>704.312261</td>\n",
       "      <td>700.000000</td>\n",
       "      <td>630.000000</td>\n",
       "      <td>17.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3239</th>\n",
       "      <td>ffc40b303849f7f5af77972db456dc63e2eebf34</td>\n",
       "      <td>aug</td>\n",
       "      <td>2020</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>Charlotte</td>\n",
       "      <td>787.770723</td>\n",
       "      <td>792.215168</td>\n",
       "      <td>781.910053</td>\n",
       "      <td>17.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3240</th>\n",
       "      <td>ffc40b303849f7f5af77972db456dc63e2eebf34</td>\n",
       "      <td>aug</td>\n",
       "      <td>2020</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>Charlotte</td>\n",
       "      <td>786.731438</td>\n",
       "      <td>785.922591</td>\n",
       "      <td>785.559242</td>\n",
       "      <td>17.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>10890 rows × 13 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                    Identifier Month  Year HvacMode  \\\n",
       "0     004dfca396d696b8a7ce7dd71862b78384f55976   aug  2017     cool   \n",
       "1     00bbbdb5c3b6cb8b9e78191017ecdb97dad87e47   aug  2017     auto   \n",
       "2     00e5498cd188bdf09a9a1a902e6084b200cbbdef   aug  2017     auto   \n",
       "3     00e5498cd188bdf09a9a1a902e6084b200cbbdef   aug  2017     auto   \n",
       "4     00eb3069b0d62a9be8ed0bcf0b7ca35860a7b716   aug  2017     auto   \n",
       "...                                        ...   ...   ...      ...   \n",
       "3236  ff62cf17a37a458b09c22b255b1317cf85257110   aug  2020     cool   \n",
       "3237  ff662f2d3b89e4b539e7917a75b87fc605074bcd   aug  2020     cool   \n",
       "3238  ff6a096f122e1d780dbd7448eeaf4b87810ddb33   aug  2020     auto   \n",
       "3239  ffc40b303849f7f5af77972db456dc63e2eebf34   aug  2020     cool   \n",
       "3240  ffc40b303849f7f5af77972db456dc63e2eebf34   aug  2020     cool   \n",
       "\n",
       "     CalendarEvent          City  Temperature_ctrl_ave  \\\n",
       "0             hold     Charlotte            772.839416   \n",
       "1             auto  Jacksonville            753.210191   \n",
       "2             auto          Wade            779.000000   \n",
       "3             hold          Wade            757.897436   \n",
       "4             hold     Mint Hill            740.389105   \n",
       "...            ...           ...                   ...   \n",
       "3236          hold   Wake Forest            750.974895   \n",
       "3237          auto          Cary            744.613158   \n",
       "3238          auto      Whitsett            704.312261   \n",
       "3239          auto     Charlotte            787.770723   \n",
       "3240          hold     Charlotte            786.731438   \n",
       "\n",
       "      TemperatureExpectedCool_ave  TemperatureExpectedHeat_ave  \\\n",
       "0                      778.583942                   773.751825   \n",
       "1                      756.821656                   693.175159   \n",
       "2                      785.000000                   640.000000   \n",
       "3                      760.000000                   680.000000   \n",
       "4                      740.280156                   679.719844   \n",
       "...                           ...                          ...   \n",
       "3236                   750.297071                   750.138075   \n",
       "3237                   741.868421                   739.700000   \n",
       "3238                   700.000000                   630.000000   \n",
       "3239                   792.215168                   781.910053   \n",
       "3240                   785.922591                   785.559242   \n",
       "\n",
       "      Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \n",
       "0                    85.0              True         False             True  \n",
       "1                     5.0             False         False             True  \n",
       "2                    55.0             False         False            False  \n",
       "3                    55.0             False         False            False  \n",
       "4                    40.0              True         False             True  \n",
       "...                   ...               ...           ...              ...  \n",
       "3236                  5.0              True         False             True  \n",
       "3237                 10.0             False         False            False  \n",
       "3238                 17.0             False         False            False  \n",
       "3239                 17.0             False         False            False  \n",
       "3240                 17.0             False         False            False  \n",
       "\n",
       "[10890 rows x 13 columns]"
      ]
     },
     "execution_count": 159,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# https://stackoverflow.com/questions/63886787/how-to-create-a-dataframe-from-multiple-csv-files\n",
    "# Initialize an empty dataframe\n",
    "NC_aug = pd.DataFrame()\n",
    "\n",
    "# Iterate through files and contents, then concatenate the data into the dataframe\n",
    "for file in files:\n",
    "    df = pd.read_csv(\"data/day/NC/aug/\" + file)\n",
    "    NC_aug = pd.concat([NC_aug, df])\n",
    "    \n",
    "NC_aug"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 160,
   "metadata": {},
   "outputs": [],
   "source": [
    "NC_aug.to_csv(\"Scraper_Output/State_Month_Day/NC/NC_aug.csv\", header=True, index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "\n",
    "## September"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "\n",
    "## October"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "\n",
    "## November"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "\n",
    "## December"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2017 December Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 161,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "dec_2017 = pd.read_csv(\"../data_large/NC-day/2017-dec-day-NC.csv\")\n",
    "\n",
    "# dec_2017"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 162,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>c9646006b6f1320b62cd0a489b98d724c8b3afff</td>\n",
       "      <td>2017-12-21 13:40:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>658</td>\n",
       "      <td>665</td>\n",
       "      <td>665</td>\n",
       "      <td>NC</td>\n",
       "      <td>Tarboro</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>b561319e1d44f031114a08344d28812d66c7fc1f</td>\n",
       "      <td>2017-12-31 16:05:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>710</td>\n",
       "      <td>795</td>\n",
       "      <td>715</td>\n",
       "      <td>NC</td>\n",
       "      <td>Charlotte</td>\n",
       "      <td>10</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>e06d742e4fd07cff434746ea4ac38753a70b6d98</td>\n",
       "      <td>2017-12-08 12:40:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>730</td>\n",
       "      <td>734</td>\n",
       "      <td>734</td>\n",
       "      <td>NC</td>\n",
       "      <td>Durham</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>db4ba9b872450ddc981c4d14cd9d4c376860ddb6</td>\n",
       "      <td>2017-12-01 13:40:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>658</td>\n",
       "      <td>655</td>\n",
       "      <td>655</td>\n",
       "      <td>NC</td>\n",
       "      <td>Knightdale</td>\n",
       "      <td>37</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>04dd518bf8f1884f9f669346b66a3f68af1b423f</td>\n",
       "      <td>2017-12-02 13:00:00 UTC</td>\n",
       "      <td>auxHeatOnly</td>\n",
       "      <td>hold</td>\n",
       "      <td>681</td>\n",
       "      <td>685</td>\n",
       "      <td>685</td>\n",
       "      <td>NC</td>\n",
       "      <td>Wallace</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1259784</th>\n",
       "      <td>019b7a54e437581948f495dbf790790468b796bd</td>\n",
       "      <td>2017-12-09 12:15:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>713</td>\n",
       "      <td>765</td>\n",
       "      <td>715</td>\n",
       "      <td>NC</td>\n",
       "      <td>Cornelius</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1259785</th>\n",
       "      <td>aef96dd4f7df0d508193ddb1ca080ac6cb7741e8</td>\n",
       "      <td>2017-12-31 07:30:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>715</td>\n",
       "      <td>765</td>\n",
       "      <td>715</td>\n",
       "      <td>NC</td>\n",
       "      <td>Charlotte</td>\n",
       "      <td>120</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1259786</th>\n",
       "      <td>33008b03223e678c72f11f6501d1ec4889a3c804</td>\n",
       "      <td>2017-12-29 17:25:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>707</td>\n",
       "      <td>765</td>\n",
       "      <td>715</td>\n",
       "      <td>NC</td>\n",
       "      <td>Oak Ridge</td>\n",
       "      <td>20</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1259787</th>\n",
       "      <td>efbfe8cdaed00c264a580ac27aa26bec8cad0d1e</td>\n",
       "      <td>2017-12-26 18:20:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>687</td>\n",
       "      <td>765</td>\n",
       "      <td>705</td>\n",
       "      <td>NC</td>\n",
       "      <td>Wilmington</td>\n",
       "      <td>40</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1259788</th>\n",
       "      <td>4449cb7943e1d648171e270c8e255af4c33d0d3d</td>\n",
       "      <td>2017-12-04 10:40:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>714</td>\n",
       "      <td>765</td>\n",
       "      <td>715</td>\n",
       "      <td>NC</td>\n",
       "      <td>Greensboro</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1208665 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                       Identifier                date_time  \\\n",
       "2        c9646006b6f1320b62cd0a489b98d724c8b3afff  2017-12-21 13:40:00 UTC   \n",
       "3        b561319e1d44f031114a08344d28812d66c7fc1f  2017-12-31 16:05:00 UTC   \n",
       "4        e06d742e4fd07cff434746ea4ac38753a70b6d98  2017-12-08 12:40:00 UTC   \n",
       "5        db4ba9b872450ddc981c4d14cd9d4c376860ddb6  2017-12-01 13:40:00 UTC   \n",
       "6        04dd518bf8f1884f9f669346b66a3f68af1b423f  2017-12-02 13:00:00 UTC   \n",
       "...                                           ...                      ...   \n",
       "1259784  019b7a54e437581948f495dbf790790468b796bd  2017-12-09 12:15:00 UTC   \n",
       "1259785  aef96dd4f7df0d508193ddb1ca080ac6cb7741e8  2017-12-31 07:30:00 UTC   \n",
       "1259786  33008b03223e678c72f11f6501d1ec4889a3c804  2017-12-29 17:25:00 UTC   \n",
       "1259787  efbfe8cdaed00c264a580ac27aa26bec8cad0d1e  2017-12-26 18:20:00 UTC   \n",
       "1259788  4449cb7943e1d648171e270c8e255af4c33d0d3d  2017-12-04 10:40:00 UTC   \n",
       "\n",
       "            HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "2               heat          hold               658                      665   \n",
       "3               auto          hold               710                      795   \n",
       "4               heat          hold               730                      734   \n",
       "5               heat          hold               658                      655   \n",
       "6        auxHeatOnly          hold               681                      685   \n",
       "...              ...           ...               ...                      ...   \n",
       "1259784         auto          hold               713                      765   \n",
       "1259785         auto          auto               715                      765   \n",
       "1259786         auto          hold               707                      765   \n",
       "1259787         auto          hold               687                      765   \n",
       "1259788         auto          auto               714                      765   \n",
       "\n",
       "         TemperatureExpectedHeat ProvinceState        City  \\\n",
       "2                            665            NC     Tarboro   \n",
       "3                            715            NC   Charlotte   \n",
       "4                            734            NC      Durham   \n",
       "5                            655            NC  Knightdale   \n",
       "6                            685            NC     Wallace   \n",
       "...                          ...           ...         ...   \n",
       "1259784                      715            NC   Cornelius   \n",
       "1259785                      715            NC   Charlotte   \n",
       "1259786                      715            NC   Oak Ridge   \n",
       "1259787                      705            NC  Wilmington   \n",
       "1259788                      715            NC  Greensboro   \n",
       "\n",
       "         Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "2                          0             False         False            False   \n",
       "3                         10              True         False             True   \n",
       "4                         10             False         False            False   \n",
       "5                         37             False         False            False   \n",
       "6                          0             False         False             True   \n",
       "...                      ...               ...           ...              ...   \n",
       "1259784                    0             False         False            False   \n",
       "1259785                  120             False         False            False   \n",
       "1259786                   20             False         False            False   \n",
       "1259787                   40              True         False             True   \n",
       "1259788                    0             False         False            False   \n",
       "\n",
       "        Auxilliary_Heat_Fuel_Type  \n",
       "2                             Gas  \n",
       "3                        Electric  \n",
       "4                             Gas  \n",
       "5                             Gas  \n",
       "6                        Electric  \n",
       "...                           ...  \n",
       "1259784                       Gas  \n",
       "1259785                       Gas  \n",
       "1259786                       Gas  \n",
       "1259787                  Electric  \n",
       "1259788                       Gas  \n",
       "\n",
       "[1208665 rows x 14 columns]"
      ]
     },
     "execution_count": 162,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "dec_2017.drop(dec_2017[dec_2017['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "dec_2017.drop(dec_2017[dec_2017['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "dec_2017.drop(dec_2017[dec_2017['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "dec_2017.drop(dec_2017[dec_2017['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "dec_2017"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 163,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "dec_2017[\"Year\"] = \"2017\"\n",
    "dec_2017[\"Month\"] = \"dec\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 164,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "dec_2017 = dec_2017.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 165,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "dec_2017_ave = dec_2017.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# dec_2017_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 166,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "dec_2017_ave.to_csv(\"data/day/NC/dec/dec_2017_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2018 December Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 167,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "dec_2018 = pd.read_csv(\"../data_large/NC-day/2018-dec-day-NC.csv\")\n",
    "\n",
    "# dec_2018"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 168,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>c7c904dbed1d8b4fc219ebaf8a35eb89f2aff27a</td>\n",
       "      <td>2018-12-12 13:55:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>698</td>\n",
       "      <td>756</td>\n",
       "      <td>706</td>\n",
       "      <td>NC</td>\n",
       "      <td>Nags Head</td>\n",
       "      <td>30</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>70e93e613a2e69c50a13137e42306aab0666b25c</td>\n",
       "      <td>2018-12-10 16:10:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>672</td>\n",
       "      <td>669</td>\n",
       "      <td>669</td>\n",
       "      <td>NC</td>\n",
       "      <td>Burlington</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>b64cd692bc8176ee99011be1d3b973a12a098cb3</td>\n",
       "      <td>2018-12-20 19:45:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>728</td>\n",
       "      <td>736</td>\n",
       "      <td>730</td>\n",
       "      <td>NC</td>\n",
       "      <td>Morrisville</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>10ee4a5a08a4be54243ae420bb2b1bb081d94299</td>\n",
       "      <td>2018-12-02 15:30:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>693</td>\n",
       "      <td>695</td>\n",
       "      <td>695</td>\n",
       "      <td>NC</td>\n",
       "      <td>Raleigh</td>\n",
       "      <td>30</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>a3c71856e280b64edc61bcd382ac3543b7754c83</td>\n",
       "      <td>2018-12-26 12:25:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>710</td>\n",
       "      <td>773</td>\n",
       "      <td>713</td>\n",
       "      <td>NC</td>\n",
       "      <td>Rolesville</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2251854</th>\n",
       "      <td>9617aaf92fb7d9d02d9c64749080be6981f28418</td>\n",
       "      <td>2018-12-23 15:45:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>627</td>\n",
       "      <td>760</td>\n",
       "      <td>630</td>\n",
       "      <td>NC</td>\n",
       "      <td>Jackson</td>\n",
       "      <td>37</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2251855</th>\n",
       "      <td>d5c8d11fb2258ef431f0f5bbd3449eb03f8c4400</td>\n",
       "      <td>2018-12-05 12:00:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>716</td>\n",
       "      <td>760</td>\n",
       "      <td>710</td>\n",
       "      <td>NC</td>\n",
       "      <td>Knightdale</td>\n",
       "      <td>37</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2251856</th>\n",
       "      <td>a89043a6db6435ce3f5d2e7800529a67a9bdbccc</td>\n",
       "      <td>2018-12-07 15:15:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>692</td>\n",
       "      <td>760</td>\n",
       "      <td>690</td>\n",
       "      <td>NC</td>\n",
       "      <td>Charlotte</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2251857</th>\n",
       "      <td>cc68e69e5fcfdba14c03428ffdf2e0a9b6d0be9c</td>\n",
       "      <td>2018-12-24 17:05:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>708</td>\n",
       "      <td>760</td>\n",
       "      <td>710</td>\n",
       "      <td>NC</td>\n",
       "      <td>Concord</td>\n",
       "      <td>40</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2251858</th>\n",
       "      <td>31e864a282eea5243339482ba0296c4c6396bc41</td>\n",
       "      <td>2018-12-06 16:00:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>713</td>\n",
       "      <td>760</td>\n",
       "      <td>710</td>\n",
       "      <td>NC</td>\n",
       "      <td>Summerfield</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2164999 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                       Identifier                date_time  \\\n",
       "0        c7c904dbed1d8b4fc219ebaf8a35eb89f2aff27a  2018-12-12 13:55:00 UTC   \n",
       "1        70e93e613a2e69c50a13137e42306aab0666b25c  2018-12-10 16:10:00 UTC   \n",
       "3        b64cd692bc8176ee99011be1d3b973a12a098cb3  2018-12-20 19:45:00 UTC   \n",
       "4        10ee4a5a08a4be54243ae420bb2b1bb081d94299  2018-12-02 15:30:00 UTC   \n",
       "5        a3c71856e280b64edc61bcd382ac3543b7754c83  2018-12-26 12:25:00 UTC   \n",
       "...                                           ...                      ...   \n",
       "2251854  9617aaf92fb7d9d02d9c64749080be6981f28418  2018-12-23 15:45:00 UTC   \n",
       "2251855  d5c8d11fb2258ef431f0f5bbd3449eb03f8c4400  2018-12-05 12:00:00 UTC   \n",
       "2251856  a89043a6db6435ce3f5d2e7800529a67a9bdbccc  2018-12-07 15:15:00 UTC   \n",
       "2251857  cc68e69e5fcfdba14c03428ffdf2e0a9b6d0be9c  2018-12-24 17:05:00 UTC   \n",
       "2251858  31e864a282eea5243339482ba0296c4c6396bc41  2018-12-06 16:00:00 UTC   \n",
       "\n",
       "        HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0           auto          hold               698                      756   \n",
       "1           heat          hold               672                      669   \n",
       "3           heat          hold               728                      736   \n",
       "4           heat          hold               693                      695   \n",
       "5           auto          hold               710                      773   \n",
       "...          ...           ...               ...                      ...   \n",
       "2251854     heat          auto               627                      760   \n",
       "2251855     auto          auto               716                      760   \n",
       "2251856     auto          hold               692                      760   \n",
       "2251857     auto          hold               708                      760   \n",
       "2251858     auto          auto               713                      760   \n",
       "\n",
       "         TemperatureExpectedHeat ProvinceState         City  \\\n",
       "0                            706            NC    Nags Head   \n",
       "1                            669            NC   Burlington   \n",
       "3                            730            NC  Morrisville   \n",
       "4                            695            NC      Raleigh   \n",
       "5                            713            NC   Rolesville   \n",
       "...                          ...           ...          ...   \n",
       "2251854                      630            NC      Jackson   \n",
       "2251855                      710            NC   Knightdale   \n",
       "2251856                      690            NC    Charlotte   \n",
       "2251857                      710            NC      Concord   \n",
       "2251858                      710            NC  Summerfield   \n",
       "\n",
       "         Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                         30              True         False             True   \n",
       "1                          0             False         False            False   \n",
       "3                          0             False         False            False   \n",
       "4                         30              True         False             True   \n",
       "5                          5             False         False            False   \n",
       "...                      ...               ...           ...              ...   \n",
       "2251854                   37              True         False             True   \n",
       "2251855                   37             False         False            False   \n",
       "2251856                   10             False         False             True   \n",
       "2251857                   40              True         False            False   \n",
       "2251858                    0             False         False            False   \n",
       "\n",
       "        Auxilliary_Heat_Fuel_Type  \n",
       "0                        Electric  \n",
       "1                             Gas  \n",
       "3                             Gas  \n",
       "4                        Electric  \n",
       "5                             Gas  \n",
       "...                           ...  \n",
       "2251854                  Electric  \n",
       "2251855                       Gas  \n",
       "2251856                  Electric  \n",
       "2251857                       Gas  \n",
       "2251858                       Gas  \n",
       "\n",
       "[2164999 rows x 14 columns]"
      ]
     },
     "execution_count": 168,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "dec_2018.drop(dec_2018[dec_2018['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "dec_2018.drop(dec_2018[dec_2018['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "dec_2018.drop(dec_2018[dec_2018['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "dec_2018.drop(dec_2018[dec_2018['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "dec_2018"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 169,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "dec_2018[\"Year\"] = \"2018\"\n",
    "dec_2018[\"Month\"] = \"dec\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 170,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "dec_2018 = dec_2018.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 171,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "dec_2018_ave = dec_2018.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# dec_2018_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 172,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "dec_2018_ave.to_csv(\"data/day/NC/dec/dec_2018_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2019 December Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 173,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "dec_2019 = pd.read_csv(\"../data_large/NC-day/2019-dec-day-NC.csv\")\n",
    "\n",
    "# dec_2019"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 174,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>7ded0e4a431fb2473a5eb144b737778016efd464</td>\n",
       "      <td>2019-12-07 14:20:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>697</td>\n",
       "      <td>698</td>\n",
       "      <td>698</td>\n",
       "      <td>NY</td>\n",
       "      <td>New York</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>652b48879974f8e28937dc6fa63132aa1c94acf7</td>\n",
       "      <td>2019-12-08 19:10:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>701</td>\n",
       "      <td>705</td>\n",
       "      <td>705</td>\n",
       "      <td>NY</td>\n",
       "      <td>Huntington</td>\n",
       "      <td>60</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>fc2554dc055755356a49aca66be600073e65cc42</td>\n",
       "      <td>2019-12-08 18:40:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>677</td>\n",
       "      <td>701</td>\n",
       "      <td>680</td>\n",
       "      <td>NY</td>\n",
       "      <td>Ronkonkoma</td>\n",
       "      <td>10</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>c8880d3bf569b5447eea8128d493bdcb3ba5497b</td>\n",
       "      <td>2019-12-05 19:15:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>648</td>\n",
       "      <td>704</td>\n",
       "      <td>653</td>\n",
       "      <td>NY</td>\n",
       "      <td>Manlius</td>\n",
       "      <td>15</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2366159a5da3a846ba06785dcad2434389f75604</td>\n",
       "      <td>2019-12-28 12:05:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>666</td>\n",
       "      <td>662</td>\n",
       "      <td>662</td>\n",
       "      <td>NY</td>\n",
       "      <td>Wappingers Falls</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1861818</th>\n",
       "      <td>cad4a255ea99121256ec2be08cc2c9ca567f6ba2</td>\n",
       "      <td>2019-12-16 15:55:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>NY</td>\n",
       "      <td>Buffalo</td>\n",
       "      <td>95</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1861819</th>\n",
       "      <td>82f1e853c293e5c2ce4cbe2d363a9d6a2878ec57</td>\n",
       "      <td>2019-12-03 11:15:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>666</td>\n",
       "      <td>760</td>\n",
       "      <td>670</td>\n",
       "      <td>NY</td>\n",
       "      <td>Manlius</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1861820</th>\n",
       "      <td>10c28461873adc35141df26bed3208a8bf5c59e5</td>\n",
       "      <td>2019-12-02 19:40:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>755</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>NY</td>\n",
       "      <td>Merrick</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1861821</th>\n",
       "      <td>668d732f50c033bd0592a501120e8e8e46f95408</td>\n",
       "      <td>2019-12-07 12:10:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>719</td>\n",
       "      <td>760</td>\n",
       "      <td>720</td>\n",
       "      <td>NY</td>\n",
       "      <td>Rochester</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1861822</th>\n",
       "      <td>86389e2606606ae52c2b450e4c7e96cb1cda84e2</td>\n",
       "      <td>2019-12-18 13:45:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>660</td>\n",
       "      <td>760</td>\n",
       "      <td>670</td>\n",
       "      <td>NY</td>\n",
       "      <td>Salina</td>\n",
       "      <td>40</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1717428 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                       Identifier                date_time  \\\n",
       "0        7ded0e4a431fb2473a5eb144b737778016efd464  2019-12-07 14:20:00 UTC   \n",
       "1        652b48879974f8e28937dc6fa63132aa1c94acf7  2019-12-08 19:10:00 UTC   \n",
       "2        fc2554dc055755356a49aca66be600073e65cc42  2019-12-08 18:40:00 UTC   \n",
       "3        c8880d3bf569b5447eea8128d493bdcb3ba5497b  2019-12-05 19:15:00 UTC   \n",
       "4        2366159a5da3a846ba06785dcad2434389f75604  2019-12-28 12:05:00 UTC   \n",
       "...                                           ...                      ...   \n",
       "1861818  cad4a255ea99121256ec2be08cc2c9ca567f6ba2  2019-12-16 15:55:00 UTC   \n",
       "1861819  82f1e853c293e5c2ce4cbe2d363a9d6a2878ec57  2019-12-03 11:15:00 UTC   \n",
       "1861820  10c28461873adc35141df26bed3208a8bf5c59e5  2019-12-02 19:40:00 UTC   \n",
       "1861821  668d732f50c033bd0592a501120e8e8e46f95408  2019-12-07 12:10:00 UTC   \n",
       "1861822  86389e2606606ae52c2b450e4c7e96cb1cda84e2  2019-12-18 13:45:00 UTC   \n",
       "\n",
       "        HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0           heat          hold               697                      698   \n",
       "1           heat          hold               701                      705   \n",
       "2           heat          hold               677                      701   \n",
       "3           heat          hold               648                      704   \n",
       "4           heat          hold               666                      662   \n",
       "...          ...           ...               ...                      ...   \n",
       "1861818     heat          hold               760                      760   \n",
       "1861819     auto          hold               666                      760   \n",
       "1861820     heat          auto               755                      760   \n",
       "1861821     heat          auto               719                      760   \n",
       "1861822     auto          hold               660                      760   \n",
       "\n",
       "         TemperatureExpectedHeat ProvinceState              City  \\\n",
       "0                            698            NY          New York   \n",
       "1                            705            NY        Huntington   \n",
       "2                            680            NY        Ronkonkoma   \n",
       "3                            653            NY           Manlius   \n",
       "4                            662            NY  Wappingers Falls   \n",
       "...                          ...           ...               ...   \n",
       "1861818                      760            NY           Buffalo   \n",
       "1861819                      670            NY           Manlius   \n",
       "1861820                      760            NY           Merrick   \n",
       "1861821                      720            NY         Rochester   \n",
       "1861822                      670            NY            Salina   \n",
       "\n",
       "         Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                          5             False         False            False   \n",
       "1                         60             False         False            False   \n",
       "2                         10              True         False            False   \n",
       "3                         15             False         False            False   \n",
       "4                         10             False         False            False   \n",
       "...                      ...               ...           ...              ...   \n",
       "1861818                   95             False         False            False   \n",
       "1861819                    0             False         False            False   \n",
       "1861820                    0             False         False            False   \n",
       "1861821                    0             False         False            False   \n",
       "1861822                   40             False         False            False   \n",
       "\n",
       "        Auxilliary_Heat_Fuel_Type  \n",
       "0                             Gas  \n",
       "1                             Gas  \n",
       "2                             Gas  \n",
       "3                             Gas  \n",
       "4                             Gas  \n",
       "...                           ...  \n",
       "1861818                       Gas  \n",
       "1861819                       Gas  \n",
       "1861820                       Gas  \n",
       "1861821                       Gas  \n",
       "1861822                       Gas  \n",
       "\n",
       "[1717428 rows x 14 columns]"
      ]
     },
     "execution_count": 174,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "dec_2019.drop(dec_2019[dec_2019['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "dec_2019.drop(dec_2019[dec_2019['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "dec_2019.drop(dec_2019[dec_2019['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "dec_2019.drop(dec_2019[dec_2019['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "dec_2019"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 175,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "dec_2019[\"Year\"] = \"2019\"\n",
    "dec_2019[\"Month\"] = \"dec\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 176,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "dec_2019 = dec_2019.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 177,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "dec_2019_ave = dec_2019.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# dec_2019_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 178,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "dec_2019_ave.to_csv(\"data/day/NC/dec/dec_2019_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2020 December Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 179,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "dec_2020 = pd.read_csv(\"../data_large/NC-day/2020-dec-day-NC.csv\")\n",
    "\n",
    "# dec_2020"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 180,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>788369e6a13855797ace59f45d3b9471a1c6afc7</td>\n",
       "      <td>2020-12-01 15:25:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>758</td>\n",
       "      <td>815</td>\n",
       "      <td>765</td>\n",
       "      <td>NC</td>\n",
       "      <td>Hope Mills</td>\n",
       "      <td>0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>4d69add5d25f31e2857debf606fb62fd121ea60d</td>\n",
       "      <td>2020-12-29 17:30:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>700</td>\n",
       "      <td>752</td>\n",
       "      <td>702</td>\n",
       "      <td>NC</td>\n",
       "      <td>Mebane</td>\n",
       "      <td>118</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>359555f3cb4d26c6aba433b19ff887f5cded269c</td>\n",
       "      <td>2020-12-26 19:10:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>667</td>\n",
       "      <td>701</td>\n",
       "      <td>701</td>\n",
       "      <td>NC</td>\n",
       "      <td>Charlotte</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>038d166ae1a23ae388fdd205e381308dab0c6ef0</td>\n",
       "      <td>2020-12-05 19:10:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>671</td>\n",
       "      <td>743</td>\n",
       "      <td>647</td>\n",
       "      <td>NC</td>\n",
       "      <td>Matthews</td>\n",
       "      <td>39</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>d6266abc21b086809b80bb5c3a1ea1f63d45ec82</td>\n",
       "      <td>2020-12-03 16:25:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>710</td>\n",
       "      <td>766</td>\n",
       "      <td>716</td>\n",
       "      <td>NC</td>\n",
       "      <td>Charlotte</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2253362</th>\n",
       "      <td>014c50b16659e19a0ce2470cb8e9d734cf2deb2a</td>\n",
       "      <td>2020-12-19 18:50:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>711</td>\n",
       "      <td>765</td>\n",
       "      <td>715</td>\n",
       "      <td>NC</td>\n",
       "      <td>Durham</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2253363</th>\n",
       "      <td>a7c7be85f495db81377cc3239759fb87d0c4ed97</td>\n",
       "      <td>2020-12-06 19:00:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>708</td>\n",
       "      <td>765</td>\n",
       "      <td>715</td>\n",
       "      <td>NC</td>\n",
       "      <td>Mebane</td>\n",
       "      <td>118</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2253364</th>\n",
       "      <td>d5181883a040a7fb4507f150553c401a2da110ff</td>\n",
       "      <td>2020-12-28 18:50:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>718</td>\n",
       "      <td>765</td>\n",
       "      <td>705</td>\n",
       "      <td>NC</td>\n",
       "      <td>Cary</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2253365</th>\n",
       "      <td>d8045657316c8317cdab8ed8b6e57a7a17c77f4b</td>\n",
       "      <td>2020-12-05 17:20:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>713</td>\n",
       "      <td>765</td>\n",
       "      <td>715</td>\n",
       "      <td>NC</td>\n",
       "      <td>St James</td>\n",
       "      <td>5</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2253366</th>\n",
       "      <td>d5181883a040a7fb4507f150553c401a2da110ff</td>\n",
       "      <td>2020-12-10 15:35:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>705</td>\n",
       "      <td>765</td>\n",
       "      <td>695</td>\n",
       "      <td>NC</td>\n",
       "      <td>Cary</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2171359 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                       Identifier                date_time  \\\n",
       "0        788369e6a13855797ace59f45d3b9471a1c6afc7  2020-12-01 15:25:00 UTC   \n",
       "1        4d69add5d25f31e2857debf606fb62fd121ea60d  2020-12-29 17:30:00 UTC   \n",
       "2        359555f3cb4d26c6aba433b19ff887f5cded269c  2020-12-26 19:10:00 UTC   \n",
       "4        038d166ae1a23ae388fdd205e381308dab0c6ef0  2020-12-05 19:10:00 UTC   \n",
       "5        d6266abc21b086809b80bb5c3a1ea1f63d45ec82  2020-12-03 16:25:00 UTC   \n",
       "...                                           ...                      ...   \n",
       "2253362  014c50b16659e19a0ce2470cb8e9d734cf2deb2a  2020-12-19 18:50:00 UTC   \n",
       "2253363  a7c7be85f495db81377cc3239759fb87d0c4ed97  2020-12-06 19:00:00 UTC   \n",
       "2253364  d5181883a040a7fb4507f150553c401a2da110ff  2020-12-28 18:50:00 UTC   \n",
       "2253365  d8045657316c8317cdab8ed8b6e57a7a17c77f4b  2020-12-05 17:20:00 UTC   \n",
       "2253366  d5181883a040a7fb4507f150553c401a2da110ff  2020-12-10 15:35:00 UTC   \n",
       "\n",
       "        HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0           heat          hold               758                      815   \n",
       "1           auto          hold               700                      752   \n",
       "2           cool          hold               667                      701   \n",
       "4           heat          hold               671                      743   \n",
       "5           auto          hold               710                      766   \n",
       "...          ...           ...               ...                      ...   \n",
       "2253362     auto          hold               711                      765   \n",
       "2253363     auto          hold               708                      765   \n",
       "2253364     auto          hold               718                      765   \n",
       "2253365     auto          hold               713                      765   \n",
       "2253366     auto          hold               705                      765   \n",
       "\n",
       "         TemperatureExpectedHeat ProvinceState        City  \\\n",
       "0                            765            NC  Hope Mills   \n",
       "1                            702            NC      Mebane   \n",
       "2                            701            NC   Charlotte   \n",
       "4                            647            NC    Matthews   \n",
       "5                            716            NC   Charlotte   \n",
       "...                          ...           ...         ...   \n",
       "2253362                      715            NC      Durham   \n",
       "2253363                      715            NC      Mebane   \n",
       "2253364                      705            NC        Cary   \n",
       "2253365                      715            NC    St James   \n",
       "2253366                      695            NC        Cary   \n",
       "\n",
       "         Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                          0              True         False             True   \n",
       "1                        118             False         False            False   \n",
       "2                          5             False         False            False   \n",
       "4                         39              True         False            False   \n",
       "5                          0             False         False            False   \n",
       "...                      ...               ...           ...              ...   \n",
       "2253362                    0             False         False            False   \n",
       "2253363                  118             False         False            False   \n",
       "2253364                    0             False         False            False   \n",
       "2253365                    5              True         False             True   \n",
       "2253366                    0             False         False            False   \n",
       "\n",
       "        Auxilliary_Heat_Fuel_Type  \n",
       "0                        Electric  \n",
       "1                             Gas  \n",
       "2                             Gas  \n",
       "4                             Gas  \n",
       "5                             Gas  \n",
       "...                           ...  \n",
       "2253362                       Gas  \n",
       "2253363                       Gas  \n",
       "2253364                       Gas  \n",
       "2253365                  Electric  \n",
       "2253366                       Gas  \n",
       "\n",
       "[2171359 rows x 14 columns]"
      ]
     },
     "execution_count": 180,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "dec_2020.drop(dec_2020[dec_2020['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "dec_2020.drop(dec_2020[dec_2020['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "dec_2020.drop(dec_2020[dec_2020['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "dec_2020.drop(dec_2020[dec_2020['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "dec_2020"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 181,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "dec_2020[\"Year\"] = \"2020\"\n",
    "dec_2020[\"Month\"] = \"dec\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 182,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "dec_2020 = dec_2020.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 183,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "dec_2020_ave = dec_2020.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# dec_2020_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 184,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "dec_2020_ave.to_csv(\"data/day/NC/dec/dec_2020_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "\n",
    "### Combine month CSV Files \n",
    "1. Read in files in folders for each state\n",
    "2. Export as combined CSV"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 185,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create variable for files in directory\n",
    "files = [f for f in os.listdir(\"data/day/NC/dec/\") if f.endswith(\".csv\")]\n",
    "\n",
    "# files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 186,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>Month</th>\n",
       "      <th>Year</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>City</th>\n",
       "      <th>Temperature_ctrl_ave</th>\n",
       "      <th>TemperatureExpectedCool_ave</th>\n",
       "      <th>TemperatureExpectedHeat_ave</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>004dfca396d696b8a7ce7dd71862b78384f55976</td>\n",
       "      <td>dec</td>\n",
       "      <td>2017</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>Charlotte</td>\n",
       "      <td>692.609636</td>\n",
       "      <td>780.000000</td>\n",
       "      <td>703.421829</td>\n",
       "      <td>85.0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>00bbbdb5c3b6cb8b9e78191017ecdb97dad87e47</td>\n",
       "      <td>dec</td>\n",
       "      <td>2017</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>Jacksonville</td>\n",
       "      <td>691.861702</td>\n",
       "      <td>769.144377</td>\n",
       "      <td>680.846505</td>\n",
       "      <td>5.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>00eb3069b0d62a9be8ed0bcf0b7ca35860a7b716</td>\n",
       "      <td>dec</td>\n",
       "      <td>2017</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>Mint Hill</td>\n",
       "      <td>682.823899</td>\n",
       "      <td>770.000000</td>\n",
       "      <td>690.000000</td>\n",
       "      <td>40.0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>00eb3069b0d62a9be8ed0bcf0b7ca35860a7b716</td>\n",
       "      <td>dec</td>\n",
       "      <td>2017</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>Mint Hill</td>\n",
       "      <td>712.126214</td>\n",
       "      <td>779.514563</td>\n",
       "      <td>712.524272</td>\n",
       "      <td>40.0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>00fe927e079676ec2e0bdd97d164267165e25e86</td>\n",
       "      <td>dec</td>\n",
       "      <td>2017</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>Mint Hill</td>\n",
       "      <td>726.811321</td>\n",
       "      <td>650.754717</td>\n",
       "      <td>644.660377</td>\n",
       "      <td>25.0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3428</th>\n",
       "      <td>ff62cf17a37a458b09c22b255b1317cf85257110</td>\n",
       "      <td>dec</td>\n",
       "      <td>2020</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>Wake Forest</td>\n",
       "      <td>693.428571</td>\n",
       "      <td>706.428571</td>\n",
       "      <td>698.857143</td>\n",
       "      <td>5.0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3429</th>\n",
       "      <td>ff62cf17a37a458b09c22b255b1317cf85257110</td>\n",
       "      <td>dec</td>\n",
       "      <td>2020</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>Wake Forest</td>\n",
       "      <td>687.103448</td>\n",
       "      <td>670.000000</td>\n",
       "      <td>670.000000</td>\n",
       "      <td>5.0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3430</th>\n",
       "      <td>ff662f2d3b89e4b539e7917a75b87fc605074bcd</td>\n",
       "      <td>dec</td>\n",
       "      <td>2020</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>Cary</td>\n",
       "      <td>670.313433</td>\n",
       "      <td>680.223881</td>\n",
       "      <td>678.721393</td>\n",
       "      <td>10.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3431</th>\n",
       "      <td>ffc40b303849f7f5af77972db456dc63e2eebf34</td>\n",
       "      <td>dec</td>\n",
       "      <td>2020</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>Charlotte</td>\n",
       "      <td>721.033333</td>\n",
       "      <td>779.766667</td>\n",
       "      <td>724.100000</td>\n",
       "      <td>17.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3432</th>\n",
       "      <td>ffc40b303849f7f5af77972db456dc63e2eebf34</td>\n",
       "      <td>dec</td>\n",
       "      <td>2020</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>Charlotte</td>\n",
       "      <td>773.500000</td>\n",
       "      <td>748.052632</td>\n",
       "      <td>747.868421</td>\n",
       "      <td>17.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>11842 rows × 13 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                    Identifier Month  Year HvacMode  \\\n",
       "0     004dfca396d696b8a7ce7dd71862b78384f55976   dec  2017     auto   \n",
       "1     00bbbdb5c3b6cb8b9e78191017ecdb97dad87e47   dec  2017     auto   \n",
       "2     00eb3069b0d62a9be8ed0bcf0b7ca35860a7b716   dec  2017     auto   \n",
       "3     00eb3069b0d62a9be8ed0bcf0b7ca35860a7b716   dec  2017     auto   \n",
       "4     00fe927e079676ec2e0bdd97d164267165e25e86   dec  2017     heat   \n",
       "...                                        ...   ...   ...      ...   \n",
       "3428  ff62cf17a37a458b09c22b255b1317cf85257110   dec  2020     heat   \n",
       "3429  ff62cf17a37a458b09c22b255b1317cf85257110   dec  2020     heat   \n",
       "3430  ff662f2d3b89e4b539e7917a75b87fc605074bcd   dec  2020     heat   \n",
       "3431  ffc40b303849f7f5af77972db456dc63e2eebf34   dec  2020     heat   \n",
       "3432  ffc40b303849f7f5af77972db456dc63e2eebf34   dec  2020     heat   \n",
       "\n",
       "     CalendarEvent          City  Temperature_ctrl_ave  \\\n",
       "0             hold     Charlotte            692.609636   \n",
       "1             auto  Jacksonville            691.861702   \n",
       "2             auto     Mint Hill            682.823899   \n",
       "3             hold     Mint Hill            712.126214   \n",
       "4             auto     Mint Hill            726.811321   \n",
       "...            ...           ...                   ...   \n",
       "3428          auto   Wake Forest            693.428571   \n",
       "3429          hold   Wake Forest            687.103448   \n",
       "3430          auto          Cary            670.313433   \n",
       "3431          auto     Charlotte            721.033333   \n",
       "3432          hold     Charlotte            773.500000   \n",
       "\n",
       "      TemperatureExpectedCool_ave  TemperatureExpectedHeat_ave  \\\n",
       "0                      780.000000                   703.421829   \n",
       "1                      769.144377                   680.846505   \n",
       "2                      770.000000                   690.000000   \n",
       "3                      779.514563                   712.524272   \n",
       "4                      650.754717                   644.660377   \n",
       "...                           ...                          ...   \n",
       "3428                   706.428571                   698.857143   \n",
       "3429                   670.000000                   670.000000   \n",
       "3430                   680.223881                   678.721393   \n",
       "3431                   779.766667                   724.100000   \n",
       "3432                   748.052632                   747.868421   \n",
       "\n",
       "      Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \n",
       "0                    85.0              True         False             True  \n",
       "1                     5.0             False         False             True  \n",
       "2                    40.0              True         False             True  \n",
       "3                    40.0              True         False             True  \n",
       "4                    25.0              True         False             True  \n",
       "...                   ...               ...           ...              ...  \n",
       "3428                  5.0              True         False             True  \n",
       "3429                  5.0              True         False             True  \n",
       "3430                 10.0             False         False            False  \n",
       "3431                 17.0             False         False            False  \n",
       "3432                 17.0             False         False            False  \n",
       "\n",
       "[11842 rows x 13 columns]"
      ]
     },
     "execution_count": 186,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# https://stackoverflow.com/questions/63886787/how-to-create-a-dataframe-from-multiple-csv-files\n",
    "# Initialize an empty dataframe\n",
    "NC_dec = pd.DataFrame()\n",
    "\n",
    "# Iterate through files and contents, then concatenate the data into the dataframe\n",
    "for file in files:\n",
    "    df = pd.read_csv(\"data/day/NC/dec/\" + file)\n",
    "    NC_dec = pd.concat([NC_dec, df])\n",
    "    \n",
    "NC_dec"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 187,
   "metadata": {},
   "outputs": [],
   "source": [
    "NC_dec.to_csv(\"Scraper_Output/State_Month_Day/NC/NC_dec.csv\", header=True, index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "----"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "----"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "\n",
    "### Combine state CSV Files \n",
    "1. Read in files in folders for each state\n",
    "2. Export as combined CSV"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 188,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create variable for files in directory\n",
    "files = [f for f in os.listdir(\"Scraper_Output/State_Month_Day/NC/\") if f.endswith(\".csv\")]\n",
    "\n",
    "# files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 189,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>Month</th>\n",
       "      <th>Year</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>City</th>\n",
       "      <th>Temperature_ctrl_ave</th>\n",
       "      <th>TemperatureExpectedCool_ave</th>\n",
       "      <th>TemperatureExpectedHeat_ave</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>004dfca396d696b8a7ce7dd71862b78384f55976</td>\n",
       "      <td>aug</td>\n",
       "      <td>2017</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>Charlotte</td>\n",
       "      <td>772.839416</td>\n",
       "      <td>778.583942</td>\n",
       "      <td>773.751825</td>\n",
       "      <td>85.0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>00bbbdb5c3b6cb8b9e78191017ecdb97dad87e47</td>\n",
       "      <td>aug</td>\n",
       "      <td>2017</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>Jacksonville</td>\n",
       "      <td>753.210191</td>\n",
       "      <td>756.821656</td>\n",
       "      <td>693.175159</td>\n",
       "      <td>5.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>00e5498cd188bdf09a9a1a902e6084b200cbbdef</td>\n",
       "      <td>aug</td>\n",
       "      <td>2017</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>Wade</td>\n",
       "      <td>779.000000</td>\n",
       "      <td>785.000000</td>\n",
       "      <td>640.000000</td>\n",
       "      <td>55.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>00e5498cd188bdf09a9a1a902e6084b200cbbdef</td>\n",
       "      <td>aug</td>\n",
       "      <td>2017</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>Wade</td>\n",
       "      <td>757.897436</td>\n",
       "      <td>760.000000</td>\n",
       "      <td>680.000000</td>\n",
       "      <td>55.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>00eb3069b0d62a9be8ed0bcf0b7ca35860a7b716</td>\n",
       "      <td>aug</td>\n",
       "      <td>2017</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>Mint Hill</td>\n",
       "      <td>740.389105</td>\n",
       "      <td>740.280156</td>\n",
       "      <td>679.719844</td>\n",
       "      <td>40.0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13246</th>\n",
       "      <td>fefc39b30cc29ec1a80954d26e0550bf864455f7</td>\n",
       "      <td>jun</td>\n",
       "      <td>2021</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>Raleigh</td>\n",
       "      <td>703.003068</td>\n",
       "      <td>700.633473</td>\n",
       "      <td>700.633473</td>\n",
       "      <td>9.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13247</th>\n",
       "      <td>ff14f7a9fbb7df32781e13d339c125fd7d6b8d4d</td>\n",
       "      <td>jun</td>\n",
       "      <td>2021</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>Charlotte</td>\n",
       "      <td>714.161491</td>\n",
       "      <td>705.863354</td>\n",
       "      <td>705.863354</td>\n",
       "      <td>50.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13248</th>\n",
       "      <td>ff53874bcab6f2d086a50ca0766bcadebd61e62e</td>\n",
       "      <td>jun</td>\n",
       "      <td>2021</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>Matthews</td>\n",
       "      <td>728.871429</td>\n",
       "      <td>721.828571</td>\n",
       "      <td>721.157143</td>\n",
       "      <td>40.0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13249</th>\n",
       "      <td>ff62cf17a37a458b09c22b255b1317cf85257110</td>\n",
       "      <td>jun</td>\n",
       "      <td>2021</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>Wake Forest</td>\n",
       "      <td>747.920245</td>\n",
       "      <td>748.024540</td>\n",
       "      <td>747.963190</td>\n",
       "      <td>5.0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13250</th>\n",
       "      <td>ffc40b303849f7f5af77972db456dc63e2eebf34</td>\n",
       "      <td>jun</td>\n",
       "      <td>2021</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>Charlotte</td>\n",
       "      <td>775.270950</td>\n",
       "      <td>776.187151</td>\n",
       "      <td>776.145251</td>\n",
       "      <td>17.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>73192 rows × 13 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                     Identifier Month  Year HvacMode  \\\n",
       "0      004dfca396d696b8a7ce7dd71862b78384f55976   aug  2017     cool   \n",
       "1      00bbbdb5c3b6cb8b9e78191017ecdb97dad87e47   aug  2017     auto   \n",
       "2      00e5498cd188bdf09a9a1a902e6084b200cbbdef   aug  2017     auto   \n",
       "3      00e5498cd188bdf09a9a1a902e6084b200cbbdef   aug  2017     auto   \n",
       "4      00eb3069b0d62a9be8ed0bcf0b7ca35860a7b716   aug  2017     auto   \n",
       "...                                         ...   ...   ...      ...   \n",
       "13246  fefc39b30cc29ec1a80954d26e0550bf864455f7   jun  2021     cool   \n",
       "13247  ff14f7a9fbb7df32781e13d339c125fd7d6b8d4d   jun  2021     cool   \n",
       "13248  ff53874bcab6f2d086a50ca0766bcadebd61e62e   jun  2021     cool   \n",
       "13249  ff62cf17a37a458b09c22b255b1317cf85257110   jun  2021     cool   \n",
       "13250  ffc40b303849f7f5af77972db456dc63e2eebf34   jun  2021     cool   \n",
       "\n",
       "      CalendarEvent          City  Temperature_ctrl_ave  \\\n",
       "0              hold     Charlotte            772.839416   \n",
       "1              auto  Jacksonville            753.210191   \n",
       "2              auto          Wade            779.000000   \n",
       "3              hold          Wade            757.897436   \n",
       "4              hold     Mint Hill            740.389105   \n",
       "...             ...           ...                   ...   \n",
       "13246          hold       Raleigh            703.003068   \n",
       "13247          hold     Charlotte            714.161491   \n",
       "13248          hold      Matthews            728.871429   \n",
       "13249          hold   Wake Forest            747.920245   \n",
       "13250          hold     Charlotte            775.270950   \n",
       "\n",
       "       TemperatureExpectedCool_ave  TemperatureExpectedHeat_ave  \\\n",
       "0                       778.583942                   773.751825   \n",
       "1                       756.821656                   693.175159   \n",
       "2                       785.000000                   640.000000   \n",
       "3                       760.000000                   680.000000   \n",
       "4                       740.280156                   679.719844   \n",
       "...                            ...                          ...   \n",
       "13246                   700.633473                   700.633473   \n",
       "13247                   705.863354                   705.863354   \n",
       "13248                   721.828571                   721.157143   \n",
       "13249                   748.024540                   747.963190   \n",
       "13250                   776.187151                   776.145251   \n",
       "\n",
       "       Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \n",
       "0                     85.0              True         False             True  \n",
       "1                      5.0             False         False             True  \n",
       "2                     55.0             False         False            False  \n",
       "3                     55.0             False         False            False  \n",
       "4                     40.0              True         False             True  \n",
       "...                    ...               ...           ...              ...  \n",
       "13246                  9.0             False         False            False  \n",
       "13247                 50.0             False         False             True  \n",
       "13248                 40.0              True         False             True  \n",
       "13249                  5.0              True         False             True  \n",
       "13250                 17.0             False         False            False  \n",
       "\n",
       "[73192 rows x 13 columns]"
      ]
     },
     "execution_count": 189,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# https://stackoverflow.com/questions/63886787/how-to-create-a-dataframe-from-multiple-csv-files\n",
    "# Initialize an empty dataframe\n",
    "NC_all = pd.DataFrame()\n",
    "\n",
    "# Iterate through files and contents, then concatenate the data into the dataframe\n",
    "for file in files:\n",
    "    df = pd.read_csv(\"Scraper_Output/State_Month_Day/NC/\" + file)\n",
    "    NC_all = pd.concat([NC_all, df])\n",
    "    \n",
    "NC_all"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 190,
   "metadata": {},
   "outputs": [],
   "source": [
    "NC_all.to_csv(\"Scraper_Output/State_Month_Day/NC_all_day.csv\", header=True, index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 191,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Unique jan_2017: ['NC']\n",
      "Unique jan_2018: ['NC']\n",
      "Unique jan_2019: ['NC']\n",
      "Unique jan_2020: ['NC']\n",
      "Unique jan_2021: ['NC']\n",
      "Unique feb_2017: ['NC']\n",
      "Unique feb_2018: ['NC']\n",
      "Unique feb_2019: ['NC']\n",
      "Unique feb_2020: ['NC']\n",
      "Unique feb_2021: ['NC']\n",
      "Unique jun_2017: ['NC']\n",
      "Unique jun_2018: ['NC']\n",
      "Unique jun_2019: ['NC']\n",
      "Unique jun_2020: ['NC']\n",
      "Unique jun_2021: ['NC']\n",
      "Unique jul_2017: ['NC']\n",
      "Unique jul_2018: ['NC']\n",
      "Unique jul_2019: ['NC']\n",
      "Unique jul_2020: ['NC']\n",
      "Unique jul_2021: ['NC']\n",
      "Unique aug_2017: ['NC']\n",
      "Unique aug_2018: ['NC']\n",
      "Unique aug_2019: ['NC']\n",
      "Unique aug_2020: ['NC']\n",
      "Unique dec_2017: ['NC']\n",
      "Unique dec_2018: ['NC']\n",
      "Unique dec_2019: ['NY']\n",
      "Unique dec_2020: ['NC']\n"
     ]
    }
   ],
   "source": [
    "# Datacheck to mNCe sure state was selected correctly in BQ sql queries\n",
    "\n",
    "print(f\"Unique jan_2017: {jan_2017['ProvinceState'].unique()}\")\n",
    "print(f\"Unique jan_2018: {jan_2018['ProvinceState'].unique()}\")\n",
    "print(f\"Unique jan_2019: {jan_2019['ProvinceState'].unique()}\")\n",
    "print(f\"Unique jan_2020: {jan_2020['ProvinceState'].unique()}\")\n",
    "print(f\"Unique jan_2021: {jan_2021['ProvinceState'].unique()}\")\n",
    "print(f\"Unique feb_2017: {feb_2017['ProvinceState'].unique()}\")\n",
    "print(f\"Unique feb_2018: {feb_2018['ProvinceState'].unique()}\")\n",
    "print(f\"Unique feb_2019: {feb_2019['ProvinceState'].unique()}\")\n",
    "print(f\"Unique feb_2020: {feb_2020['ProvinceState'].unique()}\")\n",
    "print(f\"Unique feb_2021: {feb_2021['ProvinceState'].unique()}\")\n",
    "print(f\"Unique jun_2017: {jun_2017['ProvinceState'].unique()}\")\n",
    "print(f\"Unique jun_2018: {jun_2018['ProvinceState'].unique()}\")\n",
    "print(f\"Unique jun_2019: {jun_2019['ProvinceState'].unique()}\")\n",
    "print(f\"Unique jun_2020: {jun_2020['ProvinceState'].unique()}\")\n",
    "print(f\"Unique jun_2021: {jun_2021['ProvinceState'].unique()}\")\n",
    "print(f\"Unique jul_2017: {jul_2017['ProvinceState'].unique()}\")\n",
    "print(f\"Unique jul_2018: {jul_2018['ProvinceState'].unique()}\")\n",
    "print(f\"Unique jul_2019: {jul_2019['ProvinceState'].unique()}\")\n",
    "print(f\"Unique jul_2020: {jul_2020['ProvinceState'].unique()}\")\n",
    "print(f\"Unique jul_2021: {jul_2021['ProvinceState'].unique()}\")\n",
    "print(f\"Unique aug_2017: {aug_2017['ProvinceState'].unique()}\")\n",
    "print(f\"Unique aug_2018: {aug_2018['ProvinceState'].unique()}\")\n",
    "print(f\"Unique aug_2019: {aug_2019['ProvinceState'].unique()}\")\n",
    "print(f\"Unique aug_2020: {aug_2020['ProvinceState'].unique()}\")\n",
    "print(f\"Unique dec_2017: {dec_2017['ProvinceState'].unique()}\")\n",
    "print(f\"Unique dec_2018: {dec_2018['ProvinceState'].unique()}\")\n",
    "print(f\"Unique dec_2019: {dec_2019['ProvinceState'].unique()}\")\n",
    "print(f\"Unique dec_2020: {dec_2020['ProvinceState'].unique()}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.10"
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}