DYD_MT-day.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# DYD Thermostat Data \n",
    "\n",
    "## Preprocess\n",
    "\n",
    "1. Generated CSV file from queries in BigQueary\n",
    "\n",
    "2. Data separated into states\n",
    "\n",
    "3. Aggregated the data in Pandas by month\n",
    "\n",
    "4. Combine 4 years\n",
    "\n",
    "5. Group by Identifier\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Dependencies\n",
    "import pandas as pd\n",
    "import os\n",
    "import numpy as np\n",
    "from pathlib import Path\n",
    "from datetime import datetime"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "## January"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2017 January Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "jan_2017 = pd.read_csv(\"../data_large/MT-day/2017-jan-day-MT.csv\")\n",
    "\n",
    "# jan_2017"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>5d1ff376e3fc7ee713a1138cbb661ddad9b1e257</td>\n",
       "      <td>2017-01-06T17:40:00Z</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>666</td>\n",
       "      <td>674</td>\n",
       "      <td>674</td>\n",
       "      <td>MT</td>\n",
       "      <td>Bozeman</td>\n",
       "      <td>15</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>23d834c190c3686c02c030ee0bf42003872bd591</td>\n",
       "      <td>2017-01-17T18:30:00Z</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>696</td>\n",
       "      <td>724</td>\n",
       "      <td>694</td>\n",
       "      <td>MT</td>\n",
       "      <td>Bozeman</td>\n",
       "      <td>25</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>5d1ff376e3fc7ee713a1138cbb661ddad9b1e257</td>\n",
       "      <td>2017-01-08T17:15:00Z</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>642</td>\n",
       "      <td>663</td>\n",
       "      <td>641</td>\n",
       "      <td>MT</td>\n",
       "      <td>Bozeman</td>\n",
       "      <td>15</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>5d1ff376e3fc7ee713a1138cbb661ddad9b1e257</td>\n",
       "      <td>2017-01-23T18:40:00Z</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>655</td>\n",
       "      <td>708</td>\n",
       "      <td>644</td>\n",
       "      <td>MT</td>\n",
       "      <td>Bozeman</td>\n",
       "      <td>15</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>29f5966d7808882d6c76a0a2d7091c18b3587704</td>\n",
       "      <td>2017-01-20T18:10:00Z</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>740</td>\n",
       "      <td>745</td>\n",
       "      <td>745</td>\n",
       "      <td>MT</td>\n",
       "      <td>Bozeman</td>\n",
       "      <td>9</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7920</th>\n",
       "      <td>a446833ed31f9039f78e43743749d949ab71cfe0</td>\n",
       "      <td>2017-01-08T14:20:00Z</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>652</td>\n",
       "      <td>750</td>\n",
       "      <td>700</td>\n",
       "      <td>MT</td>\n",
       "      <td>Fort Benton</td>\n",
       "      <td>16</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7921</th>\n",
       "      <td>a446833ed31f9039f78e43743749d949ab71cfe0</td>\n",
       "      <td>2017-01-06T13:55:00Z</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>657</td>\n",
       "      <td>750</td>\n",
       "      <td>700</td>\n",
       "      <td>MT</td>\n",
       "      <td>Fort Benton</td>\n",
       "      <td>16</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7922</th>\n",
       "      <td>a446833ed31f9039f78e43743749d949ab71cfe0</td>\n",
       "      <td>2017-01-06T13:50:00Z</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>661</td>\n",
       "      <td>750</td>\n",
       "      <td>700</td>\n",
       "      <td>MT</td>\n",
       "      <td>Fort Benton</td>\n",
       "      <td>16</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7923</th>\n",
       "      <td>a446833ed31f9039f78e43743749d949ab71cfe0</td>\n",
       "      <td>2017-01-08T13:30:00Z</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>651</td>\n",
       "      <td>750</td>\n",
       "      <td>700</td>\n",
       "      <td>MT</td>\n",
       "      <td>Fort Benton</td>\n",
       "      <td>16</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7924</th>\n",
       "      <td>a446833ed31f9039f78e43743749d949ab71cfe0</td>\n",
       "      <td>2017-01-06T14:05:00Z</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>652</td>\n",
       "      <td>750</td>\n",
       "      <td>700</td>\n",
       "      <td>MT</td>\n",
       "      <td>Fort Benton</td>\n",
       "      <td>16</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5354 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                    Identifier             date_time HvacMode  \\\n",
       "0     5d1ff376e3fc7ee713a1138cbb661ddad9b1e257  2017-01-06T17:40:00Z     heat   \n",
       "1     23d834c190c3686c02c030ee0bf42003872bd591  2017-01-17T18:30:00Z     heat   \n",
       "2     5d1ff376e3fc7ee713a1138cbb661ddad9b1e257  2017-01-08T17:15:00Z     heat   \n",
       "3     5d1ff376e3fc7ee713a1138cbb661ddad9b1e257  2017-01-23T18:40:00Z     heat   \n",
       "4     29f5966d7808882d6c76a0a2d7091c18b3587704  2017-01-20T18:10:00Z     heat   \n",
       "...                                        ...                   ...      ...   \n",
       "7920  a446833ed31f9039f78e43743749d949ab71cfe0  2017-01-08T14:20:00Z     auto   \n",
       "7921  a446833ed31f9039f78e43743749d949ab71cfe0  2017-01-06T13:55:00Z     auto   \n",
       "7922  a446833ed31f9039f78e43743749d949ab71cfe0  2017-01-06T13:50:00Z     auto   \n",
       "7923  a446833ed31f9039f78e43743749d949ab71cfe0  2017-01-08T13:30:00Z     auto   \n",
       "7924  a446833ed31f9039f78e43743749d949ab71cfe0  2017-01-06T14:05:00Z     auto   \n",
       "\n",
       "     CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0             hold               666                      674   \n",
       "1             hold               696                      724   \n",
       "2             hold               642                      663   \n",
       "3             hold               655                      708   \n",
       "4             hold               740                      745   \n",
       "...            ...               ...                      ...   \n",
       "7920          hold               652                      750   \n",
       "7921          hold               657                      750   \n",
       "7922          hold               661                      750   \n",
       "7923          hold               651                      750   \n",
       "7924          hold               652                      750   \n",
       "\n",
       "      TemperatureExpectedHeat ProvinceState         City  Age_of_Home__years_  \\\n",
       "0                         674            MT      Bozeman                   15   \n",
       "1                         694            MT      Bozeman                   25   \n",
       "2                         641            MT      Bozeman                   15   \n",
       "3                         644            MT      Bozeman                   15   \n",
       "4                         745            MT      Bozeman                    9   \n",
       "...                       ...           ...          ...                  ...   \n",
       "7920                      700            MT  Fort Benton                   16   \n",
       "7921                      700            MT  Fort Benton                   16   \n",
       "7922                      700            MT  Fort Benton                   16   \n",
       "7923                      700            MT  Fort Benton                   16   \n",
       "7924                      700            MT  Fort Benton                   16   \n",
       "\n",
       "      allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                 True         False            False   \n",
       "1                False         False            False   \n",
       "2                 True         False            False   \n",
       "3                 True         False            False   \n",
       "4                 True         False            False   \n",
       "...                ...           ...              ...   \n",
       "7920             False         False            False   \n",
       "7921             False         False            False   \n",
       "7922             False         False            False   \n",
       "7923             False         False            False   \n",
       "7924             False         False            False   \n",
       "\n",
       "     Auxilliary_Heat_Fuel_Type  \n",
       "0                          Gas  \n",
       "1                          Gas  \n",
       "2                          Gas  \n",
       "3                          Gas  \n",
       "4                          Gas  \n",
       "...                        ...  \n",
       "7920                       Gas  \n",
       "7921                       Gas  \n",
       "7922                       Gas  \n",
       "7923                       Gas  \n",
       "7924                       Gas  \n",
       "\n",
       "[5354 rows x 14 columns]"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "jan_2017.drop(jan_2017[jan_2017['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "jan_2017.drop(jan_2017[jan_2017['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "jan_2017.drop(jan_2017[jan_2017['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "jan_2017.drop(jan_2017[jan_2017['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "jan_2017"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "jan_2017[\"Year\"] = \"2017\"\n",
    "jan_2017[\"Month\"] = \"Jan\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "jan_2017 = jan_2017.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>Temperature_ctrl_ave</th>\n",
       "      <th>TemperatureExpectedCool_ave</th>\n",
       "      <th>TemperatureExpectedHeat_ave</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Identifier</th>\n",
       "      <th>Month</th>\n",
       "      <th>Year</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>City</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>23d834c190c3686c02c030ee0bf42003872bd591</th>\n",
       "      <th>Jan</th>\n",
       "      <th>2017</th>\n",
       "      <th>heat</th>\n",
       "      <th>hold</th>\n",
       "      <th>Bozeman</th>\n",
       "      <td>692.737226</td>\n",
       "      <td>698.912409</td>\n",
       "      <td>697.160584</td>\n",
       "      <td>25.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">29f5966d7808882d6c76a0a2d7091c18b3587704</th>\n",
       "      <th rowspan=\"2\" valign=\"top\">Jan</th>\n",
       "      <th rowspan=\"2\" valign=\"top\">2017</th>\n",
       "      <th rowspan=\"2\" valign=\"top\">heat</th>\n",
       "      <th>auto</th>\n",
       "      <th>Bozeman</th>\n",
       "      <td>690.696629</td>\n",
       "      <td>693.112360</td>\n",
       "      <td>692.719101</td>\n",
       "      <td>9.0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>hold</th>\n",
       "      <th>Bozeman</th>\n",
       "      <td>733.701031</td>\n",
       "      <td>737.855670</td>\n",
       "      <td>737.845361</td>\n",
       "      <td>9.0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">3e35e2d37f16eb3b72b2cbb145e6ed964ce01ca4</th>\n",
       "      <th rowspan=\"2\" valign=\"top\">Jan</th>\n",
       "      <th rowspan=\"2\" valign=\"top\">2017</th>\n",
       "      <th rowspan=\"2\" valign=\"top\">auto</th>\n",
       "      <th>auto</th>\n",
       "      <th>Great Falls</th>\n",
       "      <td>719.717391</td>\n",
       "      <td>783.188406</td>\n",
       "      <td>720.000000</td>\n",
       "      <td>45.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>hold</th>\n",
       "      <th>Great Falls</th>\n",
       "      <td>724.250000</td>\n",
       "      <td>780.681818</td>\n",
       "      <td>716.590909</td>\n",
       "      <td>45.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>40f358a2efb733ab89fe3bb6e2f7b0280ccb4917</th>\n",
       "      <th>Jan</th>\n",
       "      <th>2017</th>\n",
       "      <th>heat</th>\n",
       "      <th>hold</th>\n",
       "      <th>Billings</th>\n",
       "      <td>686.144385</td>\n",
       "      <td>690.454545</td>\n",
       "      <td>689.877005</td>\n",
       "      <td>70.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">5d1ff376e3fc7ee713a1138cbb661ddad9b1e257</th>\n",
       "      <th rowspan=\"2\" valign=\"top\">Jan</th>\n",
       "      <th rowspan=\"2\" valign=\"top\">2017</th>\n",
       "      <th rowspan=\"2\" valign=\"top\">heat</th>\n",
       "      <th>auto</th>\n",
       "      <th>Bozeman</th>\n",
       "      <td>593.000000</td>\n",
       "      <td>731.750000</td>\n",
       "      <td>633.750000</td>\n",
       "      <td>15.0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>hold</th>\n",
       "      <th>Bozeman</th>\n",
       "      <td>657.972222</td>\n",
       "      <td>665.787037</td>\n",
       "      <td>657.564815</td>\n",
       "      <td>15.0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">7258db280a1da00d548cd8ea30f21c7d7d4932e0</th>\n",
       "      <th rowspan=\"2\" valign=\"top\">Jan</th>\n",
       "      <th rowspan=\"2\" valign=\"top\">2017</th>\n",
       "      <th rowspan=\"2\" valign=\"top\">heat</th>\n",
       "      <th>auto</th>\n",
       "      <th>Bozeman</th>\n",
       "      <td>655.606061</td>\n",
       "      <td>660.000000</td>\n",
       "      <td>660.000000</td>\n",
       "      <td>16.0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>hold</th>\n",
       "      <th>Bozeman</th>\n",
       "      <td>666.013245</td>\n",
       "      <td>668.463576</td>\n",
       "      <td>668.430464</td>\n",
       "      <td>16.0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9d4866ebcb99c98fc731f171a8933d0a437eff2c</th>\n",
       "      <th>Jan</th>\n",
       "      <th>2017</th>\n",
       "      <th>heat</th>\n",
       "      <th>hold</th>\n",
       "      <th>Missoula</th>\n",
       "      <td>684.049793</td>\n",
       "      <td>687.705394</td>\n",
       "      <td>687.414938</td>\n",
       "      <td>25.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">9ece7da8b39fee15b9ccaa8e095adf7d0cf2a2a6</th>\n",
       "      <th rowspan=\"2\" valign=\"top\">Jan</th>\n",
       "      <th rowspan=\"2\" valign=\"top\">2017</th>\n",
       "      <th rowspan=\"2\" valign=\"top\">heat</th>\n",
       "      <th>auto</th>\n",
       "      <th>Missoula</th>\n",
       "      <td>701.791667</td>\n",
       "      <td>700.000000</td>\n",
       "      <td>700.000000</td>\n",
       "      <td>40.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>hold</th>\n",
       "      <th>Missoula</th>\n",
       "      <td>723.564460</td>\n",
       "      <td>720.198606</td>\n",
       "      <td>720.108014</td>\n",
       "      <td>40.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">a446833ed31f9039f78e43743749d949ab71cfe0</th>\n",
       "      <th rowspan=\"2\" valign=\"top\">Jan</th>\n",
       "      <th rowspan=\"2\" valign=\"top\">2017</th>\n",
       "      <th rowspan=\"2\" valign=\"top\">auto</th>\n",
       "      <th>auto</th>\n",
       "      <th>Fort Benton</th>\n",
       "      <td>677.584906</td>\n",
       "      <td>738.471698</td>\n",
       "      <td>688.471698</td>\n",
       "      <td>16.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>hold</th>\n",
       "      <th>Fort Benton</th>\n",
       "      <td>687.462810</td>\n",
       "      <td>749.285124</td>\n",
       "      <td>693.023416</td>\n",
       "      <td>16.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"3\" valign=\"top\">c90b4dfdcd0c12591663fdc2b2a3fdec6faced91</th>\n",
       "      <th rowspan=\"3\" valign=\"top\">Jan</th>\n",
       "      <th rowspan=\"3\" valign=\"top\">2017</th>\n",
       "      <th>auto</th>\n",
       "      <th>hold</th>\n",
       "      <th>Missoula</th>\n",
       "      <td>643.271605</td>\n",
       "      <td>765.000000</td>\n",
       "      <td>645.000000</td>\n",
       "      <td>60.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">heat</th>\n",
       "      <th>auto</th>\n",
       "      <th>Missoula</th>\n",
       "      <td>691.233333</td>\n",
       "      <td>720.366667</td>\n",
       "      <td>719.766667</td>\n",
       "      <td>60.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>hold</th>\n",
       "      <th>Missoula</th>\n",
       "      <td>643.176471</td>\n",
       "      <td>764.705882</td>\n",
       "      <td>645.411765</td>\n",
       "      <td>60.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">c9a27b6d42a3b1e3d96263a706132ded7796fb6f</th>\n",
       "      <th rowspan=\"2\" valign=\"top\">Jan</th>\n",
       "      <th rowspan=\"2\" valign=\"top\">2017</th>\n",
       "      <th rowspan=\"2\" valign=\"top\">auto</th>\n",
       "      <th>auto</th>\n",
       "      <th>Great Falls</th>\n",
       "      <td>718.477273</td>\n",
       "      <td>780.000000</td>\n",
       "      <td>720.000000</td>\n",
       "      <td>50.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>hold</th>\n",
       "      <th>Great Falls</th>\n",
       "      <td>706.625000</td>\n",
       "      <td>780.000000</td>\n",
       "      <td>660.000000</td>\n",
       "      <td>50.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>e4ed8ab8303e5e1de4b181ce40545279719e6069</th>\n",
       "      <th>Jan</th>\n",
       "      <th>2017</th>\n",
       "      <th>heat</th>\n",
       "      <th>hold</th>\n",
       "      <th>Bozeman</th>\n",
       "      <td>677.673993</td>\n",
       "      <td>670.726190</td>\n",
       "      <td>670.726190</td>\n",
       "      <td>10.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">f404081c89442a6b4699e93e43dc4bb72f9e2dc1</th>\n",
       "      <th rowspan=\"2\" valign=\"top\">Jan</th>\n",
       "      <th rowspan=\"2\" valign=\"top\">2017</th>\n",
       "      <th rowspan=\"2\" valign=\"top\">heat</th>\n",
       "      <th>auto</th>\n",
       "      <th>Great Falls</th>\n",
       "      <td>703.714286</td>\n",
       "      <td>710.000000</td>\n",
       "      <td>710.000000</td>\n",
       "      <td>55.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>hold</th>\n",
       "      <th>Great Falls</th>\n",
       "      <td>698.352505</td>\n",
       "      <td>701.853432</td>\n",
       "      <td>701.641929</td>\n",
       "      <td>55.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                                                        Temperature_ctrl_ave  \\\n",
       "Identifier                               Month Year HvacMode CalendarEvent City                                \n",
       "23d834c190c3686c02c030ee0bf42003872bd591 Jan   2017 heat     hold          Bozeman                692.737226   \n",
       "29f5966d7808882d6c76a0a2d7091c18b3587704 Jan   2017 heat     auto          Bozeman                690.696629   \n",
       "                                                             hold          Bozeman                733.701031   \n",
       "3e35e2d37f16eb3b72b2cbb145e6ed964ce01ca4 Jan   2017 auto     auto          Great Falls            719.717391   \n",
       "                                                             hold          Great Falls            724.250000   \n",
       "40f358a2efb733ab89fe3bb6e2f7b0280ccb4917 Jan   2017 heat     hold          Billings               686.144385   \n",
       "5d1ff376e3fc7ee713a1138cbb661ddad9b1e257 Jan   2017 heat     auto          Bozeman                593.000000   \n",
       "                                                             hold          Bozeman                657.972222   \n",
       "7258db280a1da00d548cd8ea30f21c7d7d4932e0 Jan   2017 heat     auto          Bozeman                655.606061   \n",
       "                                                             hold          Bozeman                666.013245   \n",
       "9d4866ebcb99c98fc731f171a8933d0a437eff2c Jan   2017 heat     hold          Missoula               684.049793   \n",
       "9ece7da8b39fee15b9ccaa8e095adf7d0cf2a2a6 Jan   2017 heat     auto          Missoula               701.791667   \n",
       "                                                             hold          Missoula               723.564460   \n",
       "a446833ed31f9039f78e43743749d949ab71cfe0 Jan   2017 auto     auto          Fort Benton            677.584906   \n",
       "                                                             hold          Fort Benton            687.462810   \n",
       "c90b4dfdcd0c12591663fdc2b2a3fdec6faced91 Jan   2017 auto     hold          Missoula               643.271605   \n",
       "                                                    heat     auto          Missoula               691.233333   \n",
       "                                                             hold          Missoula               643.176471   \n",
       "c9a27b6d42a3b1e3d96263a706132ded7796fb6f Jan   2017 auto     auto          Great Falls            718.477273   \n",
       "                                                             hold          Great Falls            706.625000   \n",
       "e4ed8ab8303e5e1de4b181ce40545279719e6069 Jan   2017 heat     hold          Bozeman                677.673993   \n",
       "f404081c89442a6b4699e93e43dc4bb72f9e2dc1 Jan   2017 heat     auto          Great Falls            703.714286   \n",
       "                                                             hold          Great Falls            698.352505   \n",
       "\n",
       "                                                                                        TemperatureExpectedCool_ave  \\\n",
       "Identifier                               Month Year HvacMode CalendarEvent City                                       \n",
       "23d834c190c3686c02c030ee0bf42003872bd591 Jan   2017 heat     hold          Bozeman                       698.912409   \n",
       "29f5966d7808882d6c76a0a2d7091c18b3587704 Jan   2017 heat     auto          Bozeman                       693.112360   \n",
       "                                                             hold          Bozeman                       737.855670   \n",
       "3e35e2d37f16eb3b72b2cbb145e6ed964ce01ca4 Jan   2017 auto     auto          Great Falls                   783.188406   \n",
       "                                                             hold          Great Falls                   780.681818   \n",
       "40f358a2efb733ab89fe3bb6e2f7b0280ccb4917 Jan   2017 heat     hold          Billings                      690.454545   \n",
       "5d1ff376e3fc7ee713a1138cbb661ddad9b1e257 Jan   2017 heat     auto          Bozeman                       731.750000   \n",
       "                                                             hold          Bozeman                       665.787037   \n",
       "7258db280a1da00d548cd8ea30f21c7d7d4932e0 Jan   2017 heat     auto          Bozeman                       660.000000   \n",
       "                                                             hold          Bozeman                       668.463576   \n",
       "9d4866ebcb99c98fc731f171a8933d0a437eff2c Jan   2017 heat     hold          Missoula                      687.705394   \n",
       "9ece7da8b39fee15b9ccaa8e095adf7d0cf2a2a6 Jan   2017 heat     auto          Missoula                      700.000000   \n",
       "                                                             hold          Missoula                      720.198606   \n",
       "a446833ed31f9039f78e43743749d949ab71cfe0 Jan   2017 auto     auto          Fort Benton                   738.471698   \n",
       "                                                             hold          Fort Benton                   749.285124   \n",
       "c90b4dfdcd0c12591663fdc2b2a3fdec6faced91 Jan   2017 auto     hold          Missoula                      765.000000   \n",
       "                                                    heat     auto          Missoula                      720.366667   \n",
       "                                                             hold          Missoula                      764.705882   \n",
       "c9a27b6d42a3b1e3d96263a706132ded7796fb6f Jan   2017 auto     auto          Great Falls                   780.000000   \n",
       "                                                             hold          Great Falls                   780.000000   \n",
       "e4ed8ab8303e5e1de4b181ce40545279719e6069 Jan   2017 heat     hold          Bozeman                       670.726190   \n",
       "f404081c89442a6b4699e93e43dc4bb72f9e2dc1 Jan   2017 heat     auto          Great Falls                   710.000000   \n",
       "                                                             hold          Great Falls                   701.853432   \n",
       "\n",
       "                                                                                        TemperatureExpectedHeat_ave  \\\n",
       "Identifier                               Month Year HvacMode CalendarEvent City                                       \n",
       "23d834c190c3686c02c030ee0bf42003872bd591 Jan   2017 heat     hold          Bozeman                       697.160584   \n",
       "29f5966d7808882d6c76a0a2d7091c18b3587704 Jan   2017 heat     auto          Bozeman                       692.719101   \n",
       "                                                             hold          Bozeman                       737.845361   \n",
       "3e35e2d37f16eb3b72b2cbb145e6ed964ce01ca4 Jan   2017 auto     auto          Great Falls                   720.000000   \n",
       "                                                             hold          Great Falls                   716.590909   \n",
       "40f358a2efb733ab89fe3bb6e2f7b0280ccb4917 Jan   2017 heat     hold          Billings                      689.877005   \n",
       "5d1ff376e3fc7ee713a1138cbb661ddad9b1e257 Jan   2017 heat     auto          Bozeman                       633.750000   \n",
       "                                                             hold          Bozeman                       657.564815   \n",
       "7258db280a1da00d548cd8ea30f21c7d7d4932e0 Jan   2017 heat     auto          Bozeman                       660.000000   \n",
       "                                                             hold          Bozeman                       668.430464   \n",
       "9d4866ebcb99c98fc731f171a8933d0a437eff2c Jan   2017 heat     hold          Missoula                      687.414938   \n",
       "9ece7da8b39fee15b9ccaa8e095adf7d0cf2a2a6 Jan   2017 heat     auto          Missoula                      700.000000   \n",
       "                                                             hold          Missoula                      720.108014   \n",
       "a446833ed31f9039f78e43743749d949ab71cfe0 Jan   2017 auto     auto          Fort Benton                   688.471698   \n",
       "                                                             hold          Fort Benton                   693.023416   \n",
       "c90b4dfdcd0c12591663fdc2b2a3fdec6faced91 Jan   2017 auto     hold          Missoula                      645.000000   \n",
       "                                                    heat     auto          Missoula                      719.766667   \n",
       "                                                             hold          Missoula                      645.411765   \n",
       "c9a27b6d42a3b1e3d96263a706132ded7796fb6f Jan   2017 auto     auto          Great Falls                   720.000000   \n",
       "                                                             hold          Great Falls                   660.000000   \n",
       "e4ed8ab8303e5e1de4b181ce40545279719e6069 Jan   2017 heat     hold          Bozeman                       670.726190   \n",
       "f404081c89442a6b4699e93e43dc4bb72f9e2dc1 Jan   2017 heat     auto          Great Falls                   710.000000   \n",
       "                                                             hold          Great Falls                   701.641929   \n",
       "\n",
       "                                                                                        Age_of_Home__years_  \\\n",
       "Identifier                               Month Year HvacMode CalendarEvent City                               \n",
       "23d834c190c3686c02c030ee0bf42003872bd591 Jan   2017 heat     hold          Bozeman                     25.0   \n",
       "29f5966d7808882d6c76a0a2d7091c18b3587704 Jan   2017 heat     auto          Bozeman                      9.0   \n",
       "                                                             hold          Bozeman                      9.0   \n",
       "3e35e2d37f16eb3b72b2cbb145e6ed964ce01ca4 Jan   2017 auto     auto          Great Falls                 45.0   \n",
       "                                                             hold          Great Falls                 45.0   \n",
       "40f358a2efb733ab89fe3bb6e2f7b0280ccb4917 Jan   2017 heat     hold          Billings                    70.0   \n",
       "5d1ff376e3fc7ee713a1138cbb661ddad9b1e257 Jan   2017 heat     auto          Bozeman                     15.0   \n",
       "                                                             hold          Bozeman                     15.0   \n",
       "7258db280a1da00d548cd8ea30f21c7d7d4932e0 Jan   2017 heat     auto          Bozeman                     16.0   \n",
       "                                                             hold          Bozeman                     16.0   \n",
       "9d4866ebcb99c98fc731f171a8933d0a437eff2c Jan   2017 heat     hold          Missoula                    25.0   \n",
       "9ece7da8b39fee15b9ccaa8e095adf7d0cf2a2a6 Jan   2017 heat     auto          Missoula                    40.0   \n",
       "                                                             hold          Missoula                    40.0   \n",
       "a446833ed31f9039f78e43743749d949ab71cfe0 Jan   2017 auto     auto          Fort Benton                 16.0   \n",
       "                                                             hold          Fort Benton                 16.0   \n",
       "c90b4dfdcd0c12591663fdc2b2a3fdec6faced91 Jan   2017 auto     hold          Missoula                    60.0   \n",
       "                                                    heat     auto          Missoula                    60.0   \n",
       "                                                             hold          Missoula                    60.0   \n",
       "c9a27b6d42a3b1e3d96263a706132ded7796fb6f Jan   2017 auto     auto          Great Falls                 50.0   \n",
       "                                                             hold          Great Falls                 50.0   \n",
       "e4ed8ab8303e5e1de4b181ce40545279719e6069 Jan   2017 heat     hold          Bozeman                     10.0   \n",
       "f404081c89442a6b4699e93e43dc4bb72f9e2dc1 Jan   2017 heat     auto          Great Falls                 55.0   \n",
       "                                                             hold          Great Falls                 55.0   \n",
       "\n",
       "                                                                                        allowCompWithAux  \\\n",
       "Identifier                               Month Year HvacMode CalendarEvent City                            \n",
       "23d834c190c3686c02c030ee0bf42003872bd591 Jan   2017 heat     hold          Bozeman                 False   \n",
       "29f5966d7808882d6c76a0a2d7091c18b3587704 Jan   2017 heat     auto          Bozeman                  True   \n",
       "                                                             hold          Bozeman                  True   \n",
       "3e35e2d37f16eb3b72b2cbb145e6ed964ce01ca4 Jan   2017 auto     auto          Great Falls             False   \n",
       "                                                             hold          Great Falls             False   \n",
       "40f358a2efb733ab89fe3bb6e2f7b0280ccb4917 Jan   2017 heat     hold          Billings                False   \n",
       "5d1ff376e3fc7ee713a1138cbb661ddad9b1e257 Jan   2017 heat     auto          Bozeman                  True   \n",
       "                                                             hold          Bozeman                  True   \n",
       "7258db280a1da00d548cd8ea30f21c7d7d4932e0 Jan   2017 heat     auto          Bozeman                  True   \n",
       "                                                             hold          Bozeman                  True   \n",
       "9d4866ebcb99c98fc731f171a8933d0a437eff2c Jan   2017 heat     hold          Missoula                False   \n",
       "9ece7da8b39fee15b9ccaa8e095adf7d0cf2a2a6 Jan   2017 heat     auto          Missoula                False   \n",
       "                                                             hold          Missoula                False   \n",
       "a446833ed31f9039f78e43743749d949ab71cfe0 Jan   2017 auto     auto          Fort Benton             False   \n",
       "                                                             hold          Fort Benton             False   \n",
       "c90b4dfdcd0c12591663fdc2b2a3fdec6faced91 Jan   2017 auto     hold          Missoula                False   \n",
       "                                                    heat     auto          Missoula                False   \n",
       "                                                             hold          Missoula                False   \n",
       "c9a27b6d42a3b1e3d96263a706132ded7796fb6f Jan   2017 auto     auto          Great Falls             False   \n",
       "                                                             hold          Great Falls             False   \n",
       "e4ed8ab8303e5e1de4b181ce40545279719e6069 Jan   2017 heat     hold          Bozeman                 False   \n",
       "f404081c89442a6b4699e93e43dc4bb72f9e2dc1 Jan   2017 heat     auto          Great Falls             False   \n",
       "                                                             hold          Great Falls             False   \n",
       "\n",
       "                                                                                        Has_Electric  \\\n",
       "Identifier                               Month Year HvacMode CalendarEvent City                        \n",
       "23d834c190c3686c02c030ee0bf42003872bd591 Jan   2017 heat     hold          Bozeman             False   \n",
       "29f5966d7808882d6c76a0a2d7091c18b3587704 Jan   2017 heat     auto          Bozeman             False   \n",
       "                                                             hold          Bozeman             False   \n",
       "3e35e2d37f16eb3b72b2cbb145e6ed964ce01ca4 Jan   2017 auto     auto          Great Falls         False   \n",
       "                                                             hold          Great Falls         False   \n",
       "40f358a2efb733ab89fe3bb6e2f7b0280ccb4917 Jan   2017 heat     hold          Billings            False   \n",
       "5d1ff376e3fc7ee713a1138cbb661ddad9b1e257 Jan   2017 heat     auto          Bozeman             False   \n",
       "                                                             hold          Bozeman             False   \n",
       "7258db280a1da00d548cd8ea30f21c7d7d4932e0 Jan   2017 heat     auto          Bozeman             False   \n",
       "                                                             hold          Bozeman             False   \n",
       "9d4866ebcb99c98fc731f171a8933d0a437eff2c Jan   2017 heat     hold          Missoula            False   \n",
       "9ece7da8b39fee15b9ccaa8e095adf7d0cf2a2a6 Jan   2017 heat     auto          Missoula            False   \n",
       "                                                             hold          Missoula            False   \n",
       "a446833ed31f9039f78e43743749d949ab71cfe0 Jan   2017 auto     auto          Fort Benton         False   \n",
       "                                                             hold          Fort Benton         False   \n",
       "c90b4dfdcd0c12591663fdc2b2a3fdec6faced91 Jan   2017 auto     hold          Missoula            False   \n",
       "                                                    heat     auto          Missoula            False   \n",
       "                                                             hold          Missoula            False   \n",
       "c9a27b6d42a3b1e3d96263a706132ded7796fb6f Jan   2017 auto     auto          Great Falls         False   \n",
       "                                                             hold          Great Falls         False   \n",
       "e4ed8ab8303e5e1de4b181ce40545279719e6069 Jan   2017 heat     hold          Bozeman             False   \n",
       "f404081c89442a6b4699e93e43dc4bb72f9e2dc1 Jan   2017 heat     auto          Great Falls         False   \n",
       "                                                             hold          Great Falls         False   \n",
       "\n",
       "                                                                                        Has_a_Heat_Pump  \n",
       "Identifier                               Month Year HvacMode CalendarEvent City                          \n",
       "23d834c190c3686c02c030ee0bf42003872bd591 Jan   2017 heat     hold          Bozeman                False  \n",
       "29f5966d7808882d6c76a0a2d7091c18b3587704 Jan   2017 heat     auto          Bozeman                False  \n",
       "                                                             hold          Bozeman                False  \n",
       "3e35e2d37f16eb3b72b2cbb145e6ed964ce01ca4 Jan   2017 auto     auto          Great Falls            False  \n",
       "                                                             hold          Great Falls            False  \n",
       "40f358a2efb733ab89fe3bb6e2f7b0280ccb4917 Jan   2017 heat     hold          Billings               False  \n",
       "5d1ff376e3fc7ee713a1138cbb661ddad9b1e257 Jan   2017 heat     auto          Bozeman                False  \n",
       "                                                             hold          Bozeman                False  \n",
       "7258db280a1da00d548cd8ea30f21c7d7d4932e0 Jan   2017 heat     auto          Bozeman                False  \n",
       "                                                             hold          Bozeman                False  \n",
       "9d4866ebcb99c98fc731f171a8933d0a437eff2c Jan   2017 heat     hold          Missoula                True  \n",
       "9ece7da8b39fee15b9ccaa8e095adf7d0cf2a2a6 Jan   2017 heat     auto          Missoula               False  \n",
       "                                                             hold          Missoula               False  \n",
       "a446833ed31f9039f78e43743749d949ab71cfe0 Jan   2017 auto     auto          Fort Benton            False  \n",
       "                                                             hold          Fort Benton            False  \n",
       "c90b4dfdcd0c12591663fdc2b2a3fdec6faced91 Jan   2017 auto     hold          Missoula               False  \n",
       "                                                    heat     auto          Missoula               False  \n",
       "                                                             hold          Missoula               False  \n",
       "c9a27b6d42a3b1e3d96263a706132ded7796fb6f Jan   2017 auto     auto          Great Falls            False  \n",
       "                                                             hold          Great Falls            False  \n",
       "e4ed8ab8303e5e1de4b181ce40545279719e6069 Jan   2017 heat     hold          Bozeman                False  \n",
       "f404081c89442a6b4699e93e43dc4bb72f9e2dc1 Jan   2017 heat     auto          Great Falls            False  \n",
       "                                                             hold          Great Falls            False  "
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "jan_2017_ave = jan_2017.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "jan_2017_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "jan_2017_ave.to_csv(\"data/day/MT/jan/jan_2017_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2018 January Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "jan_2018 = pd.read_csv(\"../data_large/MT-day/2018-jan-day-MT.csv\")\n",
    "\n",
    "# jan_2018"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>666c08e73f2bee116322fafbf2fc1b2972422b81</td>\n",
       "      <td>2018-01-20T15:55:00Z</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>672</td>\n",
       "      <td>737</td>\n",
       "      <td>623</td>\n",
       "      <td>MT</td>\n",
       "      <td>Bozeman</td>\n",
       "      <td>7</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0728208cff5242993896da88a882e076094b765f</td>\n",
       "      <td>2018-01-10T13:15:00Z</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>634</td>\n",
       "      <td>745</td>\n",
       "      <td>678</td>\n",
       "      <td>MT</td>\n",
       "      <td>Billings</td>\n",
       "      <td>50</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>037988cc24eaeaa08f7002dbc926d02c8f9c0f1c</td>\n",
       "      <td>2018-01-30T15:05:00Z</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>595</td>\n",
       "      <td>666</td>\n",
       "      <td>666</td>\n",
       "      <td>MT</td>\n",
       "      <td>Eureka</td>\n",
       "      <td>10</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>29f5966d7808882d6c76a0a2d7091c18b3587704</td>\n",
       "      <td>2018-01-15T19:45:00Z</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>727</td>\n",
       "      <td>805</td>\n",
       "      <td>745</td>\n",
       "      <td>MT</td>\n",
       "      <td>Bozeman</td>\n",
       "      <td>9</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>29f5966d7808882d6c76a0a2d7091c18b3587704</td>\n",
       "      <td>2018-01-11T18:20:00Z</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>734</td>\n",
       "      <td>785</td>\n",
       "      <td>735</td>\n",
       "      <td>MT</td>\n",
       "      <td>Bozeman</td>\n",
       "      <td>9</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16236</th>\n",
       "      <td>e4ed8ab8303e5e1de4b181ce40545279719e6069</td>\n",
       "      <td>2018-01-09T16:20:00Z</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>656</td>\n",
       "      <td>650</td>\n",
       "      <td>650</td>\n",
       "      <td>MT</td>\n",
       "      <td>Bozeman</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16237</th>\n",
       "      <td>e4ed8ab8303e5e1de4b181ce40545279719e6069</td>\n",
       "      <td>2018-01-13T13:55:00Z</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>654</td>\n",
       "      <td>650</td>\n",
       "      <td>650</td>\n",
       "      <td>MT</td>\n",
       "      <td>Bozeman</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16238</th>\n",
       "      <td>e4ed8ab8303e5e1de4b181ce40545279719e6069</td>\n",
       "      <td>2018-01-02T19:50:00Z</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>680</td>\n",
       "      <td>650</td>\n",
       "      <td>650</td>\n",
       "      <td>MT</td>\n",
       "      <td>Bozeman</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16239</th>\n",
       "      <td>e4ed8ab8303e5e1de4b181ce40545279719e6069</td>\n",
       "      <td>2018-01-08T19:30:00Z</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>657</td>\n",
       "      <td>650</td>\n",
       "      <td>650</td>\n",
       "      <td>MT</td>\n",
       "      <td>Bozeman</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16240</th>\n",
       "      <td>e4ed8ab8303e5e1de4b181ce40545279719e6069</td>\n",
       "      <td>2018-01-09T19:55:00Z</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>659</td>\n",
       "      <td>650</td>\n",
       "      <td>650</td>\n",
       "      <td>MT</td>\n",
       "      <td>Bozeman</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>16043 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                     Identifier             date_time  \\\n",
       "0      666c08e73f2bee116322fafbf2fc1b2972422b81  2018-01-20T15:55:00Z   \n",
       "1      0728208cff5242993896da88a882e076094b765f  2018-01-10T13:15:00Z   \n",
       "2      037988cc24eaeaa08f7002dbc926d02c8f9c0f1c  2018-01-30T15:05:00Z   \n",
       "3      29f5966d7808882d6c76a0a2d7091c18b3587704  2018-01-15T19:45:00Z   \n",
       "4      29f5966d7808882d6c76a0a2d7091c18b3587704  2018-01-11T18:20:00Z   \n",
       "...                                         ...                   ...   \n",
       "16236  e4ed8ab8303e5e1de4b181ce40545279719e6069  2018-01-09T16:20:00Z   \n",
       "16237  e4ed8ab8303e5e1de4b181ce40545279719e6069  2018-01-13T13:55:00Z   \n",
       "16238  e4ed8ab8303e5e1de4b181ce40545279719e6069  2018-01-02T19:50:00Z   \n",
       "16239  e4ed8ab8303e5e1de4b181ce40545279719e6069  2018-01-08T19:30:00Z   \n",
       "16240  e4ed8ab8303e5e1de4b181ce40545279719e6069  2018-01-09T19:55:00Z   \n",
       "\n",
       "      HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0         heat          hold               672                      737   \n",
       "1         heat          auto               634                      745   \n",
       "2         heat          hold               595                      666   \n",
       "3         auto          hold               727                      805   \n",
       "4         auto          hold               734                      785   \n",
       "...        ...           ...               ...                      ...   \n",
       "16236     heat          hold               656                      650   \n",
       "16237     heat          hold               654                      650   \n",
       "16238     heat          hold               680                      650   \n",
       "16239     heat          hold               657                      650   \n",
       "16240     heat          hold               659                      650   \n",
       "\n",
       "       TemperatureExpectedHeat ProvinceState      City  Age_of_Home__years_  \\\n",
       "0                          623            MT   Bozeman                    7   \n",
       "1                          678            MT  Billings                   50   \n",
       "2                          666            MT    Eureka                   10   \n",
       "3                          745            MT   Bozeman                    9   \n",
       "4                          735            MT   Bozeman                    9   \n",
       "...                        ...           ...       ...                  ...   \n",
       "16236                      650            MT   Bozeman                   10   \n",
       "16237                      650            MT   Bozeman                   10   \n",
       "16238                      650            MT   Bozeman                   10   \n",
       "16239                      650            MT   Bozeman                   10   \n",
       "16240                      650            MT   Bozeman                   10   \n",
       "\n",
       "       allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                 False         False            False   \n",
       "1                 False         False            False   \n",
       "2                  True         False             True   \n",
       "3                  True         False            False   \n",
       "4                  True         False            False   \n",
       "...                 ...           ...              ...   \n",
       "16236             False         False            False   \n",
       "16237             False         False            False   \n",
       "16238             False         False            False   \n",
       "16239             False         False            False   \n",
       "16240             False         False            False   \n",
       "\n",
       "      Auxilliary_Heat_Fuel_Type  \n",
       "0                           Gas  \n",
       "1                           Gas  \n",
       "2                      Electric  \n",
       "3                           Gas  \n",
       "4                           Gas  \n",
       "...                         ...  \n",
       "16236                       Gas  \n",
       "16237                       Gas  \n",
       "16238                       Gas  \n",
       "16239                       Gas  \n",
       "16240                       Gas  \n",
       "\n",
       "[16043 rows x 14 columns]"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "jan_2018.drop(jan_2018[jan_2018['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "jan_2018.drop(jan_2018[jan_2018['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "jan_2018.drop(jan_2018[jan_2018['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "jan_2018.drop(jan_2018[jan_2018['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "jan_2018"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "jan_2018[\"Year\"] = \"2018\"\n",
    "jan_2018[\"Month\"] = \"Jan\"\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "jan_2018 = jan_2018.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "jan_2018_ave = jan_2018.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# jan_2018_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "jan_2018_ave.to_csv(\"data/day/MT/jan/jan_2018_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2019 January Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "jan_2019 = pd.read_csv(\"../data_large/MT-day/2019-jan-day-MT.csv\")\n",
    "\n",
    "# jan_2019"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>23d834c190c3686c02c030ee0bf42003872bd591</td>\n",
       "      <td>2019-01-05 18:05:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>706</td>\n",
       "      <td>775</td>\n",
       "      <td>661</td>\n",
       "      <td>MT</td>\n",
       "      <td>Bozeman</td>\n",
       "      <td>25</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>34fbbdfdbc62b907f2cb4f26559df29b230273e3</td>\n",
       "      <td>2019-01-02 16:25:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>687</td>\n",
       "      <td>695</td>\n",
       "      <td>684</td>\n",
       "      <td>MT</td>\n",
       "      <td>Missoula</td>\n",
       "      <td>25</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>34fbbdfdbc62b907f2cb4f26559df29b230273e3</td>\n",
       "      <td>2019-01-03 14:05:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>679</td>\n",
       "      <td>687</td>\n",
       "      <td>687</td>\n",
       "      <td>MT</td>\n",
       "      <td>Missoula</td>\n",
       "      <td>25</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>34fbbdfdbc62b907f2cb4f26559df29b230273e3</td>\n",
       "      <td>2019-01-03 13:45:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>683</td>\n",
       "      <td>687</td>\n",
       "      <td>687</td>\n",
       "      <td>MT</td>\n",
       "      <td>Missoula</td>\n",
       "      <td>25</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>34fbbdfdbc62b907f2cb4f26559df29b230273e3</td>\n",
       "      <td>2019-01-30 15:55:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>684</td>\n",
       "      <td>691</td>\n",
       "      <td>688</td>\n",
       "      <td>MT</td>\n",
       "      <td>Missoula</td>\n",
       "      <td>25</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33951</th>\n",
       "      <td>98427a2f54b32505d3f1a3e9dfe4acfb78346762</td>\n",
       "      <td>2019-01-24 14:15:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>758</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>MT</td>\n",
       "      <td>Great Falls</td>\n",
       "      <td>70</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33952</th>\n",
       "      <td>98427a2f54b32505d3f1a3e9dfe4acfb78346762</td>\n",
       "      <td>2019-01-14 18:55:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>747</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>MT</td>\n",
       "      <td>Great Falls</td>\n",
       "      <td>70</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33953</th>\n",
       "      <td>98427a2f54b32505d3f1a3e9dfe4acfb78346762</td>\n",
       "      <td>2019-01-13 14:20:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>755</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>MT</td>\n",
       "      <td>Great Falls</td>\n",
       "      <td>70</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33954</th>\n",
       "      <td>98427a2f54b32505d3f1a3e9dfe4acfb78346762</td>\n",
       "      <td>2019-01-17 15:05:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>758</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>MT</td>\n",
       "      <td>Great Falls</td>\n",
       "      <td>70</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33955</th>\n",
       "      <td>98427a2f54b32505d3f1a3e9dfe4acfb78346762</td>\n",
       "      <td>2019-01-02 15:00:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>754</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>MT</td>\n",
       "      <td>Great Falls</td>\n",
       "      <td>70</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>26041 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                     Identifier                date_time  \\\n",
       "0      23d834c190c3686c02c030ee0bf42003872bd591  2019-01-05 18:05:00 UTC   \n",
       "1      34fbbdfdbc62b907f2cb4f26559df29b230273e3  2019-01-02 16:25:00 UTC   \n",
       "2      34fbbdfdbc62b907f2cb4f26559df29b230273e3  2019-01-03 14:05:00 UTC   \n",
       "3      34fbbdfdbc62b907f2cb4f26559df29b230273e3  2019-01-03 13:45:00 UTC   \n",
       "4      34fbbdfdbc62b907f2cb4f26559df29b230273e3  2019-01-30 15:55:00 UTC   \n",
       "...                                         ...                      ...   \n",
       "33951  98427a2f54b32505d3f1a3e9dfe4acfb78346762  2019-01-24 14:15:00 UTC   \n",
       "33952  98427a2f54b32505d3f1a3e9dfe4acfb78346762  2019-01-14 18:55:00 UTC   \n",
       "33953  98427a2f54b32505d3f1a3e9dfe4acfb78346762  2019-01-13 14:20:00 UTC   \n",
       "33954  98427a2f54b32505d3f1a3e9dfe4acfb78346762  2019-01-17 15:05:00 UTC   \n",
       "33955  98427a2f54b32505d3f1a3e9dfe4acfb78346762  2019-01-02 15:00:00 UTC   \n",
       "\n",
       "      HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0         heat          hold               706                      775   \n",
       "1         heat          hold               687                      695   \n",
       "2         heat          hold               679                      687   \n",
       "3         heat          hold               683                      687   \n",
       "4         heat          hold               684                      691   \n",
       "...        ...           ...               ...                      ...   \n",
       "33951     heat          hold               758                      760   \n",
       "33952     heat          hold               747                      760   \n",
       "33953     heat          hold               755                      760   \n",
       "33954     heat          hold               758                      760   \n",
       "33955     heat          hold               754                      760   \n",
       "\n",
       "       TemperatureExpectedHeat ProvinceState         City  \\\n",
       "0                          661            MT      Bozeman   \n",
       "1                          684            MT     Missoula   \n",
       "2                          687            MT     Missoula   \n",
       "3                          687            MT     Missoula   \n",
       "4                          688            MT     Missoula   \n",
       "...                        ...           ...          ...   \n",
       "33951                      760            MT  Great Falls   \n",
       "33952                      760            MT  Great Falls   \n",
       "33953                      760            MT  Great Falls   \n",
       "33954                      760            MT  Great Falls   \n",
       "33955                      760            MT  Great Falls   \n",
       "\n",
       "       Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                       25             False         False            False   \n",
       "1                       25             False         False            False   \n",
       "2                       25             False         False            False   \n",
       "3                       25             False         False            False   \n",
       "4                       25             False         False            False   \n",
       "...                    ...               ...           ...              ...   \n",
       "33951                   70             False         False            False   \n",
       "33952                   70             False         False            False   \n",
       "33953                   70             False         False            False   \n",
       "33954                   70             False         False            False   \n",
       "33955                   70             False         False            False   \n",
       "\n",
       "      Auxilliary_Heat_Fuel_Type  \n",
       "0                           Gas  \n",
       "1                           Gas  \n",
       "2                           Gas  \n",
       "3                           Gas  \n",
       "4                           Gas  \n",
       "...                         ...  \n",
       "33951                       Gas  \n",
       "33952                       Gas  \n",
       "33953                       Gas  \n",
       "33954                       Gas  \n",
       "33955                       Gas  \n",
       "\n",
       "[26041 rows x 14 columns]"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "jan_2019.drop(jan_2019[jan_2019['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "jan_2019.drop(jan_2019[jan_2019['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "jan_2019.drop(jan_2019[jan_2019['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "jan_2019.drop(jan_2019[jan_2019['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "jan_2019"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "jan_2019[\"Year\"] = \"2019\"\n",
    "jan_2019[\"Month\"] = \"Jan\"\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "jan_2019 = jan_2019.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "jan_2019_ave = jan_2019.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# jan_2019_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "jan_2019_ave.to_csv(\"data/day/MT/jan/jan_2019_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2020 January Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "jan_2020 = pd.read_csv(\"../data_large/MT-day/2020-jan-day-MT.csv\")\n",
    "\n",
    "# jan_2020"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>b9a976f581bd8ee81013df76fbd5b52f926136dc</td>\n",
       "      <td>2020-01-11 18:15:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>700</td>\n",
       "      <td>748</td>\n",
       "      <td>677</td>\n",
       "      <td>MT</td>\n",
       "      <td>Belgrade</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>9539afa609d7ffff710c3308be33b6a4a4bbd8a8</td>\n",
       "      <td>2020-01-07 17:55:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>663</td>\n",
       "      <td>739</td>\n",
       "      <td>629</td>\n",
       "      <td>MT</td>\n",
       "      <td>Missoula</td>\n",
       "      <td>9</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>23d834c190c3686c02c030ee0bf42003872bd591</td>\n",
       "      <td>2020-01-15 19:50:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>694</td>\n",
       "      <td>776</td>\n",
       "      <td>662</td>\n",
       "      <td>MT</td>\n",
       "      <td>Bozeman</td>\n",
       "      <td>25</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>7e82f7bb0ad060bdb1e139bed8aa5d5f869cf054</td>\n",
       "      <td>2020-01-10 13:55:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>665</td>\n",
       "      <td>712</td>\n",
       "      <td>663</td>\n",
       "      <td>MT</td>\n",
       "      <td>East Helena</td>\n",
       "      <td>40</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>a6b5547e34405b7e11d145b6b81f37e2850b7cd9</td>\n",
       "      <td>2020-01-16 14:15:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>689</td>\n",
       "      <td>697</td>\n",
       "      <td>687</td>\n",
       "      <td>MT</td>\n",
       "      <td>Missoula</td>\n",
       "      <td>70</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>41387</th>\n",
       "      <td>98427a2f54b32505d3f1a3e9dfe4acfb78346762</td>\n",
       "      <td>2020-01-20 17:30:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>753</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>MT</td>\n",
       "      <td>Great Falls</td>\n",
       "      <td>70</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>41388</th>\n",
       "      <td>98427a2f54b32505d3f1a3e9dfe4acfb78346762</td>\n",
       "      <td>2020-01-03 17:45:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>753</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>MT</td>\n",
       "      <td>Great Falls</td>\n",
       "      <td>70</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>41389</th>\n",
       "      <td>98427a2f54b32505d3f1a3e9dfe4acfb78346762</td>\n",
       "      <td>2020-01-02 18:25:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>MT</td>\n",
       "      <td>Great Falls</td>\n",
       "      <td>70</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>41390</th>\n",
       "      <td>99fa8d150f6b0ed05de548014a2751d573270a6b</td>\n",
       "      <td>2020-01-06 14:40:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>670</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>MT</td>\n",
       "      <td>Corvallis</td>\n",
       "      <td>10</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>41391</th>\n",
       "      <td>99fa8d150f6b0ed05de548014a2751d573270a6b</td>\n",
       "      <td>2020-01-06 14:35:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>666</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>MT</td>\n",
       "      <td>Corvallis</td>\n",
       "      <td>10</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>34911 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                     Identifier                date_time  \\\n",
       "0      b9a976f581bd8ee81013df76fbd5b52f926136dc  2020-01-11 18:15:00 UTC   \n",
       "1      9539afa609d7ffff710c3308be33b6a4a4bbd8a8  2020-01-07 17:55:00 UTC   \n",
       "2      23d834c190c3686c02c030ee0bf42003872bd591  2020-01-15 19:50:00 UTC   \n",
       "3      7e82f7bb0ad060bdb1e139bed8aa5d5f869cf054  2020-01-10 13:55:00 UTC   \n",
       "4      a6b5547e34405b7e11d145b6b81f37e2850b7cd9  2020-01-16 14:15:00 UTC   \n",
       "...                                         ...                      ...   \n",
       "41387  98427a2f54b32505d3f1a3e9dfe4acfb78346762  2020-01-20 17:30:00 UTC   \n",
       "41388  98427a2f54b32505d3f1a3e9dfe4acfb78346762  2020-01-03 17:45:00 UTC   \n",
       "41389  98427a2f54b32505d3f1a3e9dfe4acfb78346762  2020-01-02 18:25:00 UTC   \n",
       "41390  99fa8d150f6b0ed05de548014a2751d573270a6b  2020-01-06 14:40:00 UTC   \n",
       "41391  99fa8d150f6b0ed05de548014a2751d573270a6b  2020-01-06 14:35:00 UTC   \n",
       "\n",
       "      HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0         heat          hold               700                      748   \n",
       "1         auto          hold               663                      739   \n",
       "2         heat          hold               694                      776   \n",
       "3         heat          hold               665                      712   \n",
       "4         heat          auto               689                      697   \n",
       "...        ...           ...               ...                      ...   \n",
       "41387     heat          hold               753                      760   \n",
       "41388     heat          hold               753                      760   \n",
       "41389     heat          hold               760                      760   \n",
       "41390     heat          auto               670                      760   \n",
       "41391     heat          auto               666                      760   \n",
       "\n",
       "       TemperatureExpectedHeat ProvinceState         City  \\\n",
       "0                          677            MT     Belgrade   \n",
       "1                          629            MT     Missoula   \n",
       "2                          662            MT      Bozeman   \n",
       "3                          663            MT  East Helena   \n",
       "4                          687            MT     Missoula   \n",
       "...                        ...           ...          ...   \n",
       "41387                      760            MT  Great Falls   \n",
       "41388                      760            MT  Great Falls   \n",
       "41389                      760            MT  Great Falls   \n",
       "41390                      760            MT    Corvallis   \n",
       "41391                      760            MT    Corvallis   \n",
       "\n",
       "       Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                        0             False         False            False   \n",
       "1                        9             False         False            False   \n",
       "2                       25             False         False            False   \n",
       "3                       40             False         False            False   \n",
       "4                       70             False         False            False   \n",
       "...                    ...               ...           ...              ...   \n",
       "41387                   70             False         False            False   \n",
       "41388                   70             False         False            False   \n",
       "41389                   70             False         False            False   \n",
       "41390                   10              True         False            False   \n",
       "41391                   10              True         False            False   \n",
       "\n",
       "      Auxilliary_Heat_Fuel_Type  \n",
       "0                           Gas  \n",
       "1                           Gas  \n",
       "2                           Gas  \n",
       "3                           Gas  \n",
       "4                           Gas  \n",
       "...                         ...  \n",
       "41387                       Gas  \n",
       "41388                       Gas  \n",
       "41389                       Gas  \n",
       "41390                       Gas  \n",
       "41391                       Gas  \n",
       "\n",
       "[34911 rows x 14 columns]"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "jan_2020.drop(jan_2020[jan_2020['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "jan_2020.drop(jan_2020[jan_2020['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "jan_2020.drop(jan_2020[jan_2020['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "jan_2020.drop(jan_2020[jan_2020['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "jan_2020"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "jan_2020[\"Year\"] = \"2020\"\n",
    "jan_2020[\"Month\"] = \"Jan\"\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "jan_2020 = jan_2020.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "jan_2020_ave = jan_2020.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# jan_2020_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "jan_2020_ave.to_csv(\"data/day/MT/jan/jan_2020_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2021 January Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "jan_2021 = pd.read_csv(\"../data_large/MT-day/2021-jan-day-MT.csv\")\n",
    "\n",
    "# jan_2021"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>ca4dd6ff45035f8730f89c3b3ac556556fa37fef</td>\n",
       "      <td>2021-01-31T19:45:00Z</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>657</td>\n",
       "      <td>650</td>\n",
       "      <td>640</td>\n",
       "      <td>MT</td>\n",
       "      <td>Livingston</td>\n",
       "      <td>15</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>ca4dd6ff45035f8730f89c3b3ac556556fa37fef</td>\n",
       "      <td>2021-01-31T18:25:00Z</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>661</td>\n",
       "      <td>650</td>\n",
       "      <td>640</td>\n",
       "      <td>MT</td>\n",
       "      <td>Livingston</td>\n",
       "      <td>15</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>ca4dd6ff45035f8730f89c3b3ac556556fa37fef</td>\n",
       "      <td>2021-01-31T19:20:00Z</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>658</td>\n",
       "      <td>650</td>\n",
       "      <td>640</td>\n",
       "      <td>MT</td>\n",
       "      <td>Livingston</td>\n",
       "      <td>15</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3749c1cf564746f9904314c5d33b96db515c066b</td>\n",
       "      <td>2021-01-28T07:30:00Z</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>640</td>\n",
       "      <td>650</td>\n",
       "      <td>640</td>\n",
       "      <td>MT</td>\n",
       "      <td>Butte</td>\n",
       "      <td>118</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>ca4dd6ff45035f8730f89c3b3ac556556fa37fef</td>\n",
       "      <td>2021-01-31T18:45:00Z</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>660</td>\n",
       "      <td>650</td>\n",
       "      <td>640</td>\n",
       "      <td>MT</td>\n",
       "      <td>Livingston</td>\n",
       "      <td>15</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26929</th>\n",
       "      <td>b9a68c99fa994007bfbe7dfc018ec73539555478</td>\n",
       "      <td>2021-01-20T14:55:00Z</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>695</td>\n",
       "      <td>700</td>\n",
       "      <td>700</td>\n",
       "      <td>MT</td>\n",
       "      <td>Billings</td>\n",
       "      <td>60</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26930</th>\n",
       "      <td>b9a68c99fa994007bfbe7dfc018ec73539555478</td>\n",
       "      <td>2021-01-20T14:45:00Z</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>698</td>\n",
       "      <td>700</td>\n",
       "      <td>700</td>\n",
       "      <td>MT</td>\n",
       "      <td>Billings</td>\n",
       "      <td>60</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26931</th>\n",
       "      <td>b9a68c99fa994007bfbe7dfc018ec73539555478</td>\n",
       "      <td>2021-01-26T19:15:00Z</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>697</td>\n",
       "      <td>700</td>\n",
       "      <td>700</td>\n",
       "      <td>MT</td>\n",
       "      <td>Billings</td>\n",
       "      <td>60</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26932</th>\n",
       "      <td>b9a68c99fa994007bfbe7dfc018ec73539555478</td>\n",
       "      <td>2021-01-26T19:00:00Z</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>689</td>\n",
       "      <td>700</td>\n",
       "      <td>700</td>\n",
       "      <td>MT</td>\n",
       "      <td>Billings</td>\n",
       "      <td>60</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26933</th>\n",
       "      <td>bbc0d748de67c33d4c70a501da752924a90777ee</td>\n",
       "      <td>2021-01-19T18:40:00Z</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>695</td>\n",
       "      <td>750</td>\n",
       "      <td>700</td>\n",
       "      <td>MT</td>\n",
       "      <td>Kalispell</td>\n",
       "      <td>45</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>21814 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                     Identifier             date_time  \\\n",
       "0      ca4dd6ff45035f8730f89c3b3ac556556fa37fef  2021-01-31T19:45:00Z   \n",
       "1      ca4dd6ff45035f8730f89c3b3ac556556fa37fef  2021-01-31T18:25:00Z   \n",
       "2      ca4dd6ff45035f8730f89c3b3ac556556fa37fef  2021-01-31T19:20:00Z   \n",
       "3      3749c1cf564746f9904314c5d33b96db515c066b  2021-01-28T07:30:00Z   \n",
       "4      ca4dd6ff45035f8730f89c3b3ac556556fa37fef  2021-01-31T18:45:00Z   \n",
       "...                                         ...                   ...   \n",
       "26929  b9a68c99fa994007bfbe7dfc018ec73539555478  2021-01-20T14:55:00Z   \n",
       "26930  b9a68c99fa994007bfbe7dfc018ec73539555478  2021-01-20T14:45:00Z   \n",
       "26931  b9a68c99fa994007bfbe7dfc018ec73539555478  2021-01-26T19:15:00Z   \n",
       "26932  b9a68c99fa994007bfbe7dfc018ec73539555478  2021-01-26T19:00:00Z   \n",
       "26933  bbc0d748de67c33d4c70a501da752924a90777ee  2021-01-19T18:40:00Z   \n",
       "\n",
       "      HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0         heat          hold               657                      650   \n",
       "1         heat          hold               661                      650   \n",
       "2         heat          hold               658                      650   \n",
       "3         heat          hold               640                      650   \n",
       "4         heat          hold               660                      650   \n",
       "...        ...           ...               ...                      ...   \n",
       "26929     heat          hold               695                      700   \n",
       "26930     heat          hold               698                      700   \n",
       "26931     heat          hold               697                      700   \n",
       "26932     heat          hold               689                      700   \n",
       "26933     heat          hold               695                      750   \n",
       "\n",
       "       TemperatureExpectedHeat ProvinceState        City  Age_of_Home__years_  \\\n",
       "0                          640            MT  Livingston                   15   \n",
       "1                          640            MT  Livingston                   15   \n",
       "2                          640            MT  Livingston                   15   \n",
       "3                          640            MT       Butte                  118   \n",
       "4                          640            MT  Livingston                   15   \n",
       "...                        ...           ...         ...                  ...   \n",
       "26929                      700            MT    Billings                   60   \n",
       "26930                      700            MT    Billings                   60   \n",
       "26931                      700            MT    Billings                   60   \n",
       "26932                      700            MT    Billings                   60   \n",
       "26933                      700            MT   Kalispell                   45   \n",
       "\n",
       "       allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                 False         False            False   \n",
       "1                 False         False            False   \n",
       "2                 False         False            False   \n",
       "3                 False         False            False   \n",
       "4                 False         False            False   \n",
       "...                 ...           ...              ...   \n",
       "26929             False         False            False   \n",
       "26930             False         False            False   \n",
       "26931             False         False            False   \n",
       "26932             False         False            False   \n",
       "26933              True         False            False   \n",
       "\n",
       "      Auxilliary_Heat_Fuel_Type  \n",
       "0                           Gas  \n",
       "1                           Gas  \n",
       "2                           Gas  \n",
       "3                           Gas  \n",
       "4                           Gas  \n",
       "...                         ...  \n",
       "26929                       Gas  \n",
       "26930                       Gas  \n",
       "26931                       Gas  \n",
       "26932                       Gas  \n",
       "26933                       Gas  \n",
       "\n",
       "[21814 rows x 14 columns]"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "jan_2021.drop(jan_2021[jan_2021['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "jan_2021.drop(jan_2021[jan_2021['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "jan_2021.drop(jan_2021[jan_2021['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "jan_2021.drop(jan_2021[jan_2021['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "jan_2021"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "jan_2021[\"Year\"] = \"2021\"\n",
    "jan_2021[\"Month\"] = \"Jan\"\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "jan_2021 = jan_2021.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "jan_2021_ave = jan_2021.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# jan_2021_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "jan_2021_ave.to_csv(\"data/day/MT/jan/jan_2021_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "\n",
    "### Combine month CSV Files \n",
    "1. Read in files in folders for each state\n",
    "2. Export as combined CSV"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create variable for files in directory\n",
    "files = [f for f in os.listdir(\"data/day/MT/jan/\") if f.endswith(\".csv\")]\n",
    "\n",
    "# files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>Month</th>\n",
       "      <th>Year</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>City</th>\n",
       "      <th>Temperature_ctrl_ave</th>\n",
       "      <th>TemperatureExpectedCool_ave</th>\n",
       "      <th>TemperatureExpectedHeat_ave</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>23d834c190c3686c02c030ee0bf42003872bd591</td>\n",
       "      <td>Jan</td>\n",
       "      <td>2017</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>Bozeman</td>\n",
       "      <td>692.737226</td>\n",
       "      <td>698.912409</td>\n",
       "      <td>697.160584</td>\n",
       "      <td>25.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>29f5966d7808882d6c76a0a2d7091c18b3587704</td>\n",
       "      <td>Jan</td>\n",
       "      <td>2017</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>Bozeman</td>\n",
       "      <td>690.696629</td>\n",
       "      <td>693.112360</td>\n",
       "      <td>692.719101</td>\n",
       "      <td>9.0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>29f5966d7808882d6c76a0a2d7091c18b3587704</td>\n",
       "      <td>Jan</td>\n",
       "      <td>2017</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>Bozeman</td>\n",
       "      <td>733.701031</td>\n",
       "      <td>737.855670</td>\n",
       "      <td>737.845361</td>\n",
       "      <td>9.0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3e35e2d37f16eb3b72b2cbb145e6ed964ce01ca4</td>\n",
       "      <td>Jan</td>\n",
       "      <td>2017</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>Great Falls</td>\n",
       "      <td>719.717391</td>\n",
       "      <td>783.188406</td>\n",
       "      <td>720.000000</td>\n",
       "      <td>45.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>3e35e2d37f16eb3b72b2cbb145e6ed964ce01ca4</td>\n",
       "      <td>Jan</td>\n",
       "      <td>2017</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>Great Falls</td>\n",
       "      <td>724.250000</td>\n",
       "      <td>780.681818</td>\n",
       "      <td>716.590909</td>\n",
       "      <td>45.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>40</th>\n",
       "      <td>f404081c89442a6b4699e93e43dc4bb72f9e2dc1</td>\n",
       "      <td>Jan</td>\n",
       "      <td>2021</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>Great Falls</td>\n",
       "      <td>695.554745</td>\n",
       "      <td>747.761861</td>\n",
       "      <td>697.761861</td>\n",
       "      <td>55.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>41</th>\n",
       "      <td>f469a99acb054dbddd08e713d682acf5035c42df</td>\n",
       "      <td>Jan</td>\n",
       "      <td>2021</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>Helena</td>\n",
       "      <td>678.380952</td>\n",
       "      <td>680.000000</td>\n",
       "      <td>680.000000</td>\n",
       "      <td>49.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>42</th>\n",
       "      <td>f5101dfe9a0543a0dd3d8b1adbec2dd33f3047a8</td>\n",
       "      <td>Jan</td>\n",
       "      <td>2021</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>Bozeman</td>\n",
       "      <td>692.170213</td>\n",
       "      <td>699.297872</td>\n",
       "      <td>698.765957</td>\n",
       "      <td>90.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>43</th>\n",
       "      <td>fd93662943beeb4603fd63aacbbbb7abb3183470</td>\n",
       "      <td>Jan</td>\n",
       "      <td>2021</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>NASHUA</td>\n",
       "      <td>695.613636</td>\n",
       "      <td>674.136364</td>\n",
       "      <td>672.954545</td>\n",
       "      <td>30.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>44</th>\n",
       "      <td>fec2ec7ae8c5ee7ae93fe16b022bc4c1fc6f2931</td>\n",
       "      <td>Jan</td>\n",
       "      <td>2021</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>Kalispell</td>\n",
       "      <td>698.332000</td>\n",
       "      <td>708.688000</td>\n",
       "      <td>688.688000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>259 rows × 13 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                  Identifier Month  Year HvacMode  \\\n",
       "0   23d834c190c3686c02c030ee0bf42003872bd591   Jan  2017     heat   \n",
       "1   29f5966d7808882d6c76a0a2d7091c18b3587704   Jan  2017     heat   \n",
       "2   29f5966d7808882d6c76a0a2d7091c18b3587704   Jan  2017     heat   \n",
       "3   3e35e2d37f16eb3b72b2cbb145e6ed964ce01ca4   Jan  2017     auto   \n",
       "4   3e35e2d37f16eb3b72b2cbb145e6ed964ce01ca4   Jan  2017     auto   \n",
       "..                                       ...   ...   ...      ...   \n",
       "40  f404081c89442a6b4699e93e43dc4bb72f9e2dc1   Jan  2021     auto   \n",
       "41  f469a99acb054dbddd08e713d682acf5035c42df   Jan  2021     heat   \n",
       "42  f5101dfe9a0543a0dd3d8b1adbec2dd33f3047a8   Jan  2021     heat   \n",
       "43  fd93662943beeb4603fd63aacbbbb7abb3183470   Jan  2021     heat   \n",
       "44  fec2ec7ae8c5ee7ae93fe16b022bc4c1fc6f2931   Jan  2021     auto   \n",
       "\n",
       "   CalendarEvent         City  Temperature_ctrl_ave  \\\n",
       "0           hold      Bozeman            692.737226   \n",
       "1           auto      Bozeman            690.696629   \n",
       "2           hold      Bozeman            733.701031   \n",
       "3           auto  Great Falls            719.717391   \n",
       "4           hold  Great Falls            724.250000   \n",
       "..           ...          ...                   ...   \n",
       "40          hold  Great Falls            695.554745   \n",
       "41          hold       Helena            678.380952   \n",
       "42          hold      Bozeman            692.170213   \n",
       "43          hold       NASHUA            695.613636   \n",
       "44          hold    Kalispell            698.332000   \n",
       "\n",
       "    TemperatureExpectedCool_ave  TemperatureExpectedHeat_ave  \\\n",
       "0                    698.912409                   697.160584   \n",
       "1                    693.112360                   692.719101   \n",
       "2                    737.855670                   737.845361   \n",
       "3                    783.188406                   720.000000   \n",
       "4                    780.681818                   716.590909   \n",
       "..                          ...                          ...   \n",
       "40                   747.761861                   697.761861   \n",
       "41                   680.000000                   680.000000   \n",
       "42                   699.297872                   698.765957   \n",
       "43                   674.136364                   672.954545   \n",
       "44                   708.688000                   688.688000   \n",
       "\n",
       "    Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \n",
       "0                  25.0             False         False            False  \n",
       "1                   9.0              True         False            False  \n",
       "2                   9.0              True         False            False  \n",
       "3                  45.0             False         False            False  \n",
       "4                  45.0             False         False            False  \n",
       "..                  ...               ...           ...              ...  \n",
       "40                 55.0             False         False            False  \n",
       "41                 49.0             False         False            False  \n",
       "42                 90.0             False         False            False  \n",
       "43                 30.0             False         False            False  \n",
       "44                  0.0              True         False            False  \n",
       "\n",
       "[259 rows x 13 columns]"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# https://stacMToverflow.com/questions/63886787/how-to-create-a-dataframe-from-multiple-csv-files\n",
    "# Initialize an empty dataframe\n",
    "MT_jan = pd.DataFrame()\n",
    "\n",
    "# Iterate through files and contents, then concatenate the data into the dataframe\n",
    "for file in files:\n",
    "    df = pd.read_csv(\"data/day/MT/jan/\" + file)\n",
    "    MT_jan = pd.concat([MT_jan, df])\n",
    "    \n",
    "MT_jan"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "MT_jan.to_csv(\"Scraper_Output/State_Month_Day/MT/MT_jan.csv\", header=True, index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "\n",
    "## February"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2017 February Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "feb_2017 = pd.read_csv(\"../data_large/MT-day/2017-feb-day-MT.csv\")\n",
    "\n",
    "# feb_2017"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>f74a23b378011473f8b32b3fc2efbbb3b6a9e098</td>\n",
       "      <td>2017-02-28T15:20:00Z</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>699</td>\n",
       "      <td>706</td>\n",
       "      <td>694</td>\n",
       "      <td>MT</td>\n",
       "      <td>Billings</td>\n",
       "      <td>55</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>5d1ff376e3fc7ee713a1138cbb661ddad9b1e257</td>\n",
       "      <td>2017-02-27T18:20:00Z</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>619</td>\n",
       "      <td>741</td>\n",
       "      <td>635</td>\n",
       "      <td>MT</td>\n",
       "      <td>Bozeman</td>\n",
       "      <td>15</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>29f5966d7808882d6c76a0a2d7091c18b3587704</td>\n",
       "      <td>2017-02-02T14:20:00Z</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>731</td>\n",
       "      <td>735</td>\n",
       "      <td>735</td>\n",
       "      <td>MT</td>\n",
       "      <td>Bozeman</td>\n",
       "      <td>9</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>29f5966d7808882d6c76a0a2d7091c18b3587704</td>\n",
       "      <td>2017-02-02T15:55:00Z</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>734</td>\n",
       "      <td>735</td>\n",
       "      <td>735</td>\n",
       "      <td>MT</td>\n",
       "      <td>Bozeman</td>\n",
       "      <td>9</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>29f5966d7808882d6c76a0a2d7091c18b3587704</td>\n",
       "      <td>2017-02-06T18:35:00Z</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>714</td>\n",
       "      <td>715</td>\n",
       "      <td>715</td>\n",
       "      <td>MT</td>\n",
       "      <td>Bozeman</td>\n",
       "      <td>9</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7222</th>\n",
       "      <td>29f5966d7808882d6c76a0a2d7091c18b3587704</td>\n",
       "      <td>2017-02-08T19:20:00Z</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>735</td>\n",
       "      <td>740</td>\n",
       "      <td>740</td>\n",
       "      <td>MT</td>\n",
       "      <td>Bozeman</td>\n",
       "      <td>9</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7223</th>\n",
       "      <td>29f5966d7808882d6c76a0a2d7091c18b3587704</td>\n",
       "      <td>2017-02-08T19:25:00Z</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>740</td>\n",
       "      <td>740</td>\n",
       "      <td>740</td>\n",
       "      <td>MT</td>\n",
       "      <td>Bozeman</td>\n",
       "      <td>9</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7224</th>\n",
       "      <td>29f5966d7808882d6c76a0a2d7091c18b3587704</td>\n",
       "      <td>2017-02-08T19:15:00Z</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>731</td>\n",
       "      <td>740</td>\n",
       "      <td>740</td>\n",
       "      <td>MT</td>\n",
       "      <td>Bozeman</td>\n",
       "      <td>9</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7225</th>\n",
       "      <td>29f5966d7808882d6c76a0a2d7091c18b3587704</td>\n",
       "      <td>2017-02-08T19:50:00Z</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>735</td>\n",
       "      <td>740</td>\n",
       "      <td>740</td>\n",
       "      <td>MT</td>\n",
       "      <td>Bozeman</td>\n",
       "      <td>9</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7226</th>\n",
       "      <td>29f5966d7808882d6c76a0a2d7091c18b3587704</td>\n",
       "      <td>2017-02-08T19:40:00Z</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>737</td>\n",
       "      <td>740</td>\n",
       "      <td>740</td>\n",
       "      <td>MT</td>\n",
       "      <td>Bozeman</td>\n",
       "      <td>9</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>4978 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                    Identifier             date_time HvacMode  \\\n",
       "0     f74a23b378011473f8b32b3fc2efbbb3b6a9e098  2017-02-28T15:20:00Z     heat   \n",
       "1     5d1ff376e3fc7ee713a1138cbb661ddad9b1e257  2017-02-27T18:20:00Z     heat   \n",
       "2     29f5966d7808882d6c76a0a2d7091c18b3587704  2017-02-02T14:20:00Z     heat   \n",
       "3     29f5966d7808882d6c76a0a2d7091c18b3587704  2017-02-02T15:55:00Z     heat   \n",
       "4     29f5966d7808882d6c76a0a2d7091c18b3587704  2017-02-06T18:35:00Z     heat   \n",
       "...                                        ...                   ...      ...   \n",
       "7222  29f5966d7808882d6c76a0a2d7091c18b3587704  2017-02-08T19:20:00Z     heat   \n",
       "7223  29f5966d7808882d6c76a0a2d7091c18b3587704  2017-02-08T19:25:00Z     heat   \n",
       "7224  29f5966d7808882d6c76a0a2d7091c18b3587704  2017-02-08T19:15:00Z     heat   \n",
       "7225  29f5966d7808882d6c76a0a2d7091c18b3587704  2017-02-08T19:50:00Z     heat   \n",
       "7226  29f5966d7808882d6c76a0a2d7091c18b3587704  2017-02-08T19:40:00Z     heat   \n",
       "\n",
       "     CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0             auto               699                      706   \n",
       "1             auto               619                      741   \n",
       "2             hold               731                      735   \n",
       "3             hold               734                      735   \n",
       "4             hold               714                      715   \n",
       "...            ...               ...                      ...   \n",
       "7222          hold               735                      740   \n",
       "7223          hold               740                      740   \n",
       "7224          hold               731                      740   \n",
       "7225          hold               735                      740   \n",
       "7226          hold               737                      740   \n",
       "\n",
       "      TemperatureExpectedHeat ProvinceState      City  Age_of_Home__years_  \\\n",
       "0                         694            MT  Billings                   55   \n",
       "1                         635            MT   Bozeman                   15   \n",
       "2                         735            MT   Bozeman                    9   \n",
       "3                         735            MT   Bozeman                    9   \n",
       "4                         715            MT   Bozeman                    9   \n",
       "...                       ...           ...       ...                  ...   \n",
       "7222                      740            MT   Bozeman                    9   \n",
       "7223                      740            MT   Bozeman                    9   \n",
       "7224                      740            MT   Bozeman                    9   \n",
       "7225                      740            MT   Bozeman                    9   \n",
       "7226                      740            MT   Bozeman                    9   \n",
       "\n",
       "      allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                False         False            False   \n",
       "1                 True         False            False   \n",
       "2                 True         False            False   \n",
       "3                 True         False            False   \n",
       "4                 True         False            False   \n",
       "...                ...           ...              ...   \n",
       "7222              True         False            False   \n",
       "7223              True         False            False   \n",
       "7224              True         False            False   \n",
       "7225              True         False            False   \n",
       "7226              True         False            False   \n",
       "\n",
       "     Auxilliary_Heat_Fuel_Type  \n",
       "0                          Gas  \n",
       "1                          Gas  \n",
       "2                          Gas  \n",
       "3                          Gas  \n",
       "4                          Gas  \n",
       "...                        ...  \n",
       "7222                       Gas  \n",
       "7223                       Gas  \n",
       "7224                       Gas  \n",
       "7225                       Gas  \n",
       "7226                       Gas  \n",
       "\n",
       "[4978 rows x 14 columns]"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "feb_2017.drop(feb_2017[feb_2017['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "feb_2017.drop(feb_2017[feb_2017['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "feb_2017.drop(feb_2017[feb_2017['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "feb_2017.drop(feb_2017[feb_2017['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "feb_2017"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "feb_2017[\"Year\"] = \"2017\"\n",
    "feb_2017[\"Month\"] = \"feb\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "feb_2017 = feb_2017.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "feb_2017_ave = feb_2017.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# feb_2017_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "feb_2017_ave.to_csv(\"data/day/MT/feb/feb_2017_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2018 February Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "feb_2018 = pd.read_csv(\"../data_large/MT-day/2018-feb-day-MT.csv\")\n",
    "\n",
    "# feb_2018"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>a9250271635d87fc0b19dbef1fb1508e9a6f5630</td>\n",
       "      <td>2018-02-13T14:35:00Z</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>626</td>\n",
       "      <td>790</td>\n",
       "      <td>790</td>\n",
       "      <td>MT</td>\n",
       "      <td>Butte</td>\n",
       "      <td>100</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>a9250271635d87fc0b19dbef1fb1508e9a6f5630</td>\n",
       "      <td>2018-02-13T14:20:00Z</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>626</td>\n",
       "      <td>790</td>\n",
       "      <td>790</td>\n",
       "      <td>MT</td>\n",
       "      <td>Butte</td>\n",
       "      <td>100</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>a9250271635d87fc0b19dbef1fb1508e9a6f5630</td>\n",
       "      <td>2018-02-13T13:45:00Z</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>616</td>\n",
       "      <td>790</td>\n",
       "      <td>790</td>\n",
       "      <td>MT</td>\n",
       "      <td>Butte</td>\n",
       "      <td>100</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5dda6449bb5575fdf2a538f8208a3556b55c447b</td>\n",
       "      <td>2018-02-15T13:55:00Z</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>682</td>\n",
       "      <td>686</td>\n",
       "      <td>686</td>\n",
       "      <td>MT</td>\n",
       "      <td>Helena</td>\n",
       "      <td>10</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>a9250271635d87fc0b19dbef1fb1508e9a6f5630</td>\n",
       "      <td>2018-02-13T13:30:00Z</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>609</td>\n",
       "      <td>790</td>\n",
       "      <td>790</td>\n",
       "      <td>MT</td>\n",
       "      <td>Butte</td>\n",
       "      <td>100</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17624</th>\n",
       "      <td>29f5966d7808882d6c76a0a2d7091c18b3587704</td>\n",
       "      <td>2018-02-05T19:55:00Z</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>751</td>\n",
       "      <td>750</td>\n",
       "      <td>750</td>\n",
       "      <td>MT</td>\n",
       "      <td>Bozeman</td>\n",
       "      <td>9</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17625</th>\n",
       "      <td>29f5966d7808882d6c76a0a2d7091c18b3587704</td>\n",
       "      <td>2018-02-19T19:20:00Z</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>730</td>\n",
       "      <td>750</td>\n",
       "      <td>750</td>\n",
       "      <td>MT</td>\n",
       "      <td>Bozeman</td>\n",
       "      <td>9</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17626</th>\n",
       "      <td>29f5966d7808882d6c76a0a2d7091c18b3587704</td>\n",
       "      <td>2018-02-19T19:40:00Z</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>749</td>\n",
       "      <td>750</td>\n",
       "      <td>750</td>\n",
       "      <td>MT</td>\n",
       "      <td>Bozeman</td>\n",
       "      <td>9</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17627</th>\n",
       "      <td>29f5966d7808882d6c76a0a2d7091c18b3587704</td>\n",
       "      <td>2018-02-19T19:45:00Z</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>748</td>\n",
       "      <td>750</td>\n",
       "      <td>750</td>\n",
       "      <td>MT</td>\n",
       "      <td>Bozeman</td>\n",
       "      <td>9</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17628</th>\n",
       "      <td>29f5966d7808882d6c76a0a2d7091c18b3587704</td>\n",
       "      <td>2018-02-05T18:55:00Z</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>731</td>\n",
       "      <td>750</td>\n",
       "      <td>750</td>\n",
       "      <td>MT</td>\n",
       "      <td>Bozeman</td>\n",
       "      <td>9</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>14691 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                     Identifier             date_time  \\\n",
       "0      a9250271635d87fc0b19dbef1fb1508e9a6f5630  2018-02-13T14:35:00Z   \n",
       "1      a9250271635d87fc0b19dbef1fb1508e9a6f5630  2018-02-13T14:20:00Z   \n",
       "2      a9250271635d87fc0b19dbef1fb1508e9a6f5630  2018-02-13T13:45:00Z   \n",
       "4      5dda6449bb5575fdf2a538f8208a3556b55c447b  2018-02-15T13:55:00Z   \n",
       "5      a9250271635d87fc0b19dbef1fb1508e9a6f5630  2018-02-13T13:30:00Z   \n",
       "...                                         ...                   ...   \n",
       "17624  29f5966d7808882d6c76a0a2d7091c18b3587704  2018-02-05T19:55:00Z   \n",
       "17625  29f5966d7808882d6c76a0a2d7091c18b3587704  2018-02-19T19:20:00Z   \n",
       "17626  29f5966d7808882d6c76a0a2d7091c18b3587704  2018-02-19T19:40:00Z   \n",
       "17627  29f5966d7808882d6c76a0a2d7091c18b3587704  2018-02-19T19:45:00Z   \n",
       "17628  29f5966d7808882d6c76a0a2d7091c18b3587704  2018-02-05T18:55:00Z   \n",
       "\n",
       "      HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0         heat          hold               626                      790   \n",
       "1         heat          hold               626                      790   \n",
       "2         heat          hold               616                      790   \n",
       "4         heat          hold               682                      686   \n",
       "5         heat          hold               609                      790   \n",
       "...        ...           ...               ...                      ...   \n",
       "17624     heat          hold               751                      750   \n",
       "17625     heat          hold               730                      750   \n",
       "17626     heat          hold               749                      750   \n",
       "17627     heat          hold               748                      750   \n",
       "17628     heat          hold               731                      750   \n",
       "\n",
       "       TemperatureExpectedHeat ProvinceState     City  Age_of_Home__years_  \\\n",
       "0                          790            MT    Butte                  100   \n",
       "1                          790            MT    Butte                  100   \n",
       "2                          790            MT    Butte                  100   \n",
       "4                          686            MT   Helena                   10   \n",
       "5                          790            MT    Butte                  100   \n",
       "...                        ...           ...      ...                  ...   \n",
       "17624                      750            MT  Bozeman                    9   \n",
       "17625                      750            MT  Bozeman                    9   \n",
       "17626                      750            MT  Bozeman                    9   \n",
       "17627                      750            MT  Bozeman                    9   \n",
       "17628                      750            MT  Bozeman                    9   \n",
       "\n",
       "       allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                 False         False            False   \n",
       "1                 False         False            False   \n",
       "2                 False         False            False   \n",
       "4                  True         False            False   \n",
       "5                 False         False            False   \n",
       "...                 ...           ...              ...   \n",
       "17624              True         False            False   \n",
       "17625              True         False            False   \n",
       "17626              True         False            False   \n",
       "17627              True         False            False   \n",
       "17628              True         False            False   \n",
       "\n",
       "      Auxilliary_Heat_Fuel_Type  \n",
       "0                           Gas  \n",
       "1                           Gas  \n",
       "2                           Gas  \n",
       "4                           Gas  \n",
       "5                           Gas  \n",
       "...                         ...  \n",
       "17624                       Gas  \n",
       "17625                       Gas  \n",
       "17626                       Gas  \n",
       "17627                       Gas  \n",
       "17628                       Gas  \n",
       "\n",
       "[14691 rows x 14 columns]"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "feb_2018.drop(feb_2018[feb_2018['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "feb_2018.drop(feb_2018[feb_2018['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "feb_2018.drop(feb_2018[feb_2018['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "feb_2018.drop(feb_2018[feb_2018['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "feb_2018"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "feb_2018[\"Year\"] = \"2018\"\n",
    "feb_2018[\"Month\"] = \"feb\"\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "feb_2018 = feb_2018.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "feb_2018_ave = feb_2018.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# feb_2018_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "feb_2018_ave.to_csv(\"data/day/MT/feb/feb_2018_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2019 February Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "feb_2019 = pd.read_csv(\"../data_large/MT-day/2019-feb-day-MT.csv\")\n",
    "\n",
    "# feb_2019"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>ca4dd6ff45035f8730f89c3b3ac556556fa37fef</td>\n",
       "      <td>2019-02-26T13:55:00Z</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>633</td>\n",
       "      <td>743</td>\n",
       "      <td>607</td>\n",
       "      <td>MT</td>\n",
       "      <td>Livingston</td>\n",
       "      <td>15</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2013b05eece28f2e08b8cd6da888a9b04dbdc854</td>\n",
       "      <td>2019-02-05T13:05:00Z</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>670</td>\n",
       "      <td>706</td>\n",
       "      <td>676</td>\n",
       "      <td>MT</td>\n",
       "      <td>Missoula</td>\n",
       "      <td>30</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>34fbbdfdbc62b907f2cb4f26559df29b230273e3</td>\n",
       "      <td>2019-02-01T14:00:00Z</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>672</td>\n",
       "      <td>677</td>\n",
       "      <td>677</td>\n",
       "      <td>MT</td>\n",
       "      <td>Missoula</td>\n",
       "      <td>25</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>23d834c190c3686c02c030ee0bf42003872bd591</td>\n",
       "      <td>2019-02-01T19:20:00Z</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>681</td>\n",
       "      <td>685</td>\n",
       "      <td>679</td>\n",
       "      <td>MT</td>\n",
       "      <td>Bozeman</td>\n",
       "      <td>25</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2013b05eece28f2e08b8cd6da888a9b04dbdc854</td>\n",
       "      <td>2019-02-27T13:20:00Z</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>682</td>\n",
       "      <td>718</td>\n",
       "      <td>687</td>\n",
       "      <td>MT</td>\n",
       "      <td>Missoula</td>\n",
       "      <td>30</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23787</th>\n",
       "      <td>98427a2f54b32505d3f1a3e9dfe4acfb78346762</td>\n",
       "      <td>2019-02-17T15:10:00Z</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>757</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>MT</td>\n",
       "      <td>Great Falls</td>\n",
       "      <td>70</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23788</th>\n",
       "      <td>e4ed8ab8303e5e1de4b181ce40545279719e6069</td>\n",
       "      <td>2019-02-16T14:05:00Z</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>761</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>MT</td>\n",
       "      <td>Bozeman</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23789</th>\n",
       "      <td>e4ed8ab8303e5e1de4b181ce40545279719e6069</td>\n",
       "      <td>2019-02-16T14:00:00Z</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>744</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>MT</td>\n",
       "      <td>Bozeman</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23790</th>\n",
       "      <td>e4ed8ab8303e5e1de4b181ce40545279719e6069</td>\n",
       "      <td>2019-02-16T13:55:00Z</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>733</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>MT</td>\n",
       "      <td>Bozeman</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23791</th>\n",
       "      <td>e4ed8ab8303e5e1de4b181ce40545279719e6069</td>\n",
       "      <td>2019-02-16T14:15:00Z</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>765</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>MT</td>\n",
       "      <td>Bozeman</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>19472 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                     Identifier             date_time  \\\n",
       "0      ca4dd6ff45035f8730f89c3b3ac556556fa37fef  2019-02-26T13:55:00Z   \n",
       "1      2013b05eece28f2e08b8cd6da888a9b04dbdc854  2019-02-05T13:05:00Z   \n",
       "2      34fbbdfdbc62b907f2cb4f26559df29b230273e3  2019-02-01T14:00:00Z   \n",
       "3      23d834c190c3686c02c030ee0bf42003872bd591  2019-02-01T19:20:00Z   \n",
       "4      2013b05eece28f2e08b8cd6da888a9b04dbdc854  2019-02-27T13:20:00Z   \n",
       "...                                         ...                   ...   \n",
       "23787  98427a2f54b32505d3f1a3e9dfe4acfb78346762  2019-02-17T15:10:00Z   \n",
       "23788  e4ed8ab8303e5e1de4b181ce40545279719e6069  2019-02-16T14:05:00Z   \n",
       "23789  e4ed8ab8303e5e1de4b181ce40545279719e6069  2019-02-16T14:00:00Z   \n",
       "23790  e4ed8ab8303e5e1de4b181ce40545279719e6069  2019-02-16T13:55:00Z   \n",
       "23791  e4ed8ab8303e5e1de4b181ce40545279719e6069  2019-02-16T14:15:00Z   \n",
       "\n",
       "      HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0         heat          auto               633                      743   \n",
       "1         heat          hold               670                      706   \n",
       "2         heat          hold               672                      677   \n",
       "3         heat          hold               681                      685   \n",
       "4         heat          hold               682                      718   \n",
       "...        ...           ...               ...                      ...   \n",
       "23787     heat          hold               757                      760   \n",
       "23788     heat          hold               761                      760   \n",
       "23789     heat          hold               744                      760   \n",
       "23790     heat          hold               733                      760   \n",
       "23791     heat          hold               765                      760   \n",
       "\n",
       "       TemperatureExpectedHeat ProvinceState         City  \\\n",
       "0                          607            MT   Livingston   \n",
       "1                          676            MT     Missoula   \n",
       "2                          677            MT     Missoula   \n",
       "3                          679            MT      Bozeman   \n",
       "4                          687            MT     Missoula   \n",
       "...                        ...           ...          ...   \n",
       "23787                      760            MT  Great Falls   \n",
       "23788                      760            MT      Bozeman   \n",
       "23789                      760            MT      Bozeman   \n",
       "23790                      760            MT      Bozeman   \n",
       "23791                      760            MT      Bozeman   \n",
       "\n",
       "       Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                       15             False         False            False   \n",
       "1                       30              True         False            False   \n",
       "2                       25             False         False            False   \n",
       "3                       25             False         False            False   \n",
       "4                       30              True         False            False   \n",
       "...                    ...               ...           ...              ...   \n",
       "23787                   70             False         False            False   \n",
       "23788                   10             False         False            False   \n",
       "23789                   10             False         False            False   \n",
       "23790                   10             False         False            False   \n",
       "23791                   10             False         False            False   \n",
       "\n",
       "      Auxilliary_Heat_Fuel_Type  \n",
       "0                           Gas  \n",
       "1                           Gas  \n",
       "2                           Gas  \n",
       "3                           Gas  \n",
       "4                           Gas  \n",
       "...                         ...  \n",
       "23787                       Gas  \n",
       "23788                       Gas  \n",
       "23789                       Gas  \n",
       "23790                       Gas  \n",
       "23791                       Gas  \n",
       "\n",
       "[19472 rows x 14 columns]"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "feb_2019.drop(feb_2019[feb_2019['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "feb_2019.drop(feb_2019[feb_2019['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "feb_2019.drop(feb_2019[feb_2019['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "feb_2019.drop(feb_2019[feb_2019['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "feb_2019"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "feb_2019[\"Year\"] = \"2019\"\n",
    "feb_2019[\"Month\"] = \"feb\"\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "feb_2019 = feb_2019.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "feb_2019_ave = feb_2019.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# feb_2019_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "feb_2019_ave.to_csv(\"data/day/MT/feb/feb_2019_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2020 February Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "feb_2020 = pd.read_csv(\"../data_large/MT-day/2020-feb-day-MT.csv\")\n",
    "\n",
    "# feb_2020"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>ca4dd6ff45035f8730f89c3b3ac556556fa37fef</td>\n",
       "      <td>2020-02-23 19:55:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>701</td>\n",
       "      <td>784</td>\n",
       "      <td>609</td>\n",
       "      <td>MT</td>\n",
       "      <td>Livingston</td>\n",
       "      <td>15</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>9539afa609d7ffff710c3308be33b6a4a4bbd8a8</td>\n",
       "      <td>2020-02-11 19:30:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>695</td>\n",
       "      <td>774</td>\n",
       "      <td>724</td>\n",
       "      <td>MT</td>\n",
       "      <td>Missoula</td>\n",
       "      <td>9</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>ca4dd6ff45035f8730f89c3b3ac556556fa37fef</td>\n",
       "      <td>2020-02-02 19:55:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>697</td>\n",
       "      <td>742</td>\n",
       "      <td>654</td>\n",
       "      <td>MT</td>\n",
       "      <td>Livingston</td>\n",
       "      <td>15</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>a6b5547e34405b7e11d145b6b81f37e2850b7cd9</td>\n",
       "      <td>2020-02-11 13:30:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>684</td>\n",
       "      <td>694</td>\n",
       "      <td>688</td>\n",
       "      <td>MT</td>\n",
       "      <td>Missoula</td>\n",
       "      <td>70</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>666c08e73f2bee116322fafbf2fc1b2972422b81</td>\n",
       "      <td>2020-02-02 15:55:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>705</td>\n",
       "      <td>777</td>\n",
       "      <td>701</td>\n",
       "      <td>MT</td>\n",
       "      <td>Bozeman</td>\n",
       "      <td>7</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>35147</th>\n",
       "      <td>98427a2f54b32505d3f1a3e9dfe4acfb78346762</td>\n",
       "      <td>2020-02-07 14:10:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>751</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>MT</td>\n",
       "      <td>Great Falls</td>\n",
       "      <td>70</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>35148</th>\n",
       "      <td>98427a2f54b32505d3f1a3e9dfe4acfb78346762</td>\n",
       "      <td>2020-02-05 19:20:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>751</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>MT</td>\n",
       "      <td>Great Falls</td>\n",
       "      <td>70</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>35149</th>\n",
       "      <td>98427a2f54b32505d3f1a3e9dfe4acfb78346762</td>\n",
       "      <td>2020-02-25 16:40:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>758</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>MT</td>\n",
       "      <td>Great Falls</td>\n",
       "      <td>70</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>35150</th>\n",
       "      <td>98427a2f54b32505d3f1a3e9dfe4acfb78346762</td>\n",
       "      <td>2020-02-28 18:40:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>758</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>MT</td>\n",
       "      <td>Great Falls</td>\n",
       "      <td>70</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>35151</th>\n",
       "      <td>98427a2f54b32505d3f1a3e9dfe4acfb78346762</td>\n",
       "      <td>2020-02-07 13:40:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>753</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>MT</td>\n",
       "      <td>Great Falls</td>\n",
       "      <td>70</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>30139 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                     Identifier                date_time  \\\n",
       "0      ca4dd6ff45035f8730f89c3b3ac556556fa37fef  2020-02-23 19:55:00 UTC   \n",
       "1      9539afa609d7ffff710c3308be33b6a4a4bbd8a8  2020-02-11 19:30:00 UTC   \n",
       "2      ca4dd6ff45035f8730f89c3b3ac556556fa37fef  2020-02-02 19:55:00 UTC   \n",
       "3      a6b5547e34405b7e11d145b6b81f37e2850b7cd9  2020-02-11 13:30:00 UTC   \n",
       "4      666c08e73f2bee116322fafbf2fc1b2972422b81  2020-02-02 15:55:00 UTC   \n",
       "...                                         ...                      ...   \n",
       "35147  98427a2f54b32505d3f1a3e9dfe4acfb78346762  2020-02-07 14:10:00 UTC   \n",
       "35148  98427a2f54b32505d3f1a3e9dfe4acfb78346762  2020-02-05 19:20:00 UTC   \n",
       "35149  98427a2f54b32505d3f1a3e9dfe4acfb78346762  2020-02-25 16:40:00 UTC   \n",
       "35150  98427a2f54b32505d3f1a3e9dfe4acfb78346762  2020-02-28 18:40:00 UTC   \n",
       "35151  98427a2f54b32505d3f1a3e9dfe4acfb78346762  2020-02-07 13:40:00 UTC   \n",
       "\n",
       "      HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0         heat          auto               701                      784   \n",
       "1         auto          hold               695                      774   \n",
       "2         heat          auto               697                      742   \n",
       "3         heat          hold               684                      694   \n",
       "4         auto          hold               705                      777   \n",
       "...        ...           ...               ...                      ...   \n",
       "35147     heat          hold               751                      760   \n",
       "35148     heat          hold               751                      760   \n",
       "35149     heat          hold               758                      760   \n",
       "35150     heat          hold               758                      760   \n",
       "35151     heat          hold               753                      760   \n",
       "\n",
       "       TemperatureExpectedHeat ProvinceState         City  \\\n",
       "0                          609            MT   Livingston   \n",
       "1                          724            MT     Missoula   \n",
       "2                          654            MT   Livingston   \n",
       "3                          688            MT     Missoula   \n",
       "4                          701            MT      Bozeman   \n",
       "...                        ...           ...          ...   \n",
       "35147                      760            MT  Great Falls   \n",
       "35148                      760            MT  Great Falls   \n",
       "35149                      760            MT  Great Falls   \n",
       "35150                      760            MT  Great Falls   \n",
       "35151                      760            MT  Great Falls   \n",
       "\n",
       "       Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                       15             False         False            False   \n",
       "1                        9             False         False            False   \n",
       "2                       15             False         False            False   \n",
       "3                       70             False         False            False   \n",
       "4                        7             False         False            False   \n",
       "...                    ...               ...           ...              ...   \n",
       "35147                   70             False         False            False   \n",
       "35148                   70             False         False            False   \n",
       "35149                   70             False         False            False   \n",
       "35150                   70             False         False            False   \n",
       "35151                   70             False         False            False   \n",
       "\n",
       "      Auxilliary_Heat_Fuel_Type  \n",
       "0                           Gas  \n",
       "1                           Gas  \n",
       "2                           Gas  \n",
       "3                           Gas  \n",
       "4                           Gas  \n",
       "...                         ...  \n",
       "35147                       Gas  \n",
       "35148                       Gas  \n",
       "35149                       Gas  \n",
       "35150                       Gas  \n",
       "35151                       Gas  \n",
       "\n",
       "[30139 rows x 14 columns]"
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "feb_2020.drop(feb_2020[feb_2020['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "feb_2020.drop(feb_2020[feb_2020['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "feb_2020.drop(feb_2020[feb_2020['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "feb_2020.drop(feb_2020[feb_2020['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "feb_2020"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "feb_2020[\"Year\"] = \"2020\"\n",
    "feb_2020[\"Month\"] = \"feb\"\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "feb_2020 = feb_2020.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "feb_2020_ave = feb_2020.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# feb_2020_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "feb_2020_ave.to_csv(\"data/day/MT/feb/feb_2020_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2021 February Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "feb_2021 = pd.read_csv(\"../data_large/MT-day/2021-feb-day-MT.csv\")\n",
    "\n",
    "# feb_2021"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>f74a23b378011473f8b32b3fc2efbbb3b6a9e098</td>\n",
       "      <td>2021-02-10T15:55:00Z</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>698</td>\n",
       "      <td>709</td>\n",
       "      <td>709</td>\n",
       "      <td>MT</td>\n",
       "      <td>Billings</td>\n",
       "      <td>55</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>23d834c190c3686c02c030ee0bf42003872bd591</td>\n",
       "      <td>2021-02-12T19:05:00Z</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>699</td>\n",
       "      <td>790</td>\n",
       "      <td>790</td>\n",
       "      <td>MT</td>\n",
       "      <td>Bozeman</td>\n",
       "      <td>25</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3749c1cf564746f9904314c5d33b96db515c066b</td>\n",
       "      <td>2021-02-23T07:50:00Z</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>640</td>\n",
       "      <td>650</td>\n",
       "      <td>638</td>\n",
       "      <td>MT</td>\n",
       "      <td>Butte</td>\n",
       "      <td>118</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>34fbbdfdbc62b907f2cb4f26559df29b230273e3</td>\n",
       "      <td>2021-02-22T17:20:00Z</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>696</td>\n",
       "      <td>702</td>\n",
       "      <td>699</td>\n",
       "      <td>MT</td>\n",
       "      <td>Missoula</td>\n",
       "      <td>25</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>666c08e73f2bee116322fafbf2fc1b2972422b81</td>\n",
       "      <td>2021-02-17T14:55:00Z</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>686</td>\n",
       "      <td>727</td>\n",
       "      <td>682</td>\n",
       "      <td>MT</td>\n",
       "      <td>Bozeman</td>\n",
       "      <td>7</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28765</th>\n",
       "      <td>b9a68c99fa994007bfbe7dfc018ec73539555478</td>\n",
       "      <td>2021-02-17T16:35:00Z</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>698</td>\n",
       "      <td>700</td>\n",
       "      <td>700</td>\n",
       "      <td>MT</td>\n",
       "      <td>Billings</td>\n",
       "      <td>60</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28766</th>\n",
       "      <td>b9a68c99fa994007bfbe7dfc018ec73539555478</td>\n",
       "      <td>2021-02-20T15:55:00Z</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>695</td>\n",
       "      <td>700</td>\n",
       "      <td>700</td>\n",
       "      <td>MT</td>\n",
       "      <td>Billings</td>\n",
       "      <td>60</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28767</th>\n",
       "      <td>b9a68c99fa994007bfbe7dfc018ec73539555478</td>\n",
       "      <td>2021-02-19T19:55:00Z</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>696</td>\n",
       "      <td>700</td>\n",
       "      <td>700</td>\n",
       "      <td>MT</td>\n",
       "      <td>Billings</td>\n",
       "      <td>60</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28768</th>\n",
       "      <td>b9a68c99fa994007bfbe7dfc018ec73539555478</td>\n",
       "      <td>2021-02-10T16:05:00Z</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>699</td>\n",
       "      <td>700</td>\n",
       "      <td>700</td>\n",
       "      <td>MT</td>\n",
       "      <td>Billings</td>\n",
       "      <td>60</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28769</th>\n",
       "      <td>b9a68c99fa994007bfbe7dfc018ec73539555478</td>\n",
       "      <td>2021-02-19T19:10:00Z</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>698</td>\n",
       "      <td>700</td>\n",
       "      <td>700</td>\n",
       "      <td>MT</td>\n",
       "      <td>Billings</td>\n",
       "      <td>60</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>24578 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                     Identifier             date_time  \\\n",
       "0      f74a23b378011473f8b32b3fc2efbbb3b6a9e098  2021-02-10T15:55:00Z   \n",
       "1      23d834c190c3686c02c030ee0bf42003872bd591  2021-02-12T19:05:00Z   \n",
       "2      3749c1cf564746f9904314c5d33b96db515c066b  2021-02-23T07:50:00Z   \n",
       "3      34fbbdfdbc62b907f2cb4f26559df29b230273e3  2021-02-22T17:20:00Z   \n",
       "4      666c08e73f2bee116322fafbf2fc1b2972422b81  2021-02-17T14:55:00Z   \n",
       "...                                         ...                   ...   \n",
       "28765  b9a68c99fa994007bfbe7dfc018ec73539555478  2021-02-17T16:35:00Z   \n",
       "28766  b9a68c99fa994007bfbe7dfc018ec73539555478  2021-02-20T15:55:00Z   \n",
       "28767  b9a68c99fa994007bfbe7dfc018ec73539555478  2021-02-19T19:55:00Z   \n",
       "28768  b9a68c99fa994007bfbe7dfc018ec73539555478  2021-02-10T16:05:00Z   \n",
       "28769  b9a68c99fa994007bfbe7dfc018ec73539555478  2021-02-19T19:10:00Z   \n",
       "\n",
       "      HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0         heat          hold               698                      709   \n",
       "1         heat          hold               699                      790   \n",
       "2         heat          hold               640                      650   \n",
       "3         heat          hold               696                      702   \n",
       "4         auto          hold               686                      727   \n",
       "...        ...           ...               ...                      ...   \n",
       "28765     heat          hold               698                      700   \n",
       "28766     heat          hold               695                      700   \n",
       "28767     heat          hold               696                      700   \n",
       "28768     heat          hold               699                      700   \n",
       "28769     heat          hold               698                      700   \n",
       "\n",
       "       TemperatureExpectedHeat ProvinceState      City  Age_of_Home__years_  \\\n",
       "0                          709            MT  Billings                   55   \n",
       "1                          790            MT   Bozeman                   25   \n",
       "2                          638            MT     Butte                  118   \n",
       "3                          699            MT  Missoula                   25   \n",
       "4                          682            MT   Bozeman                    7   \n",
       "...                        ...           ...       ...                  ...   \n",
       "28765                      700            MT  Billings                   60   \n",
       "28766                      700            MT  Billings                   60   \n",
       "28767                      700            MT  Billings                   60   \n",
       "28768                      700            MT  Billings                   60   \n",
       "28769                      700            MT  Billings                   60   \n",
       "\n",
       "       allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                 False         False            False   \n",
       "1                 False         False            False   \n",
       "2                 False         False            False   \n",
       "3                 False         False            False   \n",
       "4                 False         False            False   \n",
       "...                 ...           ...              ...   \n",
       "28765             False         False            False   \n",
       "28766             False         False            False   \n",
       "28767             False         False            False   \n",
       "28768             False         False            False   \n",
       "28769             False         False            False   \n",
       "\n",
       "      Auxilliary_Heat_Fuel_Type  \n",
       "0                           Gas  \n",
       "1                           Gas  \n",
       "2                           Gas  \n",
       "3                           Gas  \n",
       "4                           Gas  \n",
       "...                         ...  \n",
       "28765                       Gas  \n",
       "28766                       Gas  \n",
       "28767                       Gas  \n",
       "28768                       Gas  \n",
       "28769                       Gas  \n",
       "\n",
       "[24578 rows x 14 columns]"
      ]
     },
     "execution_count": 60,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "feb_2021.drop(feb_2021[feb_2021['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "feb_2021.drop(feb_2021[feb_2021['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "feb_2021.drop(feb_2021[feb_2021['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "feb_2021.drop(feb_2021[feb_2021['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "feb_2021"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "feb_2021[\"Year\"] = \"2021\"\n",
    "feb_2021[\"Month\"] = \"feb\"\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "feb_2021 = feb_2021.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "feb_2021_ave = feb_2021.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# feb_2021_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "feb_2021_ave.to_csv(\"data/day/MT/feb/feb_2021_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "\n",
    "### Combine month CSV Files \n",
    "1. Read in files in folders for each state\n",
    "2. Export as combined CSV"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create variable for files in directory\n",
    "files = [f for f in os.listdir(\"data/day/MT/feb/\") if f.endswith(\".csv\")]\n",
    "\n",
    "# files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>Month</th>\n",
       "      <th>Year</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>City</th>\n",
       "      <th>Temperature_ctrl_ave</th>\n",
       "      <th>TemperatureExpectedCool_ave</th>\n",
       "      <th>TemperatureExpectedHeat_ave</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>23d834c190c3686c02c030ee0bf42003872bd591</td>\n",
       "      <td>feb</td>\n",
       "      <td>2017</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>Bozeman</td>\n",
       "      <td>694.828571</td>\n",
       "      <td>704.571429</td>\n",
       "      <td>704.571429</td>\n",
       "      <td>25.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>29f5966d7808882d6c76a0a2d7091c18b3587704</td>\n",
       "      <td>feb</td>\n",
       "      <td>2017</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>Bozeman</td>\n",
       "      <td>708.055556</td>\n",
       "      <td>711.055556</td>\n",
       "      <td>710.444444</td>\n",
       "      <td>9.0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>29f5966d7808882d6c76a0a2d7091c18b3587704</td>\n",
       "      <td>feb</td>\n",
       "      <td>2017</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>Bozeman</td>\n",
       "      <td>726.107895</td>\n",
       "      <td>724.047368</td>\n",
       "      <td>724.047368</td>\n",
       "      <td>9.0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3e35e2d37f16eb3b72b2cbb145e6ed964ce01ca4</td>\n",
       "      <td>feb</td>\n",
       "      <td>2017</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>Great Falls</td>\n",
       "      <td>699.500000</td>\n",
       "      <td>780.000000</td>\n",
       "      <td>720.000000</td>\n",
       "      <td>45.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>3e35e2d37f16eb3b72b2cbb145e6ed964ce01ca4</td>\n",
       "      <td>feb</td>\n",
       "      <td>2017</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>Great Falls</td>\n",
       "      <td>720.469880</td>\n",
       "      <td>780.000000</td>\n",
       "      <td>703.012048</td>\n",
       "      <td>45.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>42</th>\n",
       "      <td>f404081c89442a6b4699e93e43dc4bb72f9e2dc1</td>\n",
       "      <td>feb</td>\n",
       "      <td>2021</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>Great Falls</td>\n",
       "      <td>691.392548</td>\n",
       "      <td>747.196707</td>\n",
       "      <td>697.194107</td>\n",
       "      <td>55.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>43</th>\n",
       "      <td>f469a99acb054dbddd08e713d682acf5035c42df</td>\n",
       "      <td>feb</td>\n",
       "      <td>2021</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>Helena</td>\n",
       "      <td>594.479167</td>\n",
       "      <td>720.000000</td>\n",
       "      <td>720.000000</td>\n",
       "      <td>49.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>44</th>\n",
       "      <td>f74a23b378011473f8b32b3fc2efbbb3b6a9e098</td>\n",
       "      <td>feb</td>\n",
       "      <td>2021</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>Billings</td>\n",
       "      <td>698.000000</td>\n",
       "      <td>709.000000</td>\n",
       "      <td>709.000000</td>\n",
       "      <td>55.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>45</th>\n",
       "      <td>fd93662943beeb4603fd63aacbbbb7abb3183470</td>\n",
       "      <td>feb</td>\n",
       "      <td>2021</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>NASHUA</td>\n",
       "      <td>695.920000</td>\n",
       "      <td>700.680000</td>\n",
       "      <td>699.760000</td>\n",
       "      <td>30.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>46</th>\n",
       "      <td>fec2ec7ae8c5ee7ae93fe16b022bc4c1fc6f2931</td>\n",
       "      <td>feb</td>\n",
       "      <td>2021</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>Kalispell</td>\n",
       "      <td>748.408333</td>\n",
       "      <td>770.000000</td>\n",
       "      <td>750.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>253 rows × 13 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                  Identifier Month  Year HvacMode  \\\n",
       "0   23d834c190c3686c02c030ee0bf42003872bd591   feb  2017     heat   \n",
       "1   29f5966d7808882d6c76a0a2d7091c18b3587704   feb  2017     heat   \n",
       "2   29f5966d7808882d6c76a0a2d7091c18b3587704   feb  2017     heat   \n",
       "3   3e35e2d37f16eb3b72b2cbb145e6ed964ce01ca4   feb  2017     auto   \n",
       "4   3e35e2d37f16eb3b72b2cbb145e6ed964ce01ca4   feb  2017     auto   \n",
       "..                                       ...   ...   ...      ...   \n",
       "42  f404081c89442a6b4699e93e43dc4bb72f9e2dc1   feb  2021     auto   \n",
       "43  f469a99acb054dbddd08e713d682acf5035c42df   feb  2021     heat   \n",
       "44  f74a23b378011473f8b32b3fc2efbbb3b6a9e098   feb  2021     heat   \n",
       "45  fd93662943beeb4603fd63aacbbbb7abb3183470   feb  2021     heat   \n",
       "46  fec2ec7ae8c5ee7ae93fe16b022bc4c1fc6f2931   feb  2021     auto   \n",
       "\n",
       "   CalendarEvent         City  Temperature_ctrl_ave  \\\n",
       "0           hold      Bozeman            694.828571   \n",
       "1           auto      Bozeman            708.055556   \n",
       "2           hold      Bozeman            726.107895   \n",
       "3           auto  Great Falls            699.500000   \n",
       "4           hold  Great Falls            720.469880   \n",
       "..           ...          ...                   ...   \n",
       "42          hold  Great Falls            691.392548   \n",
       "43          hold       Helena            594.479167   \n",
       "44          hold     Billings            698.000000   \n",
       "45          hold       NASHUA            695.920000   \n",
       "46          hold    Kalispell            748.408333   \n",
       "\n",
       "    TemperatureExpectedCool_ave  TemperatureExpectedHeat_ave  \\\n",
       "0                    704.571429                   704.571429   \n",
       "1                    711.055556                   710.444444   \n",
       "2                    724.047368                   724.047368   \n",
       "3                    780.000000                   720.000000   \n",
       "4                    780.000000                   703.012048   \n",
       "..                          ...                          ...   \n",
       "42                   747.196707                   697.194107   \n",
       "43                   720.000000                   720.000000   \n",
       "44                   709.000000                   709.000000   \n",
       "45                   700.680000                   699.760000   \n",
       "46                   770.000000                   750.000000   \n",
       "\n",
       "    Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \n",
       "0                  25.0             False         False            False  \n",
       "1                   9.0              True         False            False  \n",
       "2                   9.0              True         False            False  \n",
       "3                  45.0             False         False            False  \n",
       "4                  45.0             False         False            False  \n",
       "..                  ...               ...           ...              ...  \n",
       "42                 55.0             False         False            False  \n",
       "43                 49.0             False         False            False  \n",
       "44                 55.0             False         False            False  \n",
       "45                 30.0             False         False            False  \n",
       "46                  0.0              True         False            False  \n",
       "\n",
       "[253 rows x 13 columns]"
      ]
     },
     "execution_count": 66,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# https://stacMToverflow.com/questions/63886787/how-to-create-a-dataframe-from-multiple-csv-files\n",
    "# Initialize an empty dataframe\n",
    "MT_feb = pd.DataFrame()\n",
    "\n",
    "# Iterate through files and contents, then concatenate the data into the dataframe\n",
    "for file in files:\n",
    "    df = pd.read_csv(\"data/day/MT/feb/\" + file)\n",
    "    MT_feb = pd.concat([MT_feb, df])\n",
    "    \n",
    "MT_feb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [],
   "source": [
    "MT_feb.to_csv(\"Scraper_Output/State_Month_Day/MT/MT_feb.csv\", header=True, index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "\n",
    "## March"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "\n",
    "## April"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "\n",
    "## May"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "\n",
    "## June"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2017 June Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "jun_2017 = pd.read_csv(\"../data_large/MT-day/2017-jun-day-MT.csv\")\n",
    "\n",
    "# jun_2017"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>c9a27b6d42a3b1e3d96263a706132ded7796fb6f</td>\n",
       "      <td>2017-06-25T19:35:00Z</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>720</td>\n",
       "      <td>720</td>\n",
       "      <td>640</td>\n",
       "      <td>MT</td>\n",
       "      <td>Great Falls</td>\n",
       "      <td>50</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>c9a27b6d42a3b1e3d96263a706132ded7796fb6f</td>\n",
       "      <td>2017-06-25T18:30:00Z</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>733</td>\n",
       "      <td>720</td>\n",
       "      <td>640</td>\n",
       "      <td>MT</td>\n",
       "      <td>Great Falls</td>\n",
       "      <td>50</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>c9a27b6d42a3b1e3d96263a706132ded7796fb6f</td>\n",
       "      <td>2017-06-25T18:35:00Z</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>732</td>\n",
       "      <td>720</td>\n",
       "      <td>640</td>\n",
       "      <td>MT</td>\n",
       "      <td>Great Falls</td>\n",
       "      <td>50</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>c9a27b6d42a3b1e3d96263a706132ded7796fb6f</td>\n",
       "      <td>2017-06-25T19:00:00Z</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>718</td>\n",
       "      <td>745</td>\n",
       "      <td>640</td>\n",
       "      <td>MT</td>\n",
       "      <td>Great Falls</td>\n",
       "      <td>50</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>c9a27b6d42a3b1e3d96263a706132ded7796fb6f</td>\n",
       "      <td>2017-06-25T19:05:00Z</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>717</td>\n",
       "      <td>724</td>\n",
       "      <td>640</td>\n",
       "      <td>MT</td>\n",
       "      <td>Great Falls</td>\n",
       "      <td>50</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12565</th>\n",
       "      <td>0728208cff5242993896da88a882e076094b765f</td>\n",
       "      <td>2017-06-01T12:35:00Z</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>691</td>\n",
       "      <td>740</td>\n",
       "      <td>680</td>\n",
       "      <td>MT</td>\n",
       "      <td>Billings</td>\n",
       "      <td>50</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12566</th>\n",
       "      <td>0728208cff5242993896da88a882e076094b765f</td>\n",
       "      <td>2017-06-01T12:45:00Z</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>691</td>\n",
       "      <td>740</td>\n",
       "      <td>680</td>\n",
       "      <td>MT</td>\n",
       "      <td>Billings</td>\n",
       "      <td>50</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12567</th>\n",
       "      <td>0728208cff5242993896da88a882e076094b765f</td>\n",
       "      <td>2017-06-01T12:20:00Z</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>691</td>\n",
       "      <td>740</td>\n",
       "      <td>680</td>\n",
       "      <td>MT</td>\n",
       "      <td>Billings</td>\n",
       "      <td>50</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12568</th>\n",
       "      <td>0728208cff5242993896da88a882e076094b765f</td>\n",
       "      <td>2017-06-01T12:30:00Z</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>691</td>\n",
       "      <td>740</td>\n",
       "      <td>680</td>\n",
       "      <td>MT</td>\n",
       "      <td>Billings</td>\n",
       "      <td>50</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12569</th>\n",
       "      <td>0728208cff5242993896da88a882e076094b765f</td>\n",
       "      <td>2017-06-01T12:50:00Z</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>691</td>\n",
       "      <td>740</td>\n",
       "      <td>680</td>\n",
       "      <td>MT</td>\n",
       "      <td>Billings</td>\n",
       "      <td>50</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>11713 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                     Identifier             date_time  \\\n",
       "0      c9a27b6d42a3b1e3d96263a706132ded7796fb6f  2017-06-25T19:35:00Z   \n",
       "1      c9a27b6d42a3b1e3d96263a706132ded7796fb6f  2017-06-25T18:30:00Z   \n",
       "2      c9a27b6d42a3b1e3d96263a706132ded7796fb6f  2017-06-25T18:35:00Z   \n",
       "3      c9a27b6d42a3b1e3d96263a706132ded7796fb6f  2017-06-25T19:00:00Z   \n",
       "4      c9a27b6d42a3b1e3d96263a706132ded7796fb6f  2017-06-25T19:05:00Z   \n",
       "...                                         ...                   ...   \n",
       "12565  0728208cff5242993896da88a882e076094b765f  2017-06-01T12:35:00Z   \n",
       "12566  0728208cff5242993896da88a882e076094b765f  2017-06-01T12:45:00Z   \n",
       "12567  0728208cff5242993896da88a882e076094b765f  2017-06-01T12:20:00Z   \n",
       "12568  0728208cff5242993896da88a882e076094b765f  2017-06-01T12:30:00Z   \n",
       "12569  0728208cff5242993896da88a882e076094b765f  2017-06-01T12:50:00Z   \n",
       "\n",
       "      HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0         auto          hold               720                      720   \n",
       "1         auto          hold               733                      720   \n",
       "2         auto          hold               732                      720   \n",
       "3         auto          hold               718                      745   \n",
       "4         auto          hold               717                      724   \n",
       "...        ...           ...               ...                      ...   \n",
       "12565     cool          auto               691                      740   \n",
       "12566     cool          auto               691                      740   \n",
       "12567     cool          auto               691                      740   \n",
       "12568     cool          auto               691                      740   \n",
       "12569     cool          auto               691                      740   \n",
       "\n",
       "       TemperatureExpectedHeat ProvinceState         City  \\\n",
       "0                          640            MT  Great Falls   \n",
       "1                          640            MT  Great Falls   \n",
       "2                          640            MT  Great Falls   \n",
       "3                          640            MT  Great Falls   \n",
       "4                          640            MT  Great Falls   \n",
       "...                        ...           ...          ...   \n",
       "12565                      680            MT     Billings   \n",
       "12566                      680            MT     Billings   \n",
       "12567                      680            MT     Billings   \n",
       "12568                      680            MT     Billings   \n",
       "12569                      680            MT     Billings   \n",
       "\n",
       "       Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                       50             False         False            False   \n",
       "1                       50             False         False            False   \n",
       "2                       50             False         False            False   \n",
       "3                       50             False         False            False   \n",
       "4                       50             False         False            False   \n",
       "...                    ...               ...           ...              ...   \n",
       "12565                   50             False         False            False   \n",
       "12566                   50             False         False            False   \n",
       "12567                   50             False         False            False   \n",
       "12568                   50             False         False            False   \n",
       "12569                   50             False         False            False   \n",
       "\n",
       "      Auxilliary_Heat_Fuel_Type  \n",
       "0                           Gas  \n",
       "1                           Gas  \n",
       "2                           Gas  \n",
       "3                           Gas  \n",
       "4                           Gas  \n",
       "...                         ...  \n",
       "12565                       Gas  \n",
       "12566                       Gas  \n",
       "12567                       Gas  \n",
       "12568                       Gas  \n",
       "12569                       Gas  \n",
       "\n",
       "[11713 rows x 14 columns]"
      ]
     },
     "execution_count": 69,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "jun_2017.drop(jun_2017[jun_2017['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "jun_2017.drop(jun_2017[jun_2017['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "jun_2017.drop(jun_2017[jun_2017['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "jun_2017.drop(jun_2017[jun_2017['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "jun_2017"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "jun_2017[\"Year\"] = \"2017\"\n",
    "jun_2017[\"Month\"] = \"jun\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "jun_2017 = jun_2017.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "jun_2017_ave = jun_2017.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# jun_2017_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "jun_2017_ave.to_csv(\"data/day/MT/jun/jun_2017_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2018 June Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "jun_2018 = pd.read_csv(\"../data_large/MT-day/2018-jun-day-MT.csv\")\n",
    "\n",
    "# jun_2018"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1e4188c276ca852a75feef8c5e3fc27e1f409c4d</td>\n",
       "      <td>2018-06-24T18:45:00Z</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>697</td>\n",
       "      <td>700</td>\n",
       "      <td>662</td>\n",
       "      <td>MT</td>\n",
       "      <td>Helena</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1e4188c276ca852a75feef8c5e3fc27e1f409c4d</td>\n",
       "      <td>2018-06-23T19:20:00Z</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>703</td>\n",
       "      <td>700</td>\n",
       "      <td>662</td>\n",
       "      <td>MT</td>\n",
       "      <td>Helena</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1e4188c276ca852a75feef8c5e3fc27e1f409c4d</td>\n",
       "      <td>2018-06-23T13:00:00Z</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>703</td>\n",
       "      <td>700</td>\n",
       "      <td>662</td>\n",
       "      <td>MT</td>\n",
       "      <td>Helena</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1e4188c276ca852a75feef8c5e3fc27e1f409c4d</td>\n",
       "      <td>2018-06-23T14:40:00Z</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>702</td>\n",
       "      <td>700</td>\n",
       "      <td>662</td>\n",
       "      <td>MT</td>\n",
       "      <td>Helena</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1e4188c276ca852a75feef8c5e3fc27e1f409c4d</td>\n",
       "      <td>2018-06-23T15:15:00Z</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>703</td>\n",
       "      <td>700</td>\n",
       "      <td>662</td>\n",
       "      <td>MT</td>\n",
       "      <td>Helena</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23500</th>\n",
       "      <td>b8a5d9a9f81bb0725b32a07d2ff69b1aba4bff04</td>\n",
       "      <td>2018-06-26T16:50:00Z</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>740</td>\n",
       "      <td>750</td>\n",
       "      <td>750</td>\n",
       "      <td>MT</td>\n",
       "      <td>Billings</td>\n",
       "      <td>37</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23501</th>\n",
       "      <td>b8a5d9a9f81bb0725b32a07d2ff69b1aba4bff04</td>\n",
       "      <td>2018-06-25T17:15:00Z</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>720</td>\n",
       "      <td>750</td>\n",
       "      <td>750</td>\n",
       "      <td>MT</td>\n",
       "      <td>Billings</td>\n",
       "      <td>37</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23502</th>\n",
       "      <td>b8a5d9a9f81bb0725b32a07d2ff69b1aba4bff04</td>\n",
       "      <td>2018-06-27T14:00:00Z</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>719</td>\n",
       "      <td>750</td>\n",
       "      <td>750</td>\n",
       "      <td>MT</td>\n",
       "      <td>Billings</td>\n",
       "      <td>37</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23503</th>\n",
       "      <td>b8a5d9a9f81bb0725b32a07d2ff69b1aba4bff04</td>\n",
       "      <td>2018-06-27T16:20:00Z</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>721</td>\n",
       "      <td>750</td>\n",
       "      <td>750</td>\n",
       "      <td>MT</td>\n",
       "      <td>Billings</td>\n",
       "      <td>37</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23504</th>\n",
       "      <td>b8a5d9a9f81bb0725b32a07d2ff69b1aba4bff04</td>\n",
       "      <td>2018-06-26T13:05:00Z</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>737</td>\n",
       "      <td>750</td>\n",
       "      <td>750</td>\n",
       "      <td>MT</td>\n",
       "      <td>Billings</td>\n",
       "      <td>37</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>23444 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                     Identifier             date_time  \\\n",
       "0      1e4188c276ca852a75feef8c5e3fc27e1f409c4d  2018-06-24T18:45:00Z   \n",
       "1      1e4188c276ca852a75feef8c5e3fc27e1f409c4d  2018-06-23T19:20:00Z   \n",
       "2      1e4188c276ca852a75feef8c5e3fc27e1f409c4d  2018-06-23T13:00:00Z   \n",
       "3      1e4188c276ca852a75feef8c5e3fc27e1f409c4d  2018-06-23T14:40:00Z   \n",
       "4      1e4188c276ca852a75feef8c5e3fc27e1f409c4d  2018-06-23T15:15:00Z   \n",
       "...                                         ...                   ...   \n",
       "23500  b8a5d9a9f81bb0725b32a07d2ff69b1aba4bff04  2018-06-26T16:50:00Z   \n",
       "23501  b8a5d9a9f81bb0725b32a07d2ff69b1aba4bff04  2018-06-25T17:15:00Z   \n",
       "23502  b8a5d9a9f81bb0725b32a07d2ff69b1aba4bff04  2018-06-27T14:00:00Z   \n",
       "23503  b8a5d9a9f81bb0725b32a07d2ff69b1aba4bff04  2018-06-27T16:20:00Z   \n",
       "23504  b8a5d9a9f81bb0725b32a07d2ff69b1aba4bff04  2018-06-26T13:05:00Z   \n",
       "\n",
       "      HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0         cool          auto               697                      700   \n",
       "1         cool          auto               703                      700   \n",
       "2         cool          auto               703                      700   \n",
       "3         cool          auto               702                      700   \n",
       "4         cool          auto               703                      700   \n",
       "...        ...           ...               ...                      ...   \n",
       "23500     cool          hold               740                      750   \n",
       "23501     cool          hold               720                      750   \n",
       "23502     cool          hold               719                      750   \n",
       "23503     cool          hold               721                      750   \n",
       "23504     cool          hold               737                      750   \n",
       "\n",
       "       TemperatureExpectedHeat ProvinceState      City  Age_of_Home__years_  \\\n",
       "0                          662            MT    Helena                    5   \n",
       "1                          662            MT    Helena                    5   \n",
       "2                          662            MT    Helena                    5   \n",
       "3                          662            MT    Helena                    5   \n",
       "4                          662            MT    Helena                    5   \n",
       "...                        ...           ...       ...                  ...   \n",
       "23500                      750            MT  Billings                   37   \n",
       "23501                      750            MT  Billings                   37   \n",
       "23502                      750            MT  Billings                   37   \n",
       "23503                      750            MT  Billings                   37   \n",
       "23504                      750            MT  Billings                   37   \n",
       "\n",
       "       allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                 False         False            False   \n",
       "1                 False         False            False   \n",
       "2                 False         False            False   \n",
       "3                 False         False            False   \n",
       "4                 False         False            False   \n",
       "...                 ...           ...              ...   \n",
       "23500             False         False            False   \n",
       "23501             False         False            False   \n",
       "23502             False         False            False   \n",
       "23503             False         False            False   \n",
       "23504             False         False            False   \n",
       "\n",
       "      Auxilliary_Heat_Fuel_Type  \n",
       "0                           Gas  \n",
       "1                           Gas  \n",
       "2                           Gas  \n",
       "3                           Gas  \n",
       "4                           Gas  \n",
       "...                         ...  \n",
       "23500                       Gas  \n",
       "23501                       Gas  \n",
       "23502                       Gas  \n",
       "23503                       Gas  \n",
       "23504                       Gas  \n",
       "\n",
       "[23444 rows x 14 columns]"
      ]
     },
     "execution_count": 75,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "jun_2018.drop(jun_2018[jun_2018['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "jun_2018.drop(jun_2018[jun_2018['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "jun_2018.drop(jun_2018[jun_2018['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "jun_2018.drop(jun_2018[jun_2018['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "jun_2018"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "jun_2018[\"Year\"] = \"2018\"\n",
    "jun_2018[\"Month\"] = \"jun\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "jun_2018 = jun_2018.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "jun_2018_ave = jun_2018.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# jun_2018_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "jun_2018_ave.to_csv(\"data/day/MT/jun/jun_2018_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2019 June Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "jun_2019 = pd.read_csv(\"../data_large/MT-day/2019-jun-day-MT.csv\")\n",
    "\n",
    "# jun_2019"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>7352a85776176198f0abb065b891160881b30270</td>\n",
       "      <td>2019-06-02 15:05:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>659</td>\n",
       "      <td>727</td>\n",
       "      <td>677</td>\n",
       "      <td>MT</td>\n",
       "      <td>Bozeman</td>\n",
       "      <td>15</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>7352a85776176198f0abb065b891160881b30270</td>\n",
       "      <td>2019-06-02 19:40:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>690</td>\n",
       "      <td>727</td>\n",
       "      <td>677</td>\n",
       "      <td>MT</td>\n",
       "      <td>Bozeman</td>\n",
       "      <td>15</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>7352a85776176198f0abb065b891160881b30270</td>\n",
       "      <td>2019-06-02 16:10:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>669</td>\n",
       "      <td>727</td>\n",
       "      <td>677</td>\n",
       "      <td>MT</td>\n",
       "      <td>Bozeman</td>\n",
       "      <td>15</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>7352a85776176198f0abb065b891160881b30270</td>\n",
       "      <td>2019-06-02 16:20:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>673</td>\n",
       "      <td>727</td>\n",
       "      <td>677</td>\n",
       "      <td>MT</td>\n",
       "      <td>Bozeman</td>\n",
       "      <td>15</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>7352a85776176198f0abb065b891160881b30270</td>\n",
       "      <td>2019-06-02 18:05:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>680</td>\n",
       "      <td>727</td>\n",
       "      <td>677</td>\n",
       "      <td>MT</td>\n",
       "      <td>Bozeman</td>\n",
       "      <td>15</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>44100</th>\n",
       "      <td>1fe82232930e26efc50ba4da526c354676d03dec</td>\n",
       "      <td>2019-06-08 18:30:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>682</td>\n",
       "      <td>750</td>\n",
       "      <td>680</td>\n",
       "      <td>MT</td>\n",
       "      <td>Helena</td>\n",
       "      <td>38</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>44101</th>\n",
       "      <td>1fe82232930e26efc50ba4da526c354676d03dec</td>\n",
       "      <td>2019-06-09 16:20:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>683</td>\n",
       "      <td>750</td>\n",
       "      <td>680</td>\n",
       "      <td>MT</td>\n",
       "      <td>Helena</td>\n",
       "      <td>38</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>44102</th>\n",
       "      <td>1fe82232930e26efc50ba4da526c354676d03dec</td>\n",
       "      <td>2019-06-09 18:40:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>684</td>\n",
       "      <td>750</td>\n",
       "      <td>680</td>\n",
       "      <td>MT</td>\n",
       "      <td>Helena</td>\n",
       "      <td>38</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>44103</th>\n",
       "      <td>77c344f9bb93b933ebc934a7ebba997d720726d5</td>\n",
       "      <td>2019-06-27 17:05:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>681</td>\n",
       "      <td>680</td>\n",
       "      <td>680</td>\n",
       "      <td>MT</td>\n",
       "      <td>Butte</td>\n",
       "      <td>90</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>44104</th>\n",
       "      <td>77c344f9bb93b933ebc934a7ebba997d720726d5</td>\n",
       "      <td>2019-06-27 16:50:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>681</td>\n",
       "      <td>680</td>\n",
       "      <td>680</td>\n",
       "      <td>MT</td>\n",
       "      <td>Butte</td>\n",
       "      <td>90</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>28757 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                     Identifier                date_time  \\\n",
       "0      7352a85776176198f0abb065b891160881b30270  2019-06-02 15:05:00 UTC   \n",
       "1      7352a85776176198f0abb065b891160881b30270  2019-06-02 19:40:00 UTC   \n",
       "2      7352a85776176198f0abb065b891160881b30270  2019-06-02 16:10:00 UTC   \n",
       "3      7352a85776176198f0abb065b891160881b30270  2019-06-02 16:20:00 UTC   \n",
       "4      7352a85776176198f0abb065b891160881b30270  2019-06-02 18:05:00 UTC   \n",
       "...                                         ...                      ...   \n",
       "44100  1fe82232930e26efc50ba4da526c354676d03dec  2019-06-08 18:30:00 UTC   \n",
       "44101  1fe82232930e26efc50ba4da526c354676d03dec  2019-06-09 16:20:00 UTC   \n",
       "44102  1fe82232930e26efc50ba4da526c354676d03dec  2019-06-09 18:40:00 UTC   \n",
       "44103  77c344f9bb93b933ebc934a7ebba997d720726d5  2019-06-27 17:05:00 UTC   \n",
       "44104  77c344f9bb93b933ebc934a7ebba997d720726d5  2019-06-27 16:50:00 UTC   \n",
       "\n",
       "      HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0         cool          hold               659                      727   \n",
       "1         cool          hold               690                      727   \n",
       "2         cool          hold               669                      727   \n",
       "3         cool          hold               673                      727   \n",
       "4         cool          hold               680                      727   \n",
       "...        ...           ...               ...                      ...   \n",
       "44100     auto          auto               682                      750   \n",
       "44101     auto          auto               683                      750   \n",
       "44102     auto          auto               684                      750   \n",
       "44103     heat          auto               681                      680   \n",
       "44104     heat          auto               681                      680   \n",
       "\n",
       "       TemperatureExpectedHeat ProvinceState     City  Age_of_Home__years_  \\\n",
       "0                          677            MT  Bozeman                   15   \n",
       "1                          677            MT  Bozeman                   15   \n",
       "2                          677            MT  Bozeman                   15   \n",
       "3                          677            MT  Bozeman                   15   \n",
       "4                          677            MT  Bozeman                   15   \n",
       "...                        ...           ...      ...                  ...   \n",
       "44100                      680            MT   Helena                   38   \n",
       "44101                      680            MT   Helena                   38   \n",
       "44102                      680            MT   Helena                   38   \n",
       "44103                      680            MT    Butte                   90   \n",
       "44104                      680            MT    Butte                   90   \n",
       "\n",
       "       allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                 False         False            False   \n",
       "1                 False         False            False   \n",
       "2                 False         False            False   \n",
       "3                 False         False            False   \n",
       "4                 False         False            False   \n",
       "...                 ...           ...              ...   \n",
       "44100             False         False            False   \n",
       "44101             False         False            False   \n",
       "44102             False         False            False   \n",
       "44103             False         False            False   \n",
       "44104             False         False            False   \n",
       "\n",
       "      Auxilliary_Heat_Fuel_Type  \n",
       "0                           Gas  \n",
       "1                           Gas  \n",
       "2                           Gas  \n",
       "3                           Gas  \n",
       "4                           Gas  \n",
       "...                         ...  \n",
       "44100                       Gas  \n",
       "44101                       Gas  \n",
       "44102                       Gas  \n",
       "44103                       Gas  \n",
       "44104                       Gas  \n",
       "\n",
       "[28757 rows x 14 columns]"
      ]
     },
     "execution_count": 81,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "jun_2019.drop(jun_2019[jun_2019['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "jun_2019.drop(jun_2019[jun_2019['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "jun_2019.drop(jun_2019[jun_2019['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "jun_2019.drop(jun_2019[jun_2019['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "jun_2019"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "jun_2019[\"Year\"] = \"2019\"\n",
    "jun_2019[\"Month\"] = \"jun\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "jun_2019 = jun_2019.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "jun_2019_ave = jun_2019.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# jun_2019_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "jun_2019_ave.to_csv(\"data/day/MT/jun/jun_2019_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2020 June Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "jun_2020 = pd.read_csv(\"../data_large/MT-day/2020-jun-day-MT.csv\")\n",
    "\n",
    "# jun_2020"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2f91f344abd5823f11abdfa57e08bb514aa09564</td>\n",
       "      <td>2020-06-26 18:55:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>723</td>\n",
       "      <td>701</td>\n",
       "      <td>701</td>\n",
       "      <td>MT</td>\n",
       "      <td>Billings</td>\n",
       "      <td>50</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>e0715a2e7cf7287698dc580b2f479f4eaf35e0ea</td>\n",
       "      <td>2020-06-02 12:00:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>702</td>\n",
       "      <td>762</td>\n",
       "      <td>712</td>\n",
       "      <td>MT</td>\n",
       "      <td>Kalispell</td>\n",
       "      <td>40</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>77c344f9bb93b933ebc934a7ebba997d720726d5</td>\n",
       "      <td>2020-06-09 19:25:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>660</td>\n",
       "      <td>732</td>\n",
       "      <td>642</td>\n",
       "      <td>MT</td>\n",
       "      <td>Butte</td>\n",
       "      <td>90</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>f74a23b378011473f8b32b3fc2efbbb3b6a9e098</td>\n",
       "      <td>2020-06-13 15:05:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>717</td>\n",
       "      <td>688</td>\n",
       "      <td>638</td>\n",
       "      <td>MT</td>\n",
       "      <td>Billings</td>\n",
       "      <td>55</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>f74a23b378011473f8b32b3fc2efbbb3b6a9e098</td>\n",
       "      <td>2020-06-13 16:40:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>702</td>\n",
       "      <td>688</td>\n",
       "      <td>638</td>\n",
       "      <td>MT</td>\n",
       "      <td>Billings</td>\n",
       "      <td>55</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>49047</th>\n",
       "      <td>b7a28b95c2726be560baf3656422ba13edc9398a</td>\n",
       "      <td>2020-06-06 18:10:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>729</td>\n",
       "      <td>750</td>\n",
       "      <td>700</td>\n",
       "      <td>MT</td>\n",
       "      <td>Kalispell</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>49048</th>\n",
       "      <td>b7a28b95c2726be560baf3656422ba13edc9398a</td>\n",
       "      <td>2020-06-10 12:30:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>695</td>\n",
       "      <td>750</td>\n",
       "      <td>700</td>\n",
       "      <td>MT</td>\n",
       "      <td>Kalispell</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>49049</th>\n",
       "      <td>b7a28b95c2726be560baf3656422ba13edc9398a</td>\n",
       "      <td>2020-06-11 13:25:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>719</td>\n",
       "      <td>750</td>\n",
       "      <td>700</td>\n",
       "      <td>MT</td>\n",
       "      <td>Kalispell</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>49050</th>\n",
       "      <td>b7a28b95c2726be560baf3656422ba13edc9398a</td>\n",
       "      <td>2020-06-07 16:55:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>702</td>\n",
       "      <td>750</td>\n",
       "      <td>700</td>\n",
       "      <td>MT</td>\n",
       "      <td>Kalispell</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>49051</th>\n",
       "      <td>b7a28b95c2726be560baf3656422ba13edc9398a</td>\n",
       "      <td>2020-06-07 15:20:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>704</td>\n",
       "      <td>750</td>\n",
       "      <td>700</td>\n",
       "      <td>MT</td>\n",
       "      <td>Kalispell</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>44443 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                     Identifier                date_time  \\\n",
       "0      2f91f344abd5823f11abdfa57e08bb514aa09564  2020-06-26 18:55:00 UTC   \n",
       "1      e0715a2e7cf7287698dc580b2f479f4eaf35e0ea  2020-06-02 12:00:00 UTC   \n",
       "2      77c344f9bb93b933ebc934a7ebba997d720726d5  2020-06-09 19:25:00 UTC   \n",
       "3      f74a23b378011473f8b32b3fc2efbbb3b6a9e098  2020-06-13 15:05:00 UTC   \n",
       "4      f74a23b378011473f8b32b3fc2efbbb3b6a9e098  2020-06-13 16:40:00 UTC   \n",
       "...                                         ...                      ...   \n",
       "49047  b7a28b95c2726be560baf3656422ba13edc9398a  2020-06-06 18:10:00 UTC   \n",
       "49048  b7a28b95c2726be560baf3656422ba13edc9398a  2020-06-10 12:30:00 UTC   \n",
       "49049  b7a28b95c2726be560baf3656422ba13edc9398a  2020-06-11 13:25:00 UTC   \n",
       "49050  b7a28b95c2726be560baf3656422ba13edc9398a  2020-06-07 16:55:00 UTC   \n",
       "49051  b7a28b95c2726be560baf3656422ba13edc9398a  2020-06-07 15:20:00 UTC   \n",
       "\n",
       "      HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0         cool          hold               723                      701   \n",
       "1         auto          hold               702                      762   \n",
       "2         heat          hold               660                      732   \n",
       "3         auto          hold               717                      688   \n",
       "4         auto          hold               702                      688   \n",
       "...        ...           ...               ...                      ...   \n",
       "49047     auto          hold               729                      750   \n",
       "49048     auto          hold               695                      750   \n",
       "49049     auto          hold               719                      750   \n",
       "49050     auto          hold               702                      750   \n",
       "49051     auto          hold               704                      750   \n",
       "\n",
       "       TemperatureExpectedHeat ProvinceState       City  Age_of_Home__years_  \\\n",
       "0                          701            MT   Billings                   50   \n",
       "1                          712            MT  Kalispell                   40   \n",
       "2                          642            MT      Butte                   90   \n",
       "3                          638            MT   Billings                   55   \n",
       "4                          638            MT   Billings                   55   \n",
       "...                        ...           ...        ...                  ...   \n",
       "49047                      700            MT  Kalispell                    5   \n",
       "49048                      700            MT  Kalispell                    5   \n",
       "49049                      700            MT  Kalispell                    5   \n",
       "49050                      700            MT  Kalispell                    5   \n",
       "49051                      700            MT  Kalispell                    5   \n",
       "\n",
       "       allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                  True         False            False   \n",
       "1                 False         False            False   \n",
       "2                 False         False            False   \n",
       "3                 False         False            False   \n",
       "4                 False         False            False   \n",
       "...                 ...           ...              ...   \n",
       "49047             False         False            False   \n",
       "49048             False         False            False   \n",
       "49049             False         False            False   \n",
       "49050             False         False            False   \n",
       "49051             False         False            False   \n",
       "\n",
       "      Auxilliary_Heat_Fuel_Type  \n",
       "0                           Gas  \n",
       "1                           Gas  \n",
       "2                           Gas  \n",
       "3                           Gas  \n",
       "4                           Gas  \n",
       "...                         ...  \n",
       "49047                       Gas  \n",
       "49048                       Gas  \n",
       "49049                       Gas  \n",
       "49050                       Gas  \n",
       "49051                       Gas  \n",
       "\n",
       "[44443 rows x 14 columns]"
      ]
     },
     "execution_count": 87,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "jun_2020.drop(jun_2020[jun_2020['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "jun_2020.drop(jun_2020[jun_2020['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "jun_2020.drop(jun_2020[jun_2020['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "jun_2020.drop(jun_2020[jun_2020['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "jun_2020"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "jun_2020[\"Year\"] = \"2020\"\n",
    "jun_2020[\"Month\"] = \"jun\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "jun_2020 = jun_2020.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "jun_2020_ave = jun_2020.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# jun_2020_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "jun_2020_ave.to_csv(\"data/day/MT/jun/jun_2020_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2021 June Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "jun_2021 = pd.read_csv(\"../data_large/MT-day/2021-jun-day-MT.csv\")\n",
    "\n",
    "# jun_2021"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>5630eed158cf32dc803ac57a6956f5b6e46d8095</td>\n",
       "      <td>2021-06-24 15:40:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>696</td>\n",
       "      <td>719</td>\n",
       "      <td>679</td>\n",
       "      <td>MT</td>\n",
       "      <td>Bozeman</td>\n",
       "      <td>100</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2f91f344abd5823f11abdfa57e08bb514aa09564</td>\n",
       "      <td>2021-06-25 18:25:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>723</td>\n",
       "      <td>730</td>\n",
       "      <td>721</td>\n",
       "      <td>MT</td>\n",
       "      <td>Billings</td>\n",
       "      <td>50</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>f74a23b378011473f8b32b3fc2efbbb3b6a9e098</td>\n",
       "      <td>2021-06-05 14:55:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>685</td>\n",
       "      <td>669</td>\n",
       "      <td>619</td>\n",
       "      <td>MT</td>\n",
       "      <td>Billings</td>\n",
       "      <td>55</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>f74a23b378011473f8b32b3fc2efbbb3b6a9e098</td>\n",
       "      <td>2021-06-04 16:15:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>663</td>\n",
       "      <td>663</td>\n",
       "      <td>613</td>\n",
       "      <td>MT</td>\n",
       "      <td>Billings</td>\n",
       "      <td>55</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>f74a23b378011473f8b32b3fc2efbbb3b6a9e098</td>\n",
       "      <td>2021-06-04 16:30:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>668</td>\n",
       "      <td>663</td>\n",
       "      <td>613</td>\n",
       "      <td>MT</td>\n",
       "      <td>Billings</td>\n",
       "      <td>55</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25881</th>\n",
       "      <td>4cd54fe9a866c28682055020800c8f0b5af25df5</td>\n",
       "      <td>2021-06-26 11:40:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>713</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>MT</td>\n",
       "      <td>Billings</td>\n",
       "      <td>9</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25882</th>\n",
       "      <td>4cd54fe9a866c28682055020800c8f0b5af25df5</td>\n",
       "      <td>2021-06-08 18:30:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>701</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>MT</td>\n",
       "      <td>Billings</td>\n",
       "      <td>9</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25883</th>\n",
       "      <td>4cd54fe9a866c28682055020800c8f0b5af25df5</td>\n",
       "      <td>2021-06-25 17:25:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>711</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>MT</td>\n",
       "      <td>Billings</td>\n",
       "      <td>9</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25884</th>\n",
       "      <td>4cd54fe9a866c28682055020800c8f0b5af25df5</td>\n",
       "      <td>2021-06-10 17:45:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>726</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>MT</td>\n",
       "      <td>Billings</td>\n",
       "      <td>9</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25885</th>\n",
       "      <td>4cd54fe9a866c28682055020800c8f0b5af25df5</td>\n",
       "      <td>2021-06-11 16:00:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>666</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>MT</td>\n",
       "      <td>Billings</td>\n",
       "      <td>9</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>21451 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                     Identifier                date_time  \\\n",
       "0      5630eed158cf32dc803ac57a6956f5b6e46d8095  2021-06-24 15:40:00 UTC   \n",
       "1      2f91f344abd5823f11abdfa57e08bb514aa09564  2021-06-25 18:25:00 UTC   \n",
       "2      f74a23b378011473f8b32b3fc2efbbb3b6a9e098  2021-06-05 14:55:00 UTC   \n",
       "3      f74a23b378011473f8b32b3fc2efbbb3b6a9e098  2021-06-04 16:15:00 UTC   \n",
       "4      f74a23b378011473f8b32b3fc2efbbb3b6a9e098  2021-06-04 16:30:00 UTC   \n",
       "...                                         ...                      ...   \n",
       "25881  4cd54fe9a866c28682055020800c8f0b5af25df5  2021-06-26 11:40:00 UTC   \n",
       "25882  4cd54fe9a866c28682055020800c8f0b5af25df5  2021-06-08 18:30:00 UTC   \n",
       "25883  4cd54fe9a866c28682055020800c8f0b5af25df5  2021-06-25 17:25:00 UTC   \n",
       "25884  4cd54fe9a866c28682055020800c8f0b5af25df5  2021-06-10 17:45:00 UTC   \n",
       "25885  4cd54fe9a866c28682055020800c8f0b5af25df5  2021-06-11 16:00:00 UTC   \n",
       "\n",
       "      HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0         heat          hold               696                      719   \n",
       "1         cool          hold               723                      730   \n",
       "2         auto          hold               685                      669   \n",
       "3         auto          hold               663                      663   \n",
       "4         auto          hold               668                      663   \n",
       "...        ...           ...               ...                      ...   \n",
       "25881     cool          hold               713                      760   \n",
       "25882     cool          hold               701                      760   \n",
       "25883     cool          hold               711                      760   \n",
       "25884     cool          hold               726                      760   \n",
       "25885     cool          hold               666                      760   \n",
       "\n",
       "       TemperatureExpectedHeat ProvinceState      City  Age_of_Home__years_  \\\n",
       "0                          679            MT   Bozeman                  100   \n",
       "1                          721            MT  Billings                   50   \n",
       "2                          619            MT  Billings                   55   \n",
       "3                          613            MT  Billings                   55   \n",
       "4                          613            MT  Billings                   55   \n",
       "...                        ...           ...       ...                  ...   \n",
       "25881                      760            MT  Billings                    9   \n",
       "25882                      760            MT  Billings                    9   \n",
       "25883                      760            MT  Billings                    9   \n",
       "25884                      760            MT  Billings                    9   \n",
       "25885                      760            MT  Billings                    9   \n",
       "\n",
       "       allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                 False         False            False   \n",
       "1                  True         False            False   \n",
       "2                 False         False            False   \n",
       "3                 False         False            False   \n",
       "4                 False         False            False   \n",
       "...                 ...           ...              ...   \n",
       "25881              True         False            False   \n",
       "25882              True         False            False   \n",
       "25883              True         False            False   \n",
       "25884              True         False            False   \n",
       "25885              True         False            False   \n",
       "\n",
       "      Auxilliary_Heat_Fuel_Type  \n",
       "0                           Gas  \n",
       "1                           Gas  \n",
       "2                           Gas  \n",
       "3                           Gas  \n",
       "4                           Gas  \n",
       "...                         ...  \n",
       "25881                       Gas  \n",
       "25882                       Gas  \n",
       "25883                       Gas  \n",
       "25884                       Gas  \n",
       "25885                       Gas  \n",
       "\n",
       "[21451 rows x 14 columns]"
      ]
     },
     "execution_count": 93,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "jun_2021.drop(jun_2021[jun_2021['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "jun_2021.drop(jun_2021[jun_2021['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "jun_2021.drop(jun_2021[jun_2021['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "jun_2021.drop(jun_2021[jun_2021['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "jun_2021"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "jun_2021[\"Year\"] = \"2021\"\n",
    "jun_2021[\"Month\"] = \"jun\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "jun_2021 = jun_2021.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "jun_2021_ave = jun_2021.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# jun_2021_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "jun_2021_ave.to_csv(\"data/day/MT/jun/jun_2021_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "\n",
    "### Combine month CSV Files \n",
    "1. Read in files in folders for each state\n",
    "2. Export as combined CSV"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create variable for files in directory\n",
    "files = [f for f in os.listdir(\"data/day/MT/jun/\") if f.endswith(\".csv\")]\n",
    "\n",
    "# files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>Month</th>\n",
       "      <th>Year</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>City</th>\n",
       "      <th>Temperature_ctrl_ave</th>\n",
       "      <th>TemperatureExpectedCool_ave</th>\n",
       "      <th>TemperatureExpectedHeat_ave</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0728208cff5242993896da88a882e076094b765f</td>\n",
       "      <td>jun</td>\n",
       "      <td>2017</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>Billings</td>\n",
       "      <td>690.833333</td>\n",
       "      <td>740.583333</td>\n",
       "      <td>680.000000</td>\n",
       "      <td>50.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0bc51b95b2dc23713f611e182c689ad03dc2c300</td>\n",
       "      <td>jun</td>\n",
       "      <td>2017</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>Great Falls</td>\n",
       "      <td>698.791667</td>\n",
       "      <td>710.000000</td>\n",
       "      <td>695.000000</td>\n",
       "      <td>10.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0bc51b95b2dc23713f611e182c689ad03dc2c300</td>\n",
       "      <td>jun</td>\n",
       "      <td>2017</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>Great Falls</td>\n",
       "      <td>693.491667</td>\n",
       "      <td>701.595833</td>\n",
       "      <td>701.595833</td>\n",
       "      <td>10.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2323dbdb6769370962e7858668fdc12bc0eaefcf</td>\n",
       "      <td>jun</td>\n",
       "      <td>2017</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>Deer Lodge</td>\n",
       "      <td>670.834101</td>\n",
       "      <td>650.041475</td>\n",
       "      <td>650.009217</td>\n",
       "      <td>55.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>29f5966d7808882d6c76a0a2d7091c18b3587704</td>\n",
       "      <td>jun</td>\n",
       "      <td>2017</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>Bozeman</td>\n",
       "      <td>692.416667</td>\n",
       "      <td>745.000000</td>\n",
       "      <td>695.000000</td>\n",
       "      <td>9.0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>45</th>\n",
       "      <td>f404081c89442a6b4699e93e43dc4bb72f9e2dc1</td>\n",
       "      <td>jun</td>\n",
       "      <td>2021</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>Great Falls</td>\n",
       "      <td>721.327862</td>\n",
       "      <td>730.421420</td>\n",
       "      <td>680.421420</td>\n",
       "      <td>55.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>46</th>\n",
       "      <td>f74a23b378011473f8b32b3fc2efbbb3b6a9e098</td>\n",
       "      <td>jun</td>\n",
       "      <td>2021</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>Billings</td>\n",
       "      <td>679.107143</td>\n",
       "      <td>667.160714</td>\n",
       "      <td>617.160714</td>\n",
       "      <td>55.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>47</th>\n",
       "      <td>f74a23b378011473f8b32b3fc2efbbb3b6a9e098</td>\n",
       "      <td>jun</td>\n",
       "      <td>2021</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>Billings</td>\n",
       "      <td>664.848837</td>\n",
       "      <td>655.116279</td>\n",
       "      <td>654.720930</td>\n",
       "      <td>55.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>48</th>\n",
       "      <td>fd93662943beeb4603fd63aacbbbb7abb3183470</td>\n",
       "      <td>jun</td>\n",
       "      <td>2021</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>NASHUA</td>\n",
       "      <td>745.318584</td>\n",
       "      <td>741.955752</td>\n",
       "      <td>740.814159</td>\n",
       "      <td>30.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>49</th>\n",
       "      <td>fec2ec7ae8c5ee7ae93fe16b022bc4c1fc6f2931</td>\n",
       "      <td>jun</td>\n",
       "      <td>2021</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>Kalispell</td>\n",
       "      <td>712.032132</td>\n",
       "      <td>709.202194</td>\n",
       "      <td>679.605016</td>\n",
       "      <td>0.0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>304 rows × 13 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                  Identifier Month  Year HvacMode  \\\n",
       "0   0728208cff5242993896da88a882e076094b765f   jun  2017     cool   \n",
       "1   0bc51b95b2dc23713f611e182c689ad03dc2c300   jun  2017     cool   \n",
       "2   0bc51b95b2dc23713f611e182c689ad03dc2c300   jun  2017     cool   \n",
       "3   2323dbdb6769370962e7858668fdc12bc0eaefcf   jun  2017     heat   \n",
       "4   29f5966d7808882d6c76a0a2d7091c18b3587704   jun  2017     auto   \n",
       "..                                       ...   ...   ...      ...   \n",
       "45  f404081c89442a6b4699e93e43dc4bb72f9e2dc1   jun  2021     auto   \n",
       "46  f74a23b378011473f8b32b3fc2efbbb3b6a9e098   jun  2021     auto   \n",
       "47  f74a23b378011473f8b32b3fc2efbbb3b6a9e098   jun  2021     cool   \n",
       "48  fd93662943beeb4603fd63aacbbbb7abb3183470   jun  2021     cool   \n",
       "49  fec2ec7ae8c5ee7ae93fe16b022bc4c1fc6f2931   jun  2021     auto   \n",
       "\n",
       "   CalendarEvent         City  Temperature_ctrl_ave  \\\n",
       "0           auto     Billings            690.833333   \n",
       "1           auto  Great Falls            698.791667   \n",
       "2           hold  Great Falls            693.491667   \n",
       "3           hold   Deer Lodge            670.834101   \n",
       "4           auto      Bozeman            692.416667   \n",
       "..           ...          ...                   ...   \n",
       "45          hold  Great Falls            721.327862   \n",
       "46          hold     Billings            679.107143   \n",
       "47          hold     Billings            664.848837   \n",
       "48          hold       NASHUA            745.318584   \n",
       "49          hold    Kalispell            712.032132   \n",
       "\n",
       "    TemperatureExpectedCool_ave  TemperatureExpectedHeat_ave  \\\n",
       "0                    740.583333                   680.000000   \n",
       "1                    710.000000                   695.000000   \n",
       "2                    701.595833                   701.595833   \n",
       "3                    650.041475                   650.009217   \n",
       "4                    745.000000                   695.000000   \n",
       "..                          ...                          ...   \n",
       "45                   730.421420                   680.421420   \n",
       "46                   667.160714                   617.160714   \n",
       "47                   655.116279                   654.720930   \n",
       "48                   741.955752                   740.814159   \n",
       "49                   709.202194                   679.605016   \n",
       "\n",
       "    Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \n",
       "0                  50.0             False         False            False  \n",
       "1                  10.0             False         False            False  \n",
       "2                  10.0             False         False            False  \n",
       "3                  55.0             False         False            False  \n",
       "4                   9.0              True         False            False  \n",
       "..                  ...               ...           ...              ...  \n",
       "45                 55.0             False         False            False  \n",
       "46                 55.0             False         False            False  \n",
       "47                 55.0             False         False            False  \n",
       "48                 30.0             False         False            False  \n",
       "49                  0.0              True         False            False  \n",
       "\n",
       "[304 rows x 13 columns]"
      ]
     },
     "execution_count": 99,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# https://stacMToverflow.com/questions/63886787/how-to-create-a-dataframe-from-multiple-csv-files\n",
    "# Initialize an empty dataframe\n",
    "MT_jun = pd.DataFrame()\n",
    "\n",
    "# Iterate through files and contents, then concatenate the data into the dataframe\n",
    "for file in files:\n",
    "    df = pd.read_csv(\"data/day/MT/jun/\" + file)\n",
    "    MT_jun = pd.concat([MT_jun, df])\n",
    "    \n",
    "MT_jun"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "metadata": {},
   "outputs": [],
   "source": [
    "MT_jun.to_csv(\"Scraper_Output/State_Month_Day/MT/MT_jun.csv\", header=True, index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "\n",
    "## July"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2017 July Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "jul_2017 = pd.read_csv(\"../data_large/MT-day/2017-jul-day-MT.csv\")\n",
    "\n",
    "# jul_2017"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>3e35e2d37f16eb3b72b2cbb145e6ed964ce01ca4</td>\n",
       "      <td>2017-07-13T19:55:00Z</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>746</td>\n",
       "      <td>730</td>\n",
       "      <td>680</td>\n",
       "      <td>MT</td>\n",
       "      <td>Great Falls</td>\n",
       "      <td>45</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>3e35e2d37f16eb3b72b2cbb145e6ed964ce01ca4</td>\n",
       "      <td>2017-07-13T19:50:00Z</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>746</td>\n",
       "      <td>730</td>\n",
       "      <td>680</td>\n",
       "      <td>MT</td>\n",
       "      <td>Great Falls</td>\n",
       "      <td>45</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3e35e2d37f16eb3b72b2cbb145e6ed964ce01ca4</td>\n",
       "      <td>2017-07-15T17:25:00Z</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>740</td>\n",
       "      <td>730</td>\n",
       "      <td>680</td>\n",
       "      <td>MT</td>\n",
       "      <td>Great Falls</td>\n",
       "      <td>45</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3e35e2d37f16eb3b72b2cbb145e6ed964ce01ca4</td>\n",
       "      <td>2017-07-15T18:50:00Z</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>758</td>\n",
       "      <td>730</td>\n",
       "      <td>680</td>\n",
       "      <td>MT</td>\n",
       "      <td>Great Falls</td>\n",
       "      <td>45</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>3e35e2d37f16eb3b72b2cbb145e6ed964ce01ca4</td>\n",
       "      <td>2017-07-15T16:15:00Z</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>741</td>\n",
       "      <td>730</td>\n",
       "      <td>680</td>\n",
       "      <td>MT</td>\n",
       "      <td>Great Falls</td>\n",
       "      <td>45</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14685</th>\n",
       "      <td>c9a27b6d42a3b1e3d96263a706132ded7796fb6f</td>\n",
       "      <td>2017-07-06T18:25:00Z</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>774</td>\n",
       "      <td>740</td>\n",
       "      <td>680</td>\n",
       "      <td>MT</td>\n",
       "      <td>Great Falls</td>\n",
       "      <td>50</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14686</th>\n",
       "      <td>bbdf7f0987b78887505d5f3fa975d0a42b1baabe</td>\n",
       "      <td>2017-07-30T17:10:00Z</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>723</td>\n",
       "      <td>720</td>\n",
       "      <td>680</td>\n",
       "      <td>MT</td>\n",
       "      <td>Scobey</td>\n",
       "      <td>35</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14687</th>\n",
       "      <td>bbdf7f0987b78887505d5f3fa975d0a42b1baabe</td>\n",
       "      <td>2017-07-30T17:05:00Z</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>724</td>\n",
       "      <td>720</td>\n",
       "      <td>680</td>\n",
       "      <td>MT</td>\n",
       "      <td>Scobey</td>\n",
       "      <td>35</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14688</th>\n",
       "      <td>bbdf7f0987b78887505d5f3fa975d0a42b1baabe</td>\n",
       "      <td>2017-07-30T19:50:00Z</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>722</td>\n",
       "      <td>720</td>\n",
       "      <td>680</td>\n",
       "      <td>MT</td>\n",
       "      <td>Scobey</td>\n",
       "      <td>35</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14689</th>\n",
       "      <td>c9a27b6d42a3b1e3d96263a706132ded7796fb6f</td>\n",
       "      <td>2017-07-06T19:40:00Z</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>769</td>\n",
       "      <td>740</td>\n",
       "      <td>680</td>\n",
       "      <td>MT</td>\n",
       "      <td>Great Falls</td>\n",
       "      <td>50</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>11260 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                     Identifier             date_time  \\\n",
       "0      3e35e2d37f16eb3b72b2cbb145e6ed964ce01ca4  2017-07-13T19:55:00Z   \n",
       "1      3e35e2d37f16eb3b72b2cbb145e6ed964ce01ca4  2017-07-13T19:50:00Z   \n",
       "2      3e35e2d37f16eb3b72b2cbb145e6ed964ce01ca4  2017-07-15T17:25:00Z   \n",
       "3      3e35e2d37f16eb3b72b2cbb145e6ed964ce01ca4  2017-07-15T18:50:00Z   \n",
       "4      3e35e2d37f16eb3b72b2cbb145e6ed964ce01ca4  2017-07-15T16:15:00Z   \n",
       "...                                         ...                   ...   \n",
       "14685  c9a27b6d42a3b1e3d96263a706132ded7796fb6f  2017-07-06T18:25:00Z   \n",
       "14686  bbdf7f0987b78887505d5f3fa975d0a42b1baabe  2017-07-30T17:10:00Z   \n",
       "14687  bbdf7f0987b78887505d5f3fa975d0a42b1baabe  2017-07-30T17:05:00Z   \n",
       "14688  bbdf7f0987b78887505d5f3fa975d0a42b1baabe  2017-07-30T19:50:00Z   \n",
       "14689  c9a27b6d42a3b1e3d96263a706132ded7796fb6f  2017-07-06T19:40:00Z   \n",
       "\n",
       "      HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0         auto          hold               746                      730   \n",
       "1         auto          hold               746                      730   \n",
       "2         auto          hold               740                      730   \n",
       "3         auto          hold               758                      730   \n",
       "4         auto          hold               741                      730   \n",
       "...        ...           ...               ...                      ...   \n",
       "14685     auto          hold               774                      740   \n",
       "14686     cool          auto               723                      720   \n",
       "14687     cool          auto               724                      720   \n",
       "14688     cool          auto               722                      720   \n",
       "14689     auto          hold               769                      740   \n",
       "\n",
       "       TemperatureExpectedHeat ProvinceState         City  \\\n",
       "0                          680            MT  Great Falls   \n",
       "1                          680            MT  Great Falls   \n",
       "2                          680            MT  Great Falls   \n",
       "3                          680            MT  Great Falls   \n",
       "4                          680            MT  Great Falls   \n",
       "...                        ...           ...          ...   \n",
       "14685                      680            MT  Great Falls   \n",
       "14686                      680            MT       Scobey   \n",
       "14687                      680            MT       Scobey   \n",
       "14688                      680            MT       Scobey   \n",
       "14689                      680            MT  Great Falls   \n",
       "\n",
       "       Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                       45             False         False            False   \n",
       "1                       45             False         False            False   \n",
       "2                       45             False         False            False   \n",
       "3                       45             False         False            False   \n",
       "4                       45             False         False            False   \n",
       "...                    ...               ...           ...              ...   \n",
       "14685                   50             False         False            False   \n",
       "14686                   35             False         False            False   \n",
       "14687                   35             False         False            False   \n",
       "14688                   35             False         False            False   \n",
       "14689                   50             False         False            False   \n",
       "\n",
       "      Auxilliary_Heat_Fuel_Type  \n",
       "0                           Gas  \n",
       "1                           Gas  \n",
       "2                           Gas  \n",
       "3                           Gas  \n",
       "4                           Gas  \n",
       "...                         ...  \n",
       "14685                       Gas  \n",
       "14686                       Gas  \n",
       "14687                       Gas  \n",
       "14688                       Gas  \n",
       "14689                       Gas  \n",
       "\n",
       "[11260 rows x 14 columns]"
      ]
     },
     "execution_count": 102,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "jul_2017.drop(jul_2017[jul_2017['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "jul_2017.drop(jul_2017[jul_2017['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "jul_2017.drop(jul_2017[jul_2017['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "jul_2017.drop(jul_2017[jul_2017['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "jul_2017"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "jul_2017[\"Year\"] = \"2017\"\n",
    "jul_2017[\"Month\"] = \"jul\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "jul_2017 = jul_2017.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "jul_2017_ave = jul_2017.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# jul_2017_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "jul_2017_ave.to_csv(\"data/day/MT/jul/jul_2017_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2018 July Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "jul_2018 = pd.read_csv(\"../data_large/MT-day/2018-jul-day-MT.csv\")\n",
    "\n",
    "# jul_2018"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 108,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>e4ed8ab8303e5e1de4b181ce40545279719e6069</td>\n",
       "      <td>2018-07-27T13:45:00Z</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>690</td>\n",
       "      <td>650</td>\n",
       "      <td>630</td>\n",
       "      <td>MT</td>\n",
       "      <td>Bozeman</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>e4ed8ab8303e5e1de4b181ce40545279719e6069</td>\n",
       "      <td>2018-07-16T14:00:00Z</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>680</td>\n",
       "      <td>650</td>\n",
       "      <td>630</td>\n",
       "      <td>MT</td>\n",
       "      <td>Bozeman</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>e4ed8ab8303e5e1de4b181ce40545279719e6069</td>\n",
       "      <td>2018-07-30T12:40:00Z</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>692</td>\n",
       "      <td>650</td>\n",
       "      <td>630</td>\n",
       "      <td>MT</td>\n",
       "      <td>Bozeman</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>e4ed8ab8303e5e1de4b181ce40545279719e6069</td>\n",
       "      <td>2018-07-17T14:10:00Z</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>669</td>\n",
       "      <td>650</td>\n",
       "      <td>630</td>\n",
       "      <td>MT</td>\n",
       "      <td>Bozeman</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>e4ed8ab8303e5e1de4b181ce40545279719e6069</td>\n",
       "      <td>2018-07-11T15:35:00Z</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>661</td>\n",
       "      <td>650</td>\n",
       "      <td>630</td>\n",
       "      <td>MT</td>\n",
       "      <td>Bozeman</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26346</th>\n",
       "      <td>5a1fbc37826124394a837d94e7f46800280d115f</td>\n",
       "      <td>2018-07-08T15:55:00Z</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>732</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>MT</td>\n",
       "      <td>Great Falls</td>\n",
       "      <td>70</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26347</th>\n",
       "      <td>5a1fbc37826124394a837d94e7f46800280d115f</td>\n",
       "      <td>2018-07-07T14:05:00Z</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>744</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>MT</td>\n",
       "      <td>Great Falls</td>\n",
       "      <td>70</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26348</th>\n",
       "      <td>5a1fbc37826124394a837d94e7f46800280d115f</td>\n",
       "      <td>2018-07-08T15:20:00Z</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>732</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>MT</td>\n",
       "      <td>Great Falls</td>\n",
       "      <td>70</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26349</th>\n",
       "      <td>5a1fbc37826124394a837d94e7f46800280d115f</td>\n",
       "      <td>2018-07-08T15:25:00Z</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>732</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>MT</td>\n",
       "      <td>Great Falls</td>\n",
       "      <td>70</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26350</th>\n",
       "      <td>5a1fbc37826124394a837d94e7f46800280d115f</td>\n",
       "      <td>2018-07-08T13:50:00Z</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>734</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>MT</td>\n",
       "      <td>Great Falls</td>\n",
       "      <td>70</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>23841 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                     Identifier             date_time  \\\n",
       "0      e4ed8ab8303e5e1de4b181ce40545279719e6069  2018-07-27T13:45:00Z   \n",
       "1      e4ed8ab8303e5e1de4b181ce40545279719e6069  2018-07-16T14:00:00Z   \n",
       "2      e4ed8ab8303e5e1de4b181ce40545279719e6069  2018-07-30T12:40:00Z   \n",
       "3      e4ed8ab8303e5e1de4b181ce40545279719e6069  2018-07-17T14:10:00Z   \n",
       "4      e4ed8ab8303e5e1de4b181ce40545279719e6069  2018-07-11T15:35:00Z   \n",
       "...                                         ...                   ...   \n",
       "26346  5a1fbc37826124394a837d94e7f46800280d115f  2018-07-08T15:55:00Z   \n",
       "26347  5a1fbc37826124394a837d94e7f46800280d115f  2018-07-07T14:05:00Z   \n",
       "26348  5a1fbc37826124394a837d94e7f46800280d115f  2018-07-08T15:20:00Z   \n",
       "26349  5a1fbc37826124394a837d94e7f46800280d115f  2018-07-08T15:25:00Z   \n",
       "26350  5a1fbc37826124394a837d94e7f46800280d115f  2018-07-08T13:50:00Z   \n",
       "\n",
       "      HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0         heat          hold               690                      650   \n",
       "1         heat          hold               680                      650   \n",
       "2         heat          hold               692                      650   \n",
       "3         heat          hold               669                      650   \n",
       "4         heat          hold               661                      650   \n",
       "...        ...           ...               ...                      ...   \n",
       "26346     cool          auto               732                      760   \n",
       "26347     cool          auto               744                      760   \n",
       "26348     cool          auto               732                      760   \n",
       "26349     cool          auto               732                      760   \n",
       "26350     cool          auto               734                      760   \n",
       "\n",
       "       TemperatureExpectedHeat ProvinceState         City  \\\n",
       "0                          630            MT      Bozeman   \n",
       "1                          630            MT      Bozeman   \n",
       "2                          630            MT      Bozeman   \n",
       "3                          630            MT      Bozeman   \n",
       "4                          630            MT      Bozeman   \n",
       "...                        ...           ...          ...   \n",
       "26346                      760            MT  Great Falls   \n",
       "26347                      760            MT  Great Falls   \n",
       "26348                      760            MT  Great Falls   \n",
       "26349                      760            MT  Great Falls   \n",
       "26350                      760            MT  Great Falls   \n",
       "\n",
       "       Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                       10             False         False            False   \n",
       "1                       10             False         False            False   \n",
       "2                       10             False         False            False   \n",
       "3                       10             False         False            False   \n",
       "4                       10             False         False            False   \n",
       "...                    ...               ...           ...              ...   \n",
       "26346                   70             False         False            False   \n",
       "26347                   70             False         False            False   \n",
       "26348                   70             False         False            False   \n",
       "26349                   70             False         False            False   \n",
       "26350                   70             False         False            False   \n",
       "\n",
       "      Auxilliary_Heat_Fuel_Type  \n",
       "0                           Gas  \n",
       "1                           Gas  \n",
       "2                           Gas  \n",
       "3                           Gas  \n",
       "4                           Gas  \n",
       "...                         ...  \n",
       "26346                       Gas  \n",
       "26347                       Gas  \n",
       "26348                       Gas  \n",
       "26349                       Gas  \n",
       "26350                       Gas  \n",
       "\n",
       "[23841 rows x 14 columns]"
      ]
     },
     "execution_count": 108,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "jul_2018.drop(jul_2018[jul_2018['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "jul_2018.drop(jul_2018[jul_2018['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "jul_2018.drop(jul_2018[jul_2018['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "jul_2018.drop(jul_2018[jul_2018['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "jul_2018"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "jul_2018[\"Year\"] = \"2018\"\n",
    "jul_2018[\"Month\"] = \"jul\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "jul_2018 = jul_2018.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 111,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "jul_2018_ave = jul_2018.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# jul_2018_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 112,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "jul_2018_ave.to_csv(\"data/day/MT/jul/jul_2018_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2019 July Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 113,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "jul_2019 = pd.read_csv(\"../data_large/MT-day/2019-jul-day-MT.csv\")\n",
    "\n",
    "# jul_2019"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 114,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>7352a85776176198f0abb065b891160881b30270</td>\n",
       "      <td>2019-07-13 19:25:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>714</td>\n",
       "      <td>707</td>\n",
       "      <td>707</td>\n",
       "      <td>MT</td>\n",
       "      <td>Bozeman</td>\n",
       "      <td>15</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>7352a85776176198f0abb065b891160881b30270</td>\n",
       "      <td>2019-07-14 18:00:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>709</td>\n",
       "      <td>697</td>\n",
       "      <td>697</td>\n",
       "      <td>MT</td>\n",
       "      <td>Bozeman</td>\n",
       "      <td>15</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>7352a85776176198f0abb065b891160881b30270</td>\n",
       "      <td>2019-07-14 19:55:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>718</td>\n",
       "      <td>697</td>\n",
       "      <td>697</td>\n",
       "      <td>MT</td>\n",
       "      <td>Bozeman</td>\n",
       "      <td>15</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2f91f344abd5823f11abdfa57e08bb514aa09564</td>\n",
       "      <td>2019-07-13 16:35:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>730</td>\n",
       "      <td>760</td>\n",
       "      <td>749</td>\n",
       "      <td>MT</td>\n",
       "      <td>Billings</td>\n",
       "      <td>50</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>c9a27b6d42a3b1e3d96263a706132ded7796fb6f</td>\n",
       "      <td>2019-07-04 13:55:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>709</td>\n",
       "      <td>808</td>\n",
       "      <td>658</td>\n",
       "      <td>MT</td>\n",
       "      <td>Great Falls</td>\n",
       "      <td>50</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>59306</th>\n",
       "      <td>7012732a88f8e228eeeb1ade7f2f27d367c51866</td>\n",
       "      <td>2019-07-12 13:25:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>711</td>\n",
       "      <td>780</td>\n",
       "      <td>680</td>\n",
       "      <td>MT</td>\n",
       "      <td>Bozeman</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>59307</th>\n",
       "      <td>7012732a88f8e228eeeb1ade7f2f27d367c51866</td>\n",
       "      <td>2019-07-15 18:45:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>730</td>\n",
       "      <td>780</td>\n",
       "      <td>680</td>\n",
       "      <td>MT</td>\n",
       "      <td>Bozeman</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>59308</th>\n",
       "      <td>7012732a88f8e228eeeb1ade7f2f27d367c51866</td>\n",
       "      <td>2019-07-19 17:30:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>684</td>\n",
       "      <td>780</td>\n",
       "      <td>680</td>\n",
       "      <td>MT</td>\n",
       "      <td>Bozeman</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>59309</th>\n",
       "      <td>7012732a88f8e228eeeb1ade7f2f27d367c51866</td>\n",
       "      <td>2019-07-19 18:40:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>685</td>\n",
       "      <td>780</td>\n",
       "      <td>680</td>\n",
       "      <td>MT</td>\n",
       "      <td>Bozeman</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>59310</th>\n",
       "      <td>019764d7b48dd9d6f1f4ad95451d7d0da2669405</td>\n",
       "      <td>2019-07-08 12:45:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>701</td>\n",
       "      <td>720</td>\n",
       "      <td>680</td>\n",
       "      <td>MT</td>\n",
       "      <td>Billings</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>40810 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                     Identifier                date_time  \\\n",
       "0      7352a85776176198f0abb065b891160881b30270  2019-07-13 19:25:00 UTC   \n",
       "1      7352a85776176198f0abb065b891160881b30270  2019-07-14 18:00:00 UTC   \n",
       "2      7352a85776176198f0abb065b891160881b30270  2019-07-14 19:55:00 UTC   \n",
       "3      2f91f344abd5823f11abdfa57e08bb514aa09564  2019-07-13 16:35:00 UTC   \n",
       "4      c9a27b6d42a3b1e3d96263a706132ded7796fb6f  2019-07-04 13:55:00 UTC   \n",
       "...                                         ...                      ...   \n",
       "59306  7012732a88f8e228eeeb1ade7f2f27d367c51866  2019-07-12 13:25:00 UTC   \n",
       "59307  7012732a88f8e228eeeb1ade7f2f27d367c51866  2019-07-15 18:45:00 UTC   \n",
       "59308  7012732a88f8e228eeeb1ade7f2f27d367c51866  2019-07-19 17:30:00 UTC   \n",
       "59309  7012732a88f8e228eeeb1ade7f2f27d367c51866  2019-07-19 18:40:00 UTC   \n",
       "59310  019764d7b48dd9d6f1f4ad95451d7d0da2669405  2019-07-08 12:45:00 UTC   \n",
       "\n",
       "      HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0         cool          hold               714                      707   \n",
       "1         cool          hold               709                      697   \n",
       "2         cool          hold               718                      697   \n",
       "3         cool          hold               730                      760   \n",
       "4         auto          auto               709                      808   \n",
       "...        ...           ...               ...                      ...   \n",
       "59306     cool          auto               711                      780   \n",
       "59307     cool          auto               730                      780   \n",
       "59308     cool          auto               684                      780   \n",
       "59309     cool          auto               685                      780   \n",
       "59310     auto          auto               701                      720   \n",
       "\n",
       "       TemperatureExpectedHeat ProvinceState         City  \\\n",
       "0                          707            MT      Bozeman   \n",
       "1                          697            MT      Bozeman   \n",
       "2                          697            MT      Bozeman   \n",
       "3                          749            MT     Billings   \n",
       "4                          658            MT  Great Falls   \n",
       "...                        ...           ...          ...   \n",
       "59306                      680            MT      Bozeman   \n",
       "59307                      680            MT      Bozeman   \n",
       "59308                      680            MT      Bozeman   \n",
       "59309                      680            MT      Bozeman   \n",
       "59310                      680            MT     Billings   \n",
       "\n",
       "       Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                       15             False         False            False   \n",
       "1                       15             False         False            False   \n",
       "2                       15             False         False            False   \n",
       "3                       50              True         False            False   \n",
       "4                       50             False         False            False   \n",
       "...                    ...               ...           ...              ...   \n",
       "59306                    5             False         False            False   \n",
       "59307                    5             False         False            False   \n",
       "59308                    5             False         False            False   \n",
       "59309                    5             False         False            False   \n",
       "59310                    5             False         False            False   \n",
       "\n",
       "      Auxilliary_Heat_Fuel_Type  \n",
       "0                           Gas  \n",
       "1                           Gas  \n",
       "2                           Gas  \n",
       "3                           Gas  \n",
       "4                           Gas  \n",
       "...                         ...  \n",
       "59306                       Gas  \n",
       "59307                       Gas  \n",
       "59308                       Gas  \n",
       "59309                       Gas  \n",
       "59310                       Gas  \n",
       "\n",
       "[40810 rows x 14 columns]"
      ]
     },
     "execution_count": 114,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "jul_2019.drop(jul_2019[jul_2019['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "jul_2019.drop(jul_2019[jul_2019['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "jul_2019.drop(jul_2019[jul_2019['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "jul_2019.drop(jul_2019[jul_2019['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "jul_2019"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 115,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "jul_2019[\"Year\"] = \"2019\"\n",
    "jul_2019[\"Month\"] = \"jul\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 116,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "jul_2019 = jul_2019.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 117,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "jul_2019_ave = jul_2019.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# jul_2019_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 118,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "jul_2019_ave.to_csv(\"data/day/MT/jul/jul_2019_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2020 July Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 119,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "jul_2020 = pd.read_csv(\"../data_large/MT-day/2020-jul-day-MT.csv\")\n",
    "\n",
    "# jul_2020"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 120,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>5e65748abfa7413de4b81ab0d609cf2b3d225520</td>\n",
       "      <td>2020-07-26 19:35:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>735</td>\n",
       "      <td>747</td>\n",
       "      <td>747</td>\n",
       "      <td>MT</td>\n",
       "      <td>Missoula</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>f74a23b378011473f8b32b3fc2efbbb3b6a9e098</td>\n",
       "      <td>2020-07-24 17:25:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>711</td>\n",
       "      <td>697</td>\n",
       "      <td>697</td>\n",
       "      <td>MT</td>\n",
       "      <td>Billings</td>\n",
       "      <td>55</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2f91f344abd5823f11abdfa57e08bb514aa09564</td>\n",
       "      <td>2020-07-02 18:35:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>711</td>\n",
       "      <td>701</td>\n",
       "      <td>701</td>\n",
       "      <td>MT</td>\n",
       "      <td>Billings</td>\n",
       "      <td>50</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>f74a23b378011473f8b32b3fc2efbbb3b6a9e098</td>\n",
       "      <td>2020-07-13 18:00:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>701</td>\n",
       "      <td>697</td>\n",
       "      <td>647</td>\n",
       "      <td>MT</td>\n",
       "      <td>Billings</td>\n",
       "      <td>55</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>f74a23b378011473f8b32b3fc2efbbb3b6a9e098</td>\n",
       "      <td>2020-07-23 19:30:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>711</td>\n",
       "      <td>697</td>\n",
       "      <td>697</td>\n",
       "      <td>MT</td>\n",
       "      <td>Billings</td>\n",
       "      <td>55</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>46113</th>\n",
       "      <td>1e4188c276ca852a75feef8c5e3fc27e1f409c4d</td>\n",
       "      <td>2020-07-19 19:55:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>682</td>\n",
       "      <td>680</td>\n",
       "      <td>680</td>\n",
       "      <td>MT</td>\n",
       "      <td>Helena</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>46114</th>\n",
       "      <td>1e4188c276ca852a75feef8c5e3fc27e1f409c4d</td>\n",
       "      <td>2020-07-19 19:50:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>680</td>\n",
       "      <td>680</td>\n",
       "      <td>680</td>\n",
       "      <td>MT</td>\n",
       "      <td>Helena</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>46115</th>\n",
       "      <td>1e4188c276ca852a75feef8c5e3fc27e1f409c4d</td>\n",
       "      <td>2020-07-07 14:30:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>681</td>\n",
       "      <td>680</td>\n",
       "      <td>680</td>\n",
       "      <td>MT</td>\n",
       "      <td>Helena</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>46116</th>\n",
       "      <td>1e4188c276ca852a75feef8c5e3fc27e1f409c4d</td>\n",
       "      <td>2020-07-26 19:20:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>694</td>\n",
       "      <td>680</td>\n",
       "      <td>680</td>\n",
       "      <td>MT</td>\n",
       "      <td>Helena</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>46117</th>\n",
       "      <td>1e4188c276ca852a75feef8c5e3fc27e1f409c4d</td>\n",
       "      <td>2020-07-07 13:55:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>697</td>\n",
       "      <td>680</td>\n",
       "      <td>680</td>\n",
       "      <td>MT</td>\n",
       "      <td>Helena</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>38838 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                     Identifier                date_time  \\\n",
       "0      5e65748abfa7413de4b81ab0d609cf2b3d225520  2020-07-26 19:35:00 UTC   \n",
       "1      f74a23b378011473f8b32b3fc2efbbb3b6a9e098  2020-07-24 17:25:00 UTC   \n",
       "2      2f91f344abd5823f11abdfa57e08bb514aa09564  2020-07-02 18:35:00 UTC   \n",
       "3      f74a23b378011473f8b32b3fc2efbbb3b6a9e098  2020-07-13 18:00:00 UTC   \n",
       "4      f74a23b378011473f8b32b3fc2efbbb3b6a9e098  2020-07-23 19:30:00 UTC   \n",
       "...                                         ...                      ...   \n",
       "46113  1e4188c276ca852a75feef8c5e3fc27e1f409c4d  2020-07-19 19:55:00 UTC   \n",
       "46114  1e4188c276ca852a75feef8c5e3fc27e1f409c4d  2020-07-19 19:50:00 UTC   \n",
       "46115  1e4188c276ca852a75feef8c5e3fc27e1f409c4d  2020-07-07 14:30:00 UTC   \n",
       "46116  1e4188c276ca852a75feef8c5e3fc27e1f409c4d  2020-07-26 19:20:00 UTC   \n",
       "46117  1e4188c276ca852a75feef8c5e3fc27e1f409c4d  2020-07-07 13:55:00 UTC   \n",
       "\n",
       "      HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0         cool          hold               735                      747   \n",
       "1         cool          hold               711                      697   \n",
       "2         cool          hold               711                      701   \n",
       "3         auto          hold               701                      697   \n",
       "4         cool          hold               711                      697   \n",
       "...        ...           ...               ...                      ...   \n",
       "46113     cool          auto               682                      680   \n",
       "46114     cool          auto               680                      680   \n",
       "46115     cool          hold               681                      680   \n",
       "46116     cool          hold               694                      680   \n",
       "46117     cool          hold               697                      680   \n",
       "\n",
       "       TemperatureExpectedHeat ProvinceState      City  Age_of_Home__years_  \\\n",
       "0                          747            MT  Missoula                    5   \n",
       "1                          697            MT  Billings                   55   \n",
       "2                          701            MT  Billings                   50   \n",
       "3                          647            MT  Billings                   55   \n",
       "4                          697            MT  Billings                   55   \n",
       "...                        ...           ...       ...                  ...   \n",
       "46113                      680            MT    Helena                    5   \n",
       "46114                      680            MT    Helena                    5   \n",
       "46115                      680            MT    Helena                    5   \n",
       "46116                      680            MT    Helena                    5   \n",
       "46117                      680            MT    Helena                    5   \n",
       "\n",
       "       allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                 False         False            False   \n",
       "1                 False         False            False   \n",
       "2                  True         False            False   \n",
       "3                 False         False            False   \n",
       "4                 False         False            False   \n",
       "...                 ...           ...              ...   \n",
       "46113             False         False            False   \n",
       "46114             False         False            False   \n",
       "46115             False         False            False   \n",
       "46116             False         False            False   \n",
       "46117             False         False            False   \n",
       "\n",
       "      Auxilliary_Heat_Fuel_Type  \n",
       "0                           Gas  \n",
       "1                           Gas  \n",
       "2                           Gas  \n",
       "3                           Gas  \n",
       "4                           Gas  \n",
       "...                         ...  \n",
       "46113                       Gas  \n",
       "46114                       Gas  \n",
       "46115                       Gas  \n",
       "46116                       Gas  \n",
       "46117                       Gas  \n",
       "\n",
       "[38838 rows x 14 columns]"
      ]
     },
     "execution_count": 120,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "jul_2020.drop(jul_2020[jul_2020['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "jul_2020.drop(jul_2020[jul_2020['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "jul_2020.drop(jul_2020[jul_2020['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "jul_2020.drop(jul_2020[jul_2020['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "jul_2020"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 121,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "jul_2020[\"Year\"] = \"2020\"\n",
    "jul_2020[\"Month\"] = \"jul\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 122,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "jul_2020 = jul_2020.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 123,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "jul_2020_ave = jul_2020.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# jul_2020_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 124,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "jul_2020_ave.to_csv(\"data/day/MT/jul/jul_2020_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2021 July Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 125,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "jul_2021 = pd.read_csv(\"../data_large/MT-day/2021-jul-day-MT.csv\")\n",
    "\n",
    "# jul_2021"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 126,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2f91f344abd5823f11abdfa57e08bb514aa09564</td>\n",
       "      <td>2021-07-09T19:05:00Z</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>734</td>\n",
       "      <td>730</td>\n",
       "      <td>721</td>\n",
       "      <td>MT</td>\n",
       "      <td>Billings</td>\n",
       "      <td>50</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2f91f344abd5823f11abdfa57e08bb514aa09564</td>\n",
       "      <td>2021-07-21T15:50:00Z</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>738</td>\n",
       "      <td>740</td>\n",
       "      <td>721</td>\n",
       "      <td>MT</td>\n",
       "      <td>Billings</td>\n",
       "      <td>50</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2f91f344abd5823f11abdfa57e08bb514aa09564</td>\n",
       "      <td>2021-07-09T19:20:00Z</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>733</td>\n",
       "      <td>730</td>\n",
       "      <td>721</td>\n",
       "      <td>MT</td>\n",
       "      <td>Billings</td>\n",
       "      <td>50</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2f91f344abd5823f11abdfa57e08bb514aa09564</td>\n",
       "      <td>2021-07-21T15:00:00Z</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>734</td>\n",
       "      <td>740</td>\n",
       "      <td>721</td>\n",
       "      <td>MT</td>\n",
       "      <td>Billings</td>\n",
       "      <td>50</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2f91f344abd5823f11abdfa57e08bb514aa09564</td>\n",
       "      <td>2021-07-23T18:50:00Z</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>722</td>\n",
       "      <td>730</td>\n",
       "      <td>721</td>\n",
       "      <td>MT</td>\n",
       "      <td>Billings</td>\n",
       "      <td>50</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26853</th>\n",
       "      <td>6e0c9d625f3ce2c91fb0dace7c2f92867c27badb</td>\n",
       "      <td>2021-07-15T19:40:00Z</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>788</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>MT</td>\n",
       "      <td>Scobey</td>\n",
       "      <td>50</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26854</th>\n",
       "      <td>6e0c9d625f3ce2c91fb0dace7c2f92867c27badb</td>\n",
       "      <td>2021-07-15T19:55:00Z</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>776</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>MT</td>\n",
       "      <td>Scobey</td>\n",
       "      <td>50</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26855</th>\n",
       "      <td>b9a68c99fa994007bfbe7dfc018ec73539555478</td>\n",
       "      <td>2021-07-26T19:55:00Z</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>766</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>MT</td>\n",
       "      <td>Billings</td>\n",
       "      <td>60</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26856</th>\n",
       "      <td>b9a68c99fa994007bfbe7dfc018ec73539555478</td>\n",
       "      <td>2021-07-26T19:30:00Z</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>766</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>MT</td>\n",
       "      <td>Billings</td>\n",
       "      <td>60</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26857</th>\n",
       "      <td>b9a68c99fa994007bfbe7dfc018ec73539555478</td>\n",
       "      <td>2021-07-26T19:50:00Z</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>762</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>MT</td>\n",
       "      <td>Billings</td>\n",
       "      <td>60</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>23200 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                     Identifier             date_time  \\\n",
       "0      2f91f344abd5823f11abdfa57e08bb514aa09564  2021-07-09T19:05:00Z   \n",
       "1      2f91f344abd5823f11abdfa57e08bb514aa09564  2021-07-21T15:50:00Z   \n",
       "2      2f91f344abd5823f11abdfa57e08bb514aa09564  2021-07-09T19:20:00Z   \n",
       "3      2f91f344abd5823f11abdfa57e08bb514aa09564  2021-07-21T15:00:00Z   \n",
       "4      2f91f344abd5823f11abdfa57e08bb514aa09564  2021-07-23T18:50:00Z   \n",
       "...                                         ...                   ...   \n",
       "26853  6e0c9d625f3ce2c91fb0dace7c2f92867c27badb  2021-07-15T19:40:00Z   \n",
       "26854  6e0c9d625f3ce2c91fb0dace7c2f92867c27badb  2021-07-15T19:55:00Z   \n",
       "26855  b9a68c99fa994007bfbe7dfc018ec73539555478  2021-07-26T19:55:00Z   \n",
       "26856  b9a68c99fa994007bfbe7dfc018ec73539555478  2021-07-26T19:30:00Z   \n",
       "26857  b9a68c99fa994007bfbe7dfc018ec73539555478  2021-07-26T19:50:00Z   \n",
       "\n",
       "      HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0         cool          hold               734                      730   \n",
       "1         cool          hold               738                      740   \n",
       "2         cool          hold               733                      730   \n",
       "3         cool          hold               734                      740   \n",
       "4         cool          hold               722                      730   \n",
       "...        ...           ...               ...                      ...   \n",
       "26853     cool          hold               788                      760   \n",
       "26854     cool          hold               776                      760   \n",
       "26855     cool          hold               766                      760   \n",
       "26856     cool          hold               766                      760   \n",
       "26857     cool          hold               762                      760   \n",
       "\n",
       "       TemperatureExpectedHeat ProvinceState      City  Age_of_Home__years_  \\\n",
       "0                          721            MT  Billings                   50   \n",
       "1                          721            MT  Billings                   50   \n",
       "2                          721            MT  Billings                   50   \n",
       "3                          721            MT  Billings                   50   \n",
       "4                          721            MT  Billings                   50   \n",
       "...                        ...           ...       ...                  ...   \n",
       "26853                      760            MT    Scobey                   50   \n",
       "26854                      760            MT    Scobey                   50   \n",
       "26855                      760            MT  Billings                   60   \n",
       "26856                      760            MT  Billings                   60   \n",
       "26857                      760            MT  Billings                   60   \n",
       "\n",
       "       allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                  True         False            False   \n",
       "1                  True         False            False   \n",
       "2                  True         False            False   \n",
       "3                  True         False            False   \n",
       "4                  True         False            False   \n",
       "...                 ...           ...              ...   \n",
       "26853             False         False            False   \n",
       "26854             False         False            False   \n",
       "26855             False         False            False   \n",
       "26856             False         False            False   \n",
       "26857             False         False            False   \n",
       "\n",
       "      Auxilliary_Heat_Fuel_Type  \n",
       "0                           Gas  \n",
       "1                           Gas  \n",
       "2                           Gas  \n",
       "3                           Gas  \n",
       "4                           Gas  \n",
       "...                         ...  \n",
       "26853                       Gas  \n",
       "26854                       Gas  \n",
       "26855                       Gas  \n",
       "26856                       Gas  \n",
       "26857                       Gas  \n",
       "\n",
       "[23200 rows x 14 columns]"
      ]
     },
     "execution_count": 126,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "jul_2021.drop(jul_2021[jul_2021['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "jul_2021.drop(jul_2021[jul_2021['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "jul_2021.drop(jul_2021[jul_2021['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "jul_2021.drop(jul_2021[jul_2021['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "\n",
    "jul_2021"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 127,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "jul_2021[\"Year\"] = \"2021\"\n",
    "jul_2021[\"Month\"] = \"jul\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 128,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "jul_2021 = jul_2021.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 129,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "jul_2021_ave = jul_2021.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# jul_2021_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 130,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "jul_2021_ave.to_csv(\"data/day/MT/jul/jul_2021_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "\n",
    "### Combine month CSV Files \n",
    "1. Read in files in folders for each state\n",
    "2. Export as combined CSV"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 131,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create variable for files in directory\n",
    "files = [f for f in os.listdir(\"data/day/MT/jul/\") if f.endswith(\".csv\")]\n",
    "\n",
    "# files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 132,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>Month</th>\n",
       "      <th>Year</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>City</th>\n",
       "      <th>Temperature_ctrl_ave</th>\n",
       "      <th>TemperatureExpectedCool_ave</th>\n",
       "      <th>TemperatureExpectedHeat_ave</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0bc51b95b2dc23713f611e182c689ad03dc2c300</td>\n",
       "      <td>jul</td>\n",
       "      <td>2017</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>Great Falls</td>\n",
       "      <td>708.000000</td>\n",
       "      <td>735.000000</td>\n",
       "      <td>740.000000</td>\n",
       "      <td>10.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0bc51b95b2dc23713f611e182c689ad03dc2c300</td>\n",
       "      <td>jul</td>\n",
       "      <td>2017</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>Great Falls</td>\n",
       "      <td>724.242424</td>\n",
       "      <td>740.393939</td>\n",
       "      <td>740.393939</td>\n",
       "      <td>10.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1e4188c276ca852a75feef8c5e3fc27e1f409c4d</td>\n",
       "      <td>jul</td>\n",
       "      <td>2017</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>Helena</td>\n",
       "      <td>715.400000</td>\n",
       "      <td>713.440000</td>\n",
       "      <td>699.200000</td>\n",
       "      <td>5.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1e4188c276ca852a75feef8c5e3fc27e1f409c4d</td>\n",
       "      <td>jul</td>\n",
       "      <td>2017</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>Helena</td>\n",
       "      <td>704.176471</td>\n",
       "      <td>700.000000</td>\n",
       "      <td>700.000000</td>\n",
       "      <td>5.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>29f5966d7808882d6c76a0a2d7091c18b3587704</td>\n",
       "      <td>jul</td>\n",
       "      <td>2017</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>Bozeman</td>\n",
       "      <td>702.916667</td>\n",
       "      <td>700.791667</td>\n",
       "      <td>650.791667</td>\n",
       "      <td>9.0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33</th>\n",
       "      <td>e4ed8ab8303e5e1de4b181ce40545279719e6069</td>\n",
       "      <td>jul</td>\n",
       "      <td>2021</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>Bozeman</td>\n",
       "      <td>734.737113</td>\n",
       "      <td>680.000000</td>\n",
       "      <td>680.000000</td>\n",
       "      <td>10.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>34</th>\n",
       "      <td>f404081c89442a6b4699e93e43dc4bb72f9e2dc1</td>\n",
       "      <td>jul</td>\n",
       "      <td>2021</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>Great Falls</td>\n",
       "      <td>730.131274</td>\n",
       "      <td>730.928958</td>\n",
       "      <td>680.928958</td>\n",
       "      <td>55.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>35</th>\n",
       "      <td>f74a23b378011473f8b32b3fc2efbbb3b6a9e098</td>\n",
       "      <td>jul</td>\n",
       "      <td>2021</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>Billings</td>\n",
       "      <td>659.458333</td>\n",
       "      <td>655.291667</td>\n",
       "      <td>654.833333</td>\n",
       "      <td>55.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>36</th>\n",
       "      <td>fd93662943beeb4603fd63aacbbbb7abb3183470</td>\n",
       "      <td>jul</td>\n",
       "      <td>2021</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>NASHUA</td>\n",
       "      <td>738.500000</td>\n",
       "      <td>769.791667</td>\n",
       "      <td>769.500000</td>\n",
       "      <td>30.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>37</th>\n",
       "      <td>fec2ec7ae8c5ee7ae93fe16b022bc4c1fc6f2931</td>\n",
       "      <td>jul</td>\n",
       "      <td>2021</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>Kalispell</td>\n",
       "      <td>709.250000</td>\n",
       "      <td>700.000000</td>\n",
       "      <td>680.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>253 rows × 13 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                  Identifier Month  Year HvacMode  \\\n",
       "0   0bc51b95b2dc23713f611e182c689ad03dc2c300   jul  2017     cool   \n",
       "1   0bc51b95b2dc23713f611e182c689ad03dc2c300   jul  2017     cool   \n",
       "2   1e4188c276ca852a75feef8c5e3fc27e1f409c4d   jul  2017     cool   \n",
       "3   1e4188c276ca852a75feef8c5e3fc27e1f409c4d   jul  2017     cool   \n",
       "4   29f5966d7808882d6c76a0a2d7091c18b3587704   jul  2017     auto   \n",
       "..                                       ...   ...   ...      ...   \n",
       "33  e4ed8ab8303e5e1de4b181ce40545279719e6069   jul  2021     heat   \n",
       "34  f404081c89442a6b4699e93e43dc4bb72f9e2dc1   jul  2021     auto   \n",
       "35  f74a23b378011473f8b32b3fc2efbbb3b6a9e098   jul  2021     cool   \n",
       "36  fd93662943beeb4603fd63aacbbbb7abb3183470   jul  2021     cool   \n",
       "37  fec2ec7ae8c5ee7ae93fe16b022bc4c1fc6f2931   jul  2021     auto   \n",
       "\n",
       "   CalendarEvent         City  Temperature_ctrl_ave  \\\n",
       "0           auto  Great Falls            708.000000   \n",
       "1           hold  Great Falls            724.242424   \n",
       "2           auto       Helena            715.400000   \n",
       "3           hold       Helena            704.176471   \n",
       "4           auto      Bozeman            702.916667   \n",
       "..           ...          ...                   ...   \n",
       "33          hold      Bozeman            734.737113   \n",
       "34          hold  Great Falls            730.131274   \n",
       "35          hold     Billings            659.458333   \n",
       "36          hold       NASHUA            738.500000   \n",
       "37          hold    Kalispell            709.250000   \n",
       "\n",
       "    TemperatureExpectedCool_ave  TemperatureExpectedHeat_ave  \\\n",
       "0                    735.000000                   740.000000   \n",
       "1                    740.393939                   740.393939   \n",
       "2                    713.440000                   699.200000   \n",
       "3                    700.000000                   700.000000   \n",
       "4                    700.791667                   650.791667   \n",
       "..                          ...                          ...   \n",
       "33                   680.000000                   680.000000   \n",
       "34                   730.928958                   680.928958   \n",
       "35                   655.291667                   654.833333   \n",
       "36                   769.791667                   769.500000   \n",
       "37                   700.000000                   680.000000   \n",
       "\n",
       "    Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \n",
       "0                  10.0             False         False            False  \n",
       "1                  10.0             False         False            False  \n",
       "2                   5.0             False         False            False  \n",
       "3                   5.0             False         False            False  \n",
       "4                   9.0              True         False            False  \n",
       "..                  ...               ...           ...              ...  \n",
       "33                 10.0             False         False            False  \n",
       "34                 55.0             False         False            False  \n",
       "35                 55.0             False         False            False  \n",
       "36                 30.0             False         False            False  \n",
       "37                  0.0              True         False            False  \n",
       "\n",
       "[253 rows x 13 columns]"
      ]
     },
     "execution_count": 132,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# https://stacMToverflow.com/questions/63886787/how-to-create-a-dataframe-from-multiple-csv-files\n",
    "# Initialize an empty dataframe\n",
    "MT_jul = pd.DataFrame()\n",
    "\n",
    "# Iterate through files and contents, then concatenate the data into the dataframe\n",
    "for file in files:\n",
    "    df = pd.read_csv(\"data/day/MT/jul/\" + file)\n",
    "    MT_jul = pd.concat([MT_jul, df])\n",
    "    \n",
    "MT_jul"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 133,
   "metadata": {},
   "outputs": [],
   "source": [
    "MT_jul.to_csv(\"Scraper_Output/State_Month_Day/MT/MT_jul.csv\", header=True, index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "\n",
    "## August"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2017 August Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 134,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "aug_2017 = pd.read_csv(\"../data_large/MT-day/2017-aug-day-MT.csv\")\n",
    "\n",
    "# aug_2017"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 135,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>a91496a0efdd5e852e400a057b99fb69b67eed62</td>\n",
       "      <td>2017-08-26T17:25:00Z</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>719</td>\n",
       "      <td>692</td>\n",
       "      <td>692</td>\n",
       "      <td>MT</td>\n",
       "      <td>Bozeman</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>a91496a0efdd5e852e400a057b99fb69b67eed62</td>\n",
       "      <td>2017-08-25T17:10:00Z</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>693</td>\n",
       "      <td>701</td>\n",
       "      <td>701</td>\n",
       "      <td>MT</td>\n",
       "      <td>Bozeman</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>a91496a0efdd5e852e400a057b99fb69b67eed62</td>\n",
       "      <td>2017-08-26T17:35:00Z</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>701</td>\n",
       "      <td>692</td>\n",
       "      <td>692</td>\n",
       "      <td>MT</td>\n",
       "      <td>Bozeman</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>037988cc24eaeaa08f7002dbc926d02c8f9c0f1c</td>\n",
       "      <td>2017-08-18T15:25:00Z</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>683</td>\n",
       "      <td>725</td>\n",
       "      <td>725</td>\n",
       "      <td>MT</td>\n",
       "      <td>Eureka</td>\n",
       "      <td>10</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>037988cc24eaeaa08f7002dbc926d02c8f9c0f1c</td>\n",
       "      <td>2017-08-12T15:55:00Z</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>693</td>\n",
       "      <td>724</td>\n",
       "      <td>724</td>\n",
       "      <td>MT</td>\n",
       "      <td>Eureka</td>\n",
       "      <td>10</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19565</th>\n",
       "      <td>6e0c9d625f3ce2c91fb0dace7c2f92867c27badb</td>\n",
       "      <td>2017-08-20T19:20:00Z</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>707</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>MT</td>\n",
       "      <td>Scobey</td>\n",
       "      <td>50</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19566</th>\n",
       "      <td>6e0c9d625f3ce2c91fb0dace7c2f92867c27badb</td>\n",
       "      <td>2017-08-04T14:25:00Z</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>708</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>MT</td>\n",
       "      <td>Scobey</td>\n",
       "      <td>50</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19567</th>\n",
       "      <td>6e0c9d625f3ce2c91fb0dace7c2f92867c27badb</td>\n",
       "      <td>2017-08-28T14:35:00Z</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>730</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>MT</td>\n",
       "      <td>Scobey</td>\n",
       "      <td>50</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19568</th>\n",
       "      <td>6e0c9d625f3ce2c91fb0dace7c2f92867c27badb</td>\n",
       "      <td>2017-08-24T17:55:00Z</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>755</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>MT</td>\n",
       "      <td>Scobey</td>\n",
       "      <td>50</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19569</th>\n",
       "      <td>6e0c9d625f3ce2c91fb0dace7c2f92867c27badb</td>\n",
       "      <td>2017-08-25T17:25:00Z</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>753</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>MT</td>\n",
       "      <td>Scobey</td>\n",
       "      <td>50</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>18788 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                     Identifier             date_time  \\\n",
       "0      a91496a0efdd5e852e400a057b99fb69b67eed62  2017-08-26T17:25:00Z   \n",
       "1      a91496a0efdd5e852e400a057b99fb69b67eed62  2017-08-25T17:10:00Z   \n",
       "2      a91496a0efdd5e852e400a057b99fb69b67eed62  2017-08-26T17:35:00Z   \n",
       "3      037988cc24eaeaa08f7002dbc926d02c8f9c0f1c  2017-08-18T15:25:00Z   \n",
       "4      037988cc24eaeaa08f7002dbc926d02c8f9c0f1c  2017-08-12T15:55:00Z   \n",
       "...                                         ...                   ...   \n",
       "19565  6e0c9d625f3ce2c91fb0dace7c2f92867c27badb  2017-08-20T19:20:00Z   \n",
       "19566  6e0c9d625f3ce2c91fb0dace7c2f92867c27badb  2017-08-04T14:25:00Z   \n",
       "19567  6e0c9d625f3ce2c91fb0dace7c2f92867c27badb  2017-08-28T14:35:00Z   \n",
       "19568  6e0c9d625f3ce2c91fb0dace7c2f92867c27badb  2017-08-24T17:55:00Z   \n",
       "19569  6e0c9d625f3ce2c91fb0dace7c2f92867c27badb  2017-08-25T17:25:00Z   \n",
       "\n",
       "      HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0         cool          hold               719                      692   \n",
       "1         cool          hold               693                      701   \n",
       "2         cool          hold               701                      692   \n",
       "3         cool          hold               683                      725   \n",
       "4         cool          hold               693                      724   \n",
       "...        ...           ...               ...                      ...   \n",
       "19565     cool          hold               707                      760   \n",
       "19566     cool          hold               708                      760   \n",
       "19567     cool          hold               730                      760   \n",
       "19568     cool          hold               755                      760   \n",
       "19569     cool          hold               753                      760   \n",
       "\n",
       "       TemperatureExpectedHeat ProvinceState     City  Age_of_Home__years_  \\\n",
       "0                          692            MT  Bozeman                   10   \n",
       "1                          701            MT  Bozeman                   10   \n",
       "2                          692            MT  Bozeman                   10   \n",
       "3                          725            MT   Eureka                   10   \n",
       "4                          724            MT   Eureka                   10   \n",
       "...                        ...           ...      ...                  ...   \n",
       "19565                      760            MT   Scobey                   50   \n",
       "19566                      760            MT   Scobey                   50   \n",
       "19567                      760            MT   Scobey                   50   \n",
       "19568                      760            MT   Scobey                   50   \n",
       "19569                      760            MT   Scobey                   50   \n",
       "\n",
       "       allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                 False         False            False   \n",
       "1                 False         False            False   \n",
       "2                 False         False            False   \n",
       "3                  True         False             True   \n",
       "4                  True         False             True   \n",
       "...                 ...           ...              ...   \n",
       "19565             False         False            False   \n",
       "19566             False         False            False   \n",
       "19567             False         False            False   \n",
       "19568             False         False            False   \n",
       "19569             False         False            False   \n",
       "\n",
       "      Auxilliary_Heat_Fuel_Type  \n",
       "0                           Gas  \n",
       "1                           Gas  \n",
       "2                           Gas  \n",
       "3                      Electric  \n",
       "4                      Electric  \n",
       "...                         ...  \n",
       "19565                       Gas  \n",
       "19566                       Gas  \n",
       "19567                       Gas  \n",
       "19568                       Gas  \n",
       "19569                       Gas  \n",
       "\n",
       "[18788 rows x 14 columns]"
      ]
     },
     "execution_count": 135,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "aug_2017.drop(aug_2017[aug_2017['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "aug_2017.drop(aug_2017[aug_2017['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "aug_2017.drop(aug_2017[aug_2017['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "aug_2017.drop(aug_2017[aug_2017['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "aug_2017"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 136,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "aug_2017[\"Year\"] = \"2017\"\n",
    "aug_2017[\"Month\"] = \"aug\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 137,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "aug_2017 = aug_2017.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 138,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "aug_2017_ave = aug_2017.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# aug_2017_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 139,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "aug_2017_ave.to_csv(\"data/day/MT/aug/aug_2017_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2018 August Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 140,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "aug_2018 = pd.read_csv(\"../data_large/MT-day/2018-aug-day-MT.csv\")\n",
    "\n",
    "# aug_2018"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 141,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>bbc0d748de67c33d4c70a501da752924a90777ee</td>\n",
       "      <td>2018-08-10T13:35:00Z</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>747</td>\n",
       "      <td>760</td>\n",
       "      <td>700</td>\n",
       "      <td>MT</td>\n",
       "      <td>Kalispell</td>\n",
       "      <td>45</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>bbc0d748de67c33d4c70a501da752924a90777ee</td>\n",
       "      <td>2018-08-08T12:00:00Z</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>755</td>\n",
       "      <td>770</td>\n",
       "      <td>700</td>\n",
       "      <td>MT</td>\n",
       "      <td>Kalispell</td>\n",
       "      <td>45</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>bbc0d748de67c33d4c70a501da752924a90777ee</td>\n",
       "      <td>2018-08-19T19:35:00Z</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>734</td>\n",
       "      <td>750</td>\n",
       "      <td>700</td>\n",
       "      <td>MT</td>\n",
       "      <td>Kalispell</td>\n",
       "      <td>45</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>bbc0d748de67c33d4c70a501da752924a90777ee</td>\n",
       "      <td>2018-08-08T13:15:00Z</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>750</td>\n",
       "      <td>770</td>\n",
       "      <td>700</td>\n",
       "      <td>MT</td>\n",
       "      <td>Kalispell</td>\n",
       "      <td>45</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>bbc0d748de67c33d4c70a501da752924a90777ee</td>\n",
       "      <td>2018-08-19T15:45:00Z</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>736</td>\n",
       "      <td>750</td>\n",
       "      <td>700</td>\n",
       "      <td>MT</td>\n",
       "      <td>Kalispell</td>\n",
       "      <td>45</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25131</th>\n",
       "      <td>b70abf6012a52a334b0b505e90e47e1d4bc1714e</td>\n",
       "      <td>2018-08-06T13:50:00Z</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>726</td>\n",
       "      <td>800</td>\n",
       "      <td>680</td>\n",
       "      <td>MT</td>\n",
       "      <td>Bozeman</td>\n",
       "      <td>0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25132</th>\n",
       "      <td>4089d9705650bdd49cd283f6660b95a5ed85aba7</td>\n",
       "      <td>2018-08-19T18:50:00Z</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>743</td>\n",
       "      <td>780</td>\n",
       "      <td>680</td>\n",
       "      <td>MT</td>\n",
       "      <td>missoula</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25133</th>\n",
       "      <td>b70abf6012a52a334b0b505e90e47e1d4bc1714e</td>\n",
       "      <td>2018-08-06T15:50:00Z</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>723</td>\n",
       "      <td>800</td>\n",
       "      <td>680</td>\n",
       "      <td>MT</td>\n",
       "      <td>Bozeman</td>\n",
       "      <td>0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25134</th>\n",
       "      <td>b70abf6012a52a334b0b505e90e47e1d4bc1714e</td>\n",
       "      <td>2018-08-06T18:00:00Z</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>724</td>\n",
       "      <td>800</td>\n",
       "      <td>680</td>\n",
       "      <td>MT</td>\n",
       "      <td>Bozeman</td>\n",
       "      <td>0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25135</th>\n",
       "      <td>b70abf6012a52a334b0b505e90e47e1d4bc1714e</td>\n",
       "      <td>2018-08-06T14:50:00Z</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>724</td>\n",
       "      <td>800</td>\n",
       "      <td>680</td>\n",
       "      <td>MT</td>\n",
       "      <td>Bozeman</td>\n",
       "      <td>0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>20925 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                     Identifier             date_time  \\\n",
       "0      bbc0d748de67c33d4c70a501da752924a90777ee  2018-08-10T13:35:00Z   \n",
       "1      bbc0d748de67c33d4c70a501da752924a90777ee  2018-08-08T12:00:00Z   \n",
       "2      bbc0d748de67c33d4c70a501da752924a90777ee  2018-08-19T19:35:00Z   \n",
       "3      bbc0d748de67c33d4c70a501da752924a90777ee  2018-08-08T13:15:00Z   \n",
       "4      bbc0d748de67c33d4c70a501da752924a90777ee  2018-08-19T15:45:00Z   \n",
       "...                                         ...                   ...   \n",
       "25131  b70abf6012a52a334b0b505e90e47e1d4bc1714e  2018-08-06T13:50:00Z   \n",
       "25132  4089d9705650bdd49cd283f6660b95a5ed85aba7  2018-08-19T18:50:00Z   \n",
       "25133  b70abf6012a52a334b0b505e90e47e1d4bc1714e  2018-08-06T15:50:00Z   \n",
       "25134  b70abf6012a52a334b0b505e90e47e1d4bc1714e  2018-08-06T18:00:00Z   \n",
       "25135  b70abf6012a52a334b0b505e90e47e1d4bc1714e  2018-08-06T14:50:00Z   \n",
       "\n",
       "      HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0         auto          auto               747                      760   \n",
       "1         auto          hold               755                      770   \n",
       "2         auto          auto               734                      750   \n",
       "3         auto          hold               750                      770   \n",
       "4         auto          auto               736                      750   \n",
       "...        ...           ...               ...                      ...   \n",
       "25131     auto          auto               726                      800   \n",
       "25132     cool          hold               743                      780   \n",
       "25133     auto          auto               723                      800   \n",
       "25134     auto          auto               724                      800   \n",
       "25135     auto          auto               724                      800   \n",
       "\n",
       "       TemperatureExpectedHeat ProvinceState       City  Age_of_Home__years_  \\\n",
       "0                          700            MT  Kalispell                   45   \n",
       "1                          700            MT  Kalispell                   45   \n",
       "2                          700            MT  Kalispell                   45   \n",
       "3                          700            MT  Kalispell                   45   \n",
       "4                          700            MT  Kalispell                   45   \n",
       "...                        ...           ...        ...                  ...   \n",
       "25131                      680            MT    Bozeman                    0   \n",
       "25132                      680            MT   missoula                    5   \n",
       "25133                      680            MT    Bozeman                    0   \n",
       "25134                      680            MT    Bozeman                    0   \n",
       "25135                      680            MT    Bozeman                    0   \n",
       "\n",
       "       allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                  True         False            False   \n",
       "1                  True         False            False   \n",
       "2                  True         False            False   \n",
       "3                  True         False            False   \n",
       "4                  True         False            False   \n",
       "...                 ...           ...              ...   \n",
       "25131              True         False            False   \n",
       "25132             False         False            False   \n",
       "25133              True         False            False   \n",
       "25134              True         False            False   \n",
       "25135              True         False            False   \n",
       "\n",
       "      Auxilliary_Heat_Fuel_Type  \n",
       "0                           Gas  \n",
       "1                           Gas  \n",
       "2                           Gas  \n",
       "3                           Gas  \n",
       "4                           Gas  \n",
       "...                         ...  \n",
       "25131                       Gas  \n",
       "25132                       Gas  \n",
       "25133                       Gas  \n",
       "25134                       Gas  \n",
       "25135                       Gas  \n",
       "\n",
       "[20925 rows x 14 columns]"
      ]
     },
     "execution_count": 141,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "aug_2018.drop(aug_2018[aug_2018['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "aug_2018.drop(aug_2018[aug_2018['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "aug_2018.drop(aug_2018[aug_2018['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "aug_2018.drop(aug_2018[aug_2018['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "aug_2018"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 142,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "aug_2018[\"Year\"] = \"2018\"\n",
    "aug_2018[\"Month\"] = \"aug\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 143,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "aug_2018 = aug_2018.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 144,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "aug_2018_ave = aug_2018.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# aug_2018_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 145,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "aug_2018_ave.to_csv(\"data/day/MT/aug/aug_2018_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2019 August Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 146,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "aug_2019 = pd.read_csv(\"../data_large/MT-day/2019-aug-day-MT.csv\")\n",
    "\n",
    "# aug_2019"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 147,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>af3e698ef64b8dd05c26eac5bee213b0661dfbc7</td>\n",
       "      <td>2019-08-04 11:40:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>708</td>\n",
       "      <td>790</td>\n",
       "      <td>790</td>\n",
       "      <td>MT</td>\n",
       "      <td>Helena</td>\n",
       "      <td>59</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>470ef66ea25b7872c4c3059e614ac71ba471cde8</td>\n",
       "      <td>2019-08-21 15:00:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>700</td>\n",
       "      <td>682</td>\n",
       "      <td>682</td>\n",
       "      <td>MT</td>\n",
       "      <td>Billings</td>\n",
       "      <td>9</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>af3e698ef64b8dd05c26eac5bee213b0661dfbc7</td>\n",
       "      <td>2019-08-05 13:45:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>711</td>\n",
       "      <td>800</td>\n",
       "      <td>790</td>\n",
       "      <td>MT</td>\n",
       "      <td>Helena</td>\n",
       "      <td>59</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>af3e698ef64b8dd05c26eac5bee213b0661dfbc7</td>\n",
       "      <td>2019-08-04 16:20:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>709</td>\n",
       "      <td>790</td>\n",
       "      <td>790</td>\n",
       "      <td>MT</td>\n",
       "      <td>Helena</td>\n",
       "      <td>59</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2f91f344abd5823f11abdfa57e08bb514aa09564</td>\n",
       "      <td>2019-08-10 19:50:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>748</td>\n",
       "      <td>750</td>\n",
       "      <td>749</td>\n",
       "      <td>MT</td>\n",
       "      <td>Billings</td>\n",
       "      <td>50</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>56328</th>\n",
       "      <td>bbc0d748de67c33d4c70a501da752924a90777ee</td>\n",
       "      <td>2019-08-04 14:25:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>733</td>\n",
       "      <td>770</td>\n",
       "      <td>700</td>\n",
       "      <td>MT</td>\n",
       "      <td>Kalispell</td>\n",
       "      <td>45</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>56329</th>\n",
       "      <td>bbc0d748de67c33d4c70a501da752924a90777ee</td>\n",
       "      <td>2019-08-04 18:30:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>732</td>\n",
       "      <td>770</td>\n",
       "      <td>700</td>\n",
       "      <td>MT</td>\n",
       "      <td>Kalispell</td>\n",
       "      <td>45</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>56330</th>\n",
       "      <td>bbc0d748de67c33d4c70a501da752924a90777ee</td>\n",
       "      <td>2019-08-08 11:15:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>756</td>\n",
       "      <td>760</td>\n",
       "      <td>700</td>\n",
       "      <td>MT</td>\n",
       "      <td>Kalispell</td>\n",
       "      <td>45</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>56331</th>\n",
       "      <td>bbc0d748de67c33d4c70a501da752924a90777ee</td>\n",
       "      <td>2019-08-04 16:30:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>729</td>\n",
       "      <td>770</td>\n",
       "      <td>700</td>\n",
       "      <td>MT</td>\n",
       "      <td>Kalispell</td>\n",
       "      <td>45</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>56332</th>\n",
       "      <td>bbc0d748de67c33d4c70a501da752924a90777ee</td>\n",
       "      <td>2019-08-04 13:55:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>735</td>\n",
       "      <td>770</td>\n",
       "      <td>700</td>\n",
       "      <td>MT</td>\n",
       "      <td>Kalispell</td>\n",
       "      <td>45</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>43364 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                     Identifier                date_time  \\\n",
       "0      af3e698ef64b8dd05c26eac5bee213b0661dfbc7  2019-08-04 11:40:00 UTC   \n",
       "1      470ef66ea25b7872c4c3059e614ac71ba471cde8  2019-08-21 15:00:00 UTC   \n",
       "2      af3e698ef64b8dd05c26eac5bee213b0661dfbc7  2019-08-05 13:45:00 UTC   \n",
       "3      af3e698ef64b8dd05c26eac5bee213b0661dfbc7  2019-08-04 16:20:00 UTC   \n",
       "4      2f91f344abd5823f11abdfa57e08bb514aa09564  2019-08-10 19:50:00 UTC   \n",
       "...                                         ...                      ...   \n",
       "56328  bbc0d748de67c33d4c70a501da752924a90777ee  2019-08-04 14:25:00 UTC   \n",
       "56329  bbc0d748de67c33d4c70a501da752924a90777ee  2019-08-04 18:30:00 UTC   \n",
       "56330  bbc0d748de67c33d4c70a501da752924a90777ee  2019-08-08 11:15:00 UTC   \n",
       "56331  bbc0d748de67c33d4c70a501da752924a90777ee  2019-08-04 16:30:00 UTC   \n",
       "56332  bbc0d748de67c33d4c70a501da752924a90777ee  2019-08-04 13:55:00 UTC   \n",
       "\n",
       "      HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0         cool          auto               708                      790   \n",
       "1         cool          hold               700                      682   \n",
       "2         cool          auto               711                      800   \n",
       "3         cool          auto               709                      790   \n",
       "4         cool          auto               748                      750   \n",
       "...        ...           ...               ...                      ...   \n",
       "56328     auto          auto               733                      770   \n",
       "56329     auto          auto               732                      770   \n",
       "56330     auto          auto               756                      760   \n",
       "56331     auto          auto               729                      770   \n",
       "56332     auto          auto               735                      770   \n",
       "\n",
       "       TemperatureExpectedHeat ProvinceState       City  Age_of_Home__years_  \\\n",
       "0                          790            MT     Helena                   59   \n",
       "1                          682            MT   Billings                    9   \n",
       "2                          790            MT     Helena                   59   \n",
       "3                          790            MT     Helena                   59   \n",
       "4                          749            MT   Billings                   50   \n",
       "...                        ...           ...        ...                  ...   \n",
       "56328                      700            MT  Kalispell                   45   \n",
       "56329                      700            MT  Kalispell                   45   \n",
       "56330                      700            MT  Kalispell                   45   \n",
       "56331                      700            MT  Kalispell                   45   \n",
       "56332                      700            MT  Kalispell                   45   \n",
       "\n",
       "       allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                 False         False            False   \n",
       "1                  True         False            False   \n",
       "2                 False         False            False   \n",
       "3                 False         False            False   \n",
       "4                  True         False            False   \n",
       "...                 ...           ...              ...   \n",
       "56328              True         False            False   \n",
       "56329              True         False            False   \n",
       "56330              True         False            False   \n",
       "56331              True         False            False   \n",
       "56332              True         False            False   \n",
       "\n",
       "      Auxilliary_Heat_Fuel_Type  \n",
       "0                           Gas  \n",
       "1                           Gas  \n",
       "2                           Gas  \n",
       "3                           Gas  \n",
       "4                           Gas  \n",
       "...                         ...  \n",
       "56328                       Gas  \n",
       "56329                       Gas  \n",
       "56330                       Gas  \n",
       "56331                       Gas  \n",
       "56332                       Gas  \n",
       "\n",
       "[43364 rows x 14 columns]"
      ]
     },
     "execution_count": 147,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "aug_2019.drop(aug_2019[aug_2019['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "aug_2019.drop(aug_2019[aug_2019['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "aug_2019.drop(aug_2019[aug_2019['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "aug_2019.drop(aug_2019[aug_2019['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "aug_2019"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 148,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "aug_2019[\"Year\"] = \"2019\"\n",
    "aug_2019[\"Month\"] = \"aug\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 149,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "aug_2019 = aug_2019.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 150,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "aug_2019_ave = aug_2019.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# aug_2019_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 151,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "aug_2019_ave.to_csv(\"data/day/MT/aug/aug_2019_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2020 August Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 152,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "aug_2020 = pd.read_csv(\"../data_large/MT-day/2020-aug-day-MT.csv\")\n",
    "\n",
    "# aug_2020"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 153,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>5e65748abfa7413de4b81ab0d609cf2b3d225520</td>\n",
       "      <td>2020-08-10 15:45:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>727</td>\n",
       "      <td>750</td>\n",
       "      <td>717</td>\n",
       "      <td>MT</td>\n",
       "      <td>Missoula</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>5e65748abfa7413de4b81ab0d609cf2b3d225520</td>\n",
       "      <td>2020-08-24 18:05:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>751</td>\n",
       "      <td>750</td>\n",
       "      <td>717</td>\n",
       "      <td>MT</td>\n",
       "      <td>Missoula</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>f74a23b378011473f8b32b3fc2efbbb3b6a9e098</td>\n",
       "      <td>2020-08-17 18:05:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>709</td>\n",
       "      <td>687</td>\n",
       "      <td>687</td>\n",
       "      <td>MT</td>\n",
       "      <td>Billings</td>\n",
       "      <td>55</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>5e65748abfa7413de4b81ab0d609cf2b3d225520</td>\n",
       "      <td>2020-08-14 14:05:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>712</td>\n",
       "      <td>750</td>\n",
       "      <td>717</td>\n",
       "      <td>MT</td>\n",
       "      <td>Missoula</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5e65748abfa7413de4b81ab0d609cf2b3d225520</td>\n",
       "      <td>2020-08-07 17:40:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>746</td>\n",
       "      <td>750</td>\n",
       "      <td>717</td>\n",
       "      <td>MT</td>\n",
       "      <td>Missoula</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>47827</th>\n",
       "      <td>29f5966d7808882d6c76a0a2d7091c18b3587704</td>\n",
       "      <td>2020-08-31 16:00:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>755</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>MT</td>\n",
       "      <td>Bozeman</td>\n",
       "      <td>9</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>47828</th>\n",
       "      <td>29f5966d7808882d6c76a0a2d7091c18b3587704</td>\n",
       "      <td>2020-08-31 15:35:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>756</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>MT</td>\n",
       "      <td>Bozeman</td>\n",
       "      <td>9</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>47829</th>\n",
       "      <td>29f5966d7808882d6c76a0a2d7091c18b3587704</td>\n",
       "      <td>2020-08-31 15:40:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>759</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>MT</td>\n",
       "      <td>Bozeman</td>\n",
       "      <td>9</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>47830</th>\n",
       "      <td>29f5966d7808882d6c76a0a2d7091c18b3587704</td>\n",
       "      <td>2020-08-31 15:45:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>759</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>MT</td>\n",
       "      <td>Bozeman</td>\n",
       "      <td>9</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>47831</th>\n",
       "      <td>29f5966d7808882d6c76a0a2d7091c18b3587704</td>\n",
       "      <td>2020-08-31 14:30:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>747</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>MT</td>\n",
       "      <td>Bozeman</td>\n",
       "      <td>9</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>33234 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                     Identifier                date_time  \\\n",
       "0      5e65748abfa7413de4b81ab0d609cf2b3d225520  2020-08-10 15:45:00 UTC   \n",
       "1      5e65748abfa7413de4b81ab0d609cf2b3d225520  2020-08-24 18:05:00 UTC   \n",
       "2      f74a23b378011473f8b32b3fc2efbbb3b6a9e098  2020-08-17 18:05:00 UTC   \n",
       "3      5e65748abfa7413de4b81ab0d609cf2b3d225520  2020-08-14 14:05:00 UTC   \n",
       "4      5e65748abfa7413de4b81ab0d609cf2b3d225520  2020-08-07 17:40:00 UTC   \n",
       "...                                         ...                      ...   \n",
       "47827  29f5966d7808882d6c76a0a2d7091c18b3587704  2020-08-31 16:00:00 UTC   \n",
       "47828  29f5966d7808882d6c76a0a2d7091c18b3587704  2020-08-31 15:35:00 UTC   \n",
       "47829  29f5966d7808882d6c76a0a2d7091c18b3587704  2020-08-31 15:40:00 UTC   \n",
       "47830  29f5966d7808882d6c76a0a2d7091c18b3587704  2020-08-31 15:45:00 UTC   \n",
       "47831  29f5966d7808882d6c76a0a2d7091c18b3587704  2020-08-31 14:30:00 UTC   \n",
       "\n",
       "      HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0         cool          auto               727                      750   \n",
       "1         cool          auto               751                      750   \n",
       "2         cool          hold               709                      687   \n",
       "3         cool          auto               712                      750   \n",
       "4         cool          auto               746                      750   \n",
       "...        ...           ...               ...                      ...   \n",
       "47827     heat          hold               755                      760   \n",
       "47828     heat          hold               756                      760   \n",
       "47829     heat          hold               759                      760   \n",
       "47830     heat          hold               759                      760   \n",
       "47831     heat          hold               747                      760   \n",
       "\n",
       "       TemperatureExpectedHeat ProvinceState      City  Age_of_Home__years_  \\\n",
       "0                          717            MT  Missoula                    5   \n",
       "1                          717            MT  Missoula                    5   \n",
       "2                          687            MT  Billings                   55   \n",
       "3                          717            MT  Missoula                    5   \n",
       "4                          717            MT  Missoula                    5   \n",
       "...                        ...           ...       ...                  ...   \n",
       "47827                      760            MT   Bozeman                    9   \n",
       "47828                      760            MT   Bozeman                    9   \n",
       "47829                      760            MT   Bozeman                    9   \n",
       "47830                      760            MT   Bozeman                    9   \n",
       "47831                      760            MT   Bozeman                    9   \n",
       "\n",
       "       allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                 False         False            False   \n",
       "1                 False         False            False   \n",
       "2                 False         False            False   \n",
       "3                 False         False            False   \n",
       "4                 False         False            False   \n",
       "...                 ...           ...              ...   \n",
       "47827              True         False            False   \n",
       "47828              True         False            False   \n",
       "47829              True         False            False   \n",
       "47830              True         False            False   \n",
       "47831              True         False            False   \n",
       "\n",
       "      Auxilliary_Heat_Fuel_Type  \n",
       "0                           Gas  \n",
       "1                           Gas  \n",
       "2                           Gas  \n",
       "3                           Gas  \n",
       "4                           Gas  \n",
       "...                         ...  \n",
       "47827                       Gas  \n",
       "47828                       Gas  \n",
       "47829                       Gas  \n",
       "47830                       Gas  \n",
       "47831                       Gas  \n",
       "\n",
       "[33234 rows x 14 columns]"
      ]
     },
     "execution_count": 153,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "aug_2020.drop(aug_2020[aug_2020['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "aug_2020.drop(aug_2020[aug_2020['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "aug_2020.drop(aug_2020[aug_2020['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "aug_2020.drop(aug_2020[aug_2020['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "aug_2020"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 154,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "aug_2020[\"Year\"] = \"2020\"\n",
    "aug_2020[\"Month\"] = \"aug\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 155,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "aug_2020 = aug_2020.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 156,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "aug_2020_ave = aug_2020.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# aug_2020_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 157,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "aug_2020_ave.to_csv(\"data/day/MT/aug/aug_2020_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "scrolled": true
   },
   "source": [
    "---\n",
    "\n",
    "### Combine month CSV Files \n",
    "1. Read in files in folders for each state\n",
    "2. Export as combined CSV"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 158,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create variable for files in directory\n",
    "files = [f for f in os.listdir(\"data/day/MT/aug/\") if f.endswith(\".csv\")]\n",
    "\n",
    "# files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 159,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>Month</th>\n",
       "      <th>Year</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>City</th>\n",
       "      <th>Temperature_ctrl_ave</th>\n",
       "      <th>TemperatureExpectedCool_ave</th>\n",
       "      <th>TemperatureExpectedHeat_ave</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>037988cc24eaeaa08f7002dbc926d02c8f9c0f1c</td>\n",
       "      <td>aug</td>\n",
       "      <td>2017</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>Eureka</td>\n",
       "      <td>698.184211</td>\n",
       "      <td>724.421053</td>\n",
       "      <td>734.736842</td>\n",
       "      <td>10.0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>037988cc24eaeaa08f7002dbc926d02c8f9c0f1c</td>\n",
       "      <td>aug</td>\n",
       "      <td>2017</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>Eureka</td>\n",
       "      <td>692.443119</td>\n",
       "      <td>719.359608</td>\n",
       "      <td>719.305922</td>\n",
       "      <td>10.0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0bc51b95b2dc23713f611e182c689ad03dc2c300</td>\n",
       "      <td>aug</td>\n",
       "      <td>2017</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>Great Falls</td>\n",
       "      <td>705.645833</td>\n",
       "      <td>750.000000</td>\n",
       "      <td>680.000000</td>\n",
       "      <td>10.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0bc51b95b2dc23713f611e182c689ad03dc2c300</td>\n",
       "      <td>aug</td>\n",
       "      <td>2017</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>Great Falls</td>\n",
       "      <td>709.226804</td>\n",
       "      <td>724.481100</td>\n",
       "      <td>720.621993</td>\n",
       "      <td>10.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1e4188c276ca852a75feef8c5e3fc27e1f409c4d</td>\n",
       "      <td>aug</td>\n",
       "      <td>2017</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>Helena</td>\n",
       "      <td>741.703704</td>\n",
       "      <td>740.000000</td>\n",
       "      <td>740.000000</td>\n",
       "      <td>5.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>68</th>\n",
       "      <td>f469a99acb054dbddd08e713d682acf5035c42df</td>\n",
       "      <td>aug</td>\n",
       "      <td>2020</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>Helena</td>\n",
       "      <td>763.464509</td>\n",
       "      <td>650.000000</td>\n",
       "      <td>650.000000</td>\n",
       "      <td>49.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>69</th>\n",
       "      <td>f5101dfe9a0543a0dd3d8b1adbec2dd33f3047a8</td>\n",
       "      <td>aug</td>\n",
       "      <td>2020</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>Bozeman</td>\n",
       "      <td>661.529412</td>\n",
       "      <td>653.235294</td>\n",
       "      <td>641.911765</td>\n",
       "      <td>90.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>70</th>\n",
       "      <td>f74a23b378011473f8b32b3fc2efbbb3b6a9e098</td>\n",
       "      <td>aug</td>\n",
       "      <td>2020</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>Billings</td>\n",
       "      <td>718.625000</td>\n",
       "      <td>680.000000</td>\n",
       "      <td>680.000000</td>\n",
       "      <td>55.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>71</th>\n",
       "      <td>f74a23b378011473f8b32b3fc2efbbb3b6a9e098</td>\n",
       "      <td>aug</td>\n",
       "      <td>2020</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>Billings</td>\n",
       "      <td>697.089005</td>\n",
       "      <td>683.403141</td>\n",
       "      <td>682.429319</td>\n",
       "      <td>55.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>72</th>\n",
       "      <td>fd93662943beeb4603fd63aacbbbb7abb3183470</td>\n",
       "      <td>aug</td>\n",
       "      <td>2020</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>NASHUA</td>\n",
       "      <td>710.625000</td>\n",
       "      <td>710.270833</td>\n",
       "      <td>690.000000</td>\n",
       "      <td>30.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>232 rows × 13 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                  Identifier Month  Year HvacMode  \\\n",
       "0   037988cc24eaeaa08f7002dbc926d02c8f9c0f1c   aug  2017     cool   \n",
       "1   037988cc24eaeaa08f7002dbc926d02c8f9c0f1c   aug  2017     cool   \n",
       "2   0bc51b95b2dc23713f611e182c689ad03dc2c300   aug  2017     cool   \n",
       "3   0bc51b95b2dc23713f611e182c689ad03dc2c300   aug  2017     cool   \n",
       "4   1e4188c276ca852a75feef8c5e3fc27e1f409c4d   aug  2017     cool   \n",
       "..                                       ...   ...   ...      ...   \n",
       "68  f469a99acb054dbddd08e713d682acf5035c42df   aug  2020     heat   \n",
       "69  f5101dfe9a0543a0dd3d8b1adbec2dd33f3047a8   aug  2020     heat   \n",
       "70  f74a23b378011473f8b32b3fc2efbbb3b6a9e098   aug  2020     cool   \n",
       "71  f74a23b378011473f8b32b3fc2efbbb3b6a9e098   aug  2020     cool   \n",
       "72  fd93662943beeb4603fd63aacbbbb7abb3183470   aug  2020     cool   \n",
       "\n",
       "   CalendarEvent         City  Temperature_ctrl_ave  \\\n",
       "0           auto       Eureka            698.184211   \n",
       "1           hold       Eureka            692.443119   \n",
       "2           auto  Great Falls            705.645833   \n",
       "3           hold  Great Falls            709.226804   \n",
       "4           hold       Helena            741.703704   \n",
       "..           ...          ...                   ...   \n",
       "68          auto       Helena            763.464509   \n",
       "69          hold      Bozeman            661.529412   \n",
       "70          auto     Billings            718.625000   \n",
       "71          hold     Billings            697.089005   \n",
       "72          auto       NASHUA            710.625000   \n",
       "\n",
       "    TemperatureExpectedCool_ave  TemperatureExpectedHeat_ave  \\\n",
       "0                    724.421053                   734.736842   \n",
       "1                    719.359608                   719.305922   \n",
       "2                    750.000000                   680.000000   \n",
       "3                    724.481100                   720.621993   \n",
       "4                    740.000000                   740.000000   \n",
       "..                          ...                          ...   \n",
       "68                   650.000000                   650.000000   \n",
       "69                   653.235294                   641.911765   \n",
       "70                   680.000000                   680.000000   \n",
       "71                   683.403141                   682.429319   \n",
       "72                   710.270833                   690.000000   \n",
       "\n",
       "    Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \n",
       "0                  10.0              True         False             True  \n",
       "1                  10.0              True         False             True  \n",
       "2                  10.0             False         False            False  \n",
       "3                  10.0             False         False            False  \n",
       "4                   5.0             False         False            False  \n",
       "..                  ...               ...           ...              ...  \n",
       "68                 49.0             False         False            False  \n",
       "69                 90.0             False         False            False  \n",
       "70                 55.0             False         False            False  \n",
       "71                 55.0             False         False            False  \n",
       "72                 30.0             False         False            False  \n",
       "\n",
       "[232 rows x 13 columns]"
      ]
     },
     "execution_count": 159,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# https://stacMToverflow.com/questions/63886787/how-to-create-a-dataframe-from-multiple-csv-files\n",
    "# Initialize an empty dataframe\n",
    "MT_aug = pd.DataFrame()\n",
    "\n",
    "# Iterate through files and contents, then concatenate the data into the dataframe\n",
    "for file in files:\n",
    "    df = pd.read_csv(\"data/day/MT/aug/\" + file)\n",
    "    MT_aug = pd.concat([MT_aug, df])\n",
    "    \n",
    "MT_aug"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 160,
   "metadata": {},
   "outputs": [],
   "source": [
    "MT_aug.to_csv(\"Scraper_Output/State_Month_Day/MT/MT_aug.csv\", header=True, index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "\n",
    "## September"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "\n",
    "## October"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "\n",
    "## November"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "\n",
    "## December"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2017 December Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 161,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "dec_2017 = pd.read_csv(\"../data_large/MT-day/2017-dec-day-MT.csv\")\n",
    "\n",
    "# dec_2017"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 162,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>7352a85776176198f0abb065b891160881b30270</td>\n",
       "      <td>2017-12-17T19:55:00Z</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>653</td>\n",
       "      <td>745</td>\n",
       "      <td>655</td>\n",
       "      <td>MT</td>\n",
       "      <td>Bozeman</td>\n",
       "      <td>15</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>7352a85776176198f0abb065b891160881b30270</td>\n",
       "      <td>2017-12-17T17:40:00Z</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>657</td>\n",
       "      <td>745</td>\n",
       "      <td>655</td>\n",
       "      <td>MT</td>\n",
       "      <td>Bozeman</td>\n",
       "      <td>15</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>7352a85776176198f0abb065b891160881b30270</td>\n",
       "      <td>2017-12-17T16:25:00Z</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>652</td>\n",
       "      <td>745</td>\n",
       "      <td>655</td>\n",
       "      <td>MT</td>\n",
       "      <td>Bozeman</td>\n",
       "      <td>15</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>7352a85776176198f0abb065b891160881b30270</td>\n",
       "      <td>2017-12-17T17:25:00Z</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>655</td>\n",
       "      <td>745</td>\n",
       "      <td>655</td>\n",
       "      <td>MT</td>\n",
       "      <td>Bozeman</td>\n",
       "      <td>15</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>f74a23b378011473f8b32b3fc2efbbb3b6a9e098</td>\n",
       "      <td>2017-12-28T17:35:00Z</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>658</td>\n",
       "      <td>650</td>\n",
       "      <td>645</td>\n",
       "      <td>MT</td>\n",
       "      <td>Billings</td>\n",
       "      <td>55</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18741</th>\n",
       "      <td>0bc51b95b2dc23713f611e182c689ad03dc2c300</td>\n",
       "      <td>2017-12-30T12:05:00Z</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>696</td>\n",
       "      <td>680</td>\n",
       "      <td>680</td>\n",
       "      <td>MT</td>\n",
       "      <td>Great Falls</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18742</th>\n",
       "      <td>0bc51b95b2dc23713f611e182c689ad03dc2c300</td>\n",
       "      <td>2017-12-30T13:10:00Z</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>674</td>\n",
       "      <td>680</td>\n",
       "      <td>680</td>\n",
       "      <td>MT</td>\n",
       "      <td>Great Falls</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18743</th>\n",
       "      <td>0bc51b95b2dc23713f611e182c689ad03dc2c300</td>\n",
       "      <td>2017-12-30T13:55:00Z</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>628</td>\n",
       "      <td>680</td>\n",
       "      <td>680</td>\n",
       "      <td>MT</td>\n",
       "      <td>Great Falls</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18744</th>\n",
       "      <td>0bc51b95b2dc23713f611e182c689ad03dc2c300</td>\n",
       "      <td>2017-12-30T14:10:00Z</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>634</td>\n",
       "      <td>680</td>\n",
       "      <td>680</td>\n",
       "      <td>MT</td>\n",
       "      <td>Great Falls</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18745</th>\n",
       "      <td>0bc51b95b2dc23713f611e182c689ad03dc2c300</td>\n",
       "      <td>2017-12-30T14:05:00Z</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>631</td>\n",
       "      <td>680</td>\n",
       "      <td>680</td>\n",
       "      <td>MT</td>\n",
       "      <td>Great Falls</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>14813 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                     Identifier             date_time  \\\n",
       "1      7352a85776176198f0abb065b891160881b30270  2017-12-17T19:55:00Z   \n",
       "2      7352a85776176198f0abb065b891160881b30270  2017-12-17T17:40:00Z   \n",
       "4      7352a85776176198f0abb065b891160881b30270  2017-12-17T16:25:00Z   \n",
       "5      7352a85776176198f0abb065b891160881b30270  2017-12-17T17:25:00Z   \n",
       "6      f74a23b378011473f8b32b3fc2efbbb3b6a9e098  2017-12-28T17:35:00Z   \n",
       "...                                         ...                   ...   \n",
       "18741  0bc51b95b2dc23713f611e182c689ad03dc2c300  2017-12-30T12:05:00Z   \n",
       "18742  0bc51b95b2dc23713f611e182c689ad03dc2c300  2017-12-30T13:10:00Z   \n",
       "18743  0bc51b95b2dc23713f611e182c689ad03dc2c300  2017-12-30T13:55:00Z   \n",
       "18744  0bc51b95b2dc23713f611e182c689ad03dc2c300  2017-12-30T14:10:00Z   \n",
       "18745  0bc51b95b2dc23713f611e182c689ad03dc2c300  2017-12-30T14:05:00Z   \n",
       "\n",
       "      HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "1         auto          hold               653                      745   \n",
       "2         auto          hold               657                      745   \n",
       "4         auto          hold               652                      745   \n",
       "5         auto          hold               655                      745   \n",
       "6         heat          hold               658                      650   \n",
       "...        ...           ...               ...                      ...   \n",
       "18741     heat          hold               696                      680   \n",
       "18742     heat          hold               674                      680   \n",
       "18743     heat          hold               628                      680   \n",
       "18744     heat          hold               634                      680   \n",
       "18745     heat          hold               631                      680   \n",
       "\n",
       "       TemperatureExpectedHeat ProvinceState         City  \\\n",
       "1                          655            MT      Bozeman   \n",
       "2                          655            MT      Bozeman   \n",
       "4                          655            MT      Bozeman   \n",
       "5                          655            MT      Bozeman   \n",
       "6                          645            MT     Billings   \n",
       "...                        ...           ...          ...   \n",
       "18741                      680            MT  Great Falls   \n",
       "18742                      680            MT  Great Falls   \n",
       "18743                      680            MT  Great Falls   \n",
       "18744                      680            MT  Great Falls   \n",
       "18745                      680            MT  Great Falls   \n",
       "\n",
       "       Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "1                       15             False         False            False   \n",
       "2                       15             False         False            False   \n",
       "4                       15             False         False            False   \n",
       "5                       15             False         False            False   \n",
       "6                       55             False         False            False   \n",
       "...                    ...               ...           ...              ...   \n",
       "18741                   10             False         False            False   \n",
       "18742                   10             False         False            False   \n",
       "18743                   10             False         False            False   \n",
       "18744                   10             False         False            False   \n",
       "18745                   10             False         False            False   \n",
       "\n",
       "      Auxilliary_Heat_Fuel_Type  \n",
       "1                           Gas  \n",
       "2                           Gas  \n",
       "4                           Gas  \n",
       "5                           Gas  \n",
       "6                           Gas  \n",
       "...                         ...  \n",
       "18741                       Gas  \n",
       "18742                       Gas  \n",
       "18743                       Gas  \n",
       "18744                       Gas  \n",
       "18745                       Gas  \n",
       "\n",
       "[14813 rows x 14 columns]"
      ]
     },
     "execution_count": 162,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "dec_2017.drop(dec_2017[dec_2017['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "dec_2017.drop(dec_2017[dec_2017['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "dec_2017.drop(dec_2017[dec_2017['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "dec_2017.drop(dec_2017[dec_2017['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "dec_2017"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 163,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "dec_2017[\"Year\"] = \"2017\"\n",
    "dec_2017[\"Month\"] = \"dec\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 164,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "dec_2017 = dec_2017.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 165,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "dec_2017_ave = dec_2017.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# dec_2017_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 166,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "dec_2017_ave.to_csv(\"data/day/MT/dec/dec_2017_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2018 December Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 167,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "dec_2018 = pd.read_csv(\"../data_large/MT-day/2018-dec-day-MT.csv\")\n",
    "\n",
    "# dec_2018"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 168,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>34fbbdfdbc62b907f2cb4f26559df29b230273e3</td>\n",
       "      <td>2018-12-22 14:10:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>617</td>\n",
       "      <td>650</td>\n",
       "      <td>617</td>\n",
       "      <td>MT</td>\n",
       "      <td>Missoula</td>\n",
       "      <td>25</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>34fbbdfdbc62b907f2cb4f26559df29b230273e3</td>\n",
       "      <td>2018-12-22 19:45:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>618</td>\n",
       "      <td>650</td>\n",
       "      <td>617</td>\n",
       "      <td>MT</td>\n",
       "      <td>Missoula</td>\n",
       "      <td>25</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>06941505466c4069dc9b2cd6aaf6b800a4cd5d40</td>\n",
       "      <td>2018-12-09 13:55:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>677</td>\n",
       "      <td>730</td>\n",
       "      <td>662</td>\n",
       "      <td>MT</td>\n",
       "      <td>Fairfield</td>\n",
       "      <td>15</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>34fbbdfdbc62b907f2cb4f26559df29b230273e3</td>\n",
       "      <td>2018-12-22 13:25:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>618</td>\n",
       "      <td>650</td>\n",
       "      <td>617</td>\n",
       "      <td>MT</td>\n",
       "      <td>Missoula</td>\n",
       "      <td>25</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>34fbbdfdbc62b907f2cb4f26559df29b230273e3</td>\n",
       "      <td>2018-12-22 17:40:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>616</td>\n",
       "      <td>650</td>\n",
       "      <td>617</td>\n",
       "      <td>MT</td>\n",
       "      <td>Missoula</td>\n",
       "      <td>25</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33942</th>\n",
       "      <td>a5843cc963fec301306a65ed47b1063f1370c9c7</td>\n",
       "      <td>2018-12-09 19:15:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>701</td>\n",
       "      <td>700</td>\n",
       "      <td>700</td>\n",
       "      <td>MT</td>\n",
       "      <td>Billings</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33943</th>\n",
       "      <td>a5843cc963fec301306a65ed47b1063f1370c9c7</td>\n",
       "      <td>2018-12-06 19:55:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>695</td>\n",
       "      <td>700</td>\n",
       "      <td>700</td>\n",
       "      <td>MT</td>\n",
       "      <td>Billings</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33944</th>\n",
       "      <td>a5843cc963fec301306a65ed47b1063f1370c9c7</td>\n",
       "      <td>2018-12-13 13:45:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>703</td>\n",
       "      <td>700</td>\n",
       "      <td>700</td>\n",
       "      <td>MT</td>\n",
       "      <td>Billings</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33945</th>\n",
       "      <td>bbc0d748de67c33d4c70a501da752924a90777ee</td>\n",
       "      <td>2018-12-04 12:50:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>697</td>\n",
       "      <td>750</td>\n",
       "      <td>700</td>\n",
       "      <td>MT</td>\n",
       "      <td>Kalispell</td>\n",
       "      <td>45</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33946</th>\n",
       "      <td>bbc0d748de67c33d4c70a501da752924a90777ee</td>\n",
       "      <td>2018-12-04 13:10:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>698</td>\n",
       "      <td>750</td>\n",
       "      <td>700</td>\n",
       "      <td>MT</td>\n",
       "      <td>Kalispell</td>\n",
       "      <td>45</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>27936 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                     Identifier                date_time  \\\n",
       "0      34fbbdfdbc62b907f2cb4f26559df29b230273e3  2018-12-22 14:10:00 UTC   \n",
       "1      34fbbdfdbc62b907f2cb4f26559df29b230273e3  2018-12-22 19:45:00 UTC   \n",
       "2      06941505466c4069dc9b2cd6aaf6b800a4cd5d40  2018-12-09 13:55:00 UTC   \n",
       "3      34fbbdfdbc62b907f2cb4f26559df29b230273e3  2018-12-22 13:25:00 UTC   \n",
       "4      34fbbdfdbc62b907f2cb4f26559df29b230273e3  2018-12-22 17:40:00 UTC   \n",
       "...                                         ...                      ...   \n",
       "33942  a5843cc963fec301306a65ed47b1063f1370c9c7  2018-12-09 19:15:00 UTC   \n",
       "33943  a5843cc963fec301306a65ed47b1063f1370c9c7  2018-12-06 19:55:00 UTC   \n",
       "33944  a5843cc963fec301306a65ed47b1063f1370c9c7  2018-12-13 13:45:00 UTC   \n",
       "33945  bbc0d748de67c33d4c70a501da752924a90777ee  2018-12-04 12:50:00 UTC   \n",
       "33946  bbc0d748de67c33d4c70a501da752924a90777ee  2018-12-04 13:10:00 UTC   \n",
       "\n",
       "      HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0         heat          hold               617                      650   \n",
       "1         heat          hold               618                      650   \n",
       "2         heat          auto               677                      730   \n",
       "3         heat          hold               618                      650   \n",
       "4         heat          hold               616                      650   \n",
       "...        ...           ...               ...                      ...   \n",
       "33942     heat          auto               701                      700   \n",
       "33943     heat          hold               695                      700   \n",
       "33944     heat          auto               703                      700   \n",
       "33945     heat          auto               697                      750   \n",
       "33946     heat          auto               698                      750   \n",
       "\n",
       "       TemperatureExpectedHeat ProvinceState       City  Age_of_Home__years_  \\\n",
       "0                          617            MT   Missoula                   25   \n",
       "1                          617            MT   Missoula                   25   \n",
       "2                          662            MT  Fairfield                   15   \n",
       "3                          617            MT   Missoula                   25   \n",
       "4                          617            MT   Missoula                   25   \n",
       "...                        ...           ...        ...                  ...   \n",
       "33942                      700            MT   Billings                    0   \n",
       "33943                      700            MT   Billings                    0   \n",
       "33944                      700            MT   Billings                    0   \n",
       "33945                      700            MT  Kalispell                   45   \n",
       "33946                      700            MT  Kalispell                   45   \n",
       "\n",
       "       allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                 False         False            False   \n",
       "1                 False         False            False   \n",
       "2                 False         False            False   \n",
       "3                 False         False            False   \n",
       "4                 False         False            False   \n",
       "...                 ...           ...              ...   \n",
       "33942             False         False            False   \n",
       "33943             False         False            False   \n",
       "33944             False         False            False   \n",
       "33945              True         False            False   \n",
       "33946              True         False            False   \n",
       "\n",
       "      Auxilliary_Heat_Fuel_Type  \n",
       "0                           Gas  \n",
       "1                           Gas  \n",
       "2                           Gas  \n",
       "3                           Gas  \n",
       "4                           Gas  \n",
       "...                         ...  \n",
       "33942                       Gas  \n",
       "33943                       Gas  \n",
       "33944                       Gas  \n",
       "33945                       Gas  \n",
       "33946                       Gas  \n",
       "\n",
       "[27936 rows x 14 columns]"
      ]
     },
     "execution_count": 168,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "dec_2018.drop(dec_2018[dec_2018['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "dec_2018.drop(dec_2018[dec_2018['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "dec_2018.drop(dec_2018[dec_2018['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "dec_2018.drop(dec_2018[dec_2018['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "dec_2018"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 169,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "dec_2018[\"Year\"] = \"2018\"\n",
    "dec_2018[\"Month\"] = \"dec\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 170,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "dec_2018 = dec_2018.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 171,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "dec_2018_ave = dec_2018.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# dec_2018_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 172,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "dec_2018_ave.to_csv(\"data/day/MT/dec/dec_2018_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2019 December Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 173,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "dec_2019 = pd.read_csv(\"../data_large/MT-day/2019-dec-day-MT.csv\")\n",
    "\n",
    "# dec_2019"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 174,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>7e82f7bb0ad060bdb1e139bed8aa5d5f869cf054</td>\n",
       "      <td>2019-12-01 17:40:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>677</td>\n",
       "      <td>689</td>\n",
       "      <td>689</td>\n",
       "      <td>MT</td>\n",
       "      <td>East Helena</td>\n",
       "      <td>40</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2f91f344abd5823f11abdfa57e08bb514aa09564</td>\n",
       "      <td>2019-12-19 14:55:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>715</td>\n",
       "      <td>723</td>\n",
       "      <td>718</td>\n",
       "      <td>MT</td>\n",
       "      <td>Billings</td>\n",
       "      <td>50</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>666c08e73f2bee116322fafbf2fc1b2972422b81</td>\n",
       "      <td>2019-12-31 13:55:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>710</td>\n",
       "      <td>775</td>\n",
       "      <td>714</td>\n",
       "      <td>MT</td>\n",
       "      <td>Bozeman</td>\n",
       "      <td>7</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>a6b5547e34405b7e11d145b6b81f37e2850b7cd9</td>\n",
       "      <td>2019-12-14 17:05:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>695</td>\n",
       "      <td>713</td>\n",
       "      <td>687</td>\n",
       "      <td>MT</td>\n",
       "      <td>Missoula</td>\n",
       "      <td>70</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>7e82f7bb0ad060bdb1e139bed8aa5d5f869cf054</td>\n",
       "      <td>2019-12-03 13:55:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>679</td>\n",
       "      <td>730</td>\n",
       "      <td>678</td>\n",
       "      <td>MT</td>\n",
       "      <td>East Helena</td>\n",
       "      <td>40</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>43164</th>\n",
       "      <td>98427a2f54b32505d3f1a3e9dfe4acfb78346762</td>\n",
       "      <td>2019-12-31 18:00:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>765</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>MT</td>\n",
       "      <td>Great Falls</td>\n",
       "      <td>70</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>43165</th>\n",
       "      <td>98427a2f54b32505d3f1a3e9dfe4acfb78346762</td>\n",
       "      <td>2019-12-08 16:10:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>746</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>MT</td>\n",
       "      <td>Great Falls</td>\n",
       "      <td>70</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>43166</th>\n",
       "      <td>98427a2f54b32505d3f1a3e9dfe4acfb78346762</td>\n",
       "      <td>2019-12-08 16:35:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>754</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>MT</td>\n",
       "      <td>Great Falls</td>\n",
       "      <td>70</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>43167</th>\n",
       "      <td>98427a2f54b32505d3f1a3e9dfe4acfb78346762</td>\n",
       "      <td>2019-12-05 17:40:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>752</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>MT</td>\n",
       "      <td>Great Falls</td>\n",
       "      <td>70</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>43168</th>\n",
       "      <td>98427a2f54b32505d3f1a3e9dfe4acfb78346762</td>\n",
       "      <td>2019-12-21 19:45:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>757</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>MT</td>\n",
       "      <td>Great Falls</td>\n",
       "      <td>70</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>35875 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                     Identifier                date_time  \\\n",
       "0      7e82f7bb0ad060bdb1e139bed8aa5d5f869cf054  2019-12-01 17:40:00 UTC   \n",
       "1      2f91f344abd5823f11abdfa57e08bb514aa09564  2019-12-19 14:55:00 UTC   \n",
       "2      666c08e73f2bee116322fafbf2fc1b2972422b81  2019-12-31 13:55:00 UTC   \n",
       "3      a6b5547e34405b7e11d145b6b81f37e2850b7cd9  2019-12-14 17:05:00 UTC   \n",
       "5      7e82f7bb0ad060bdb1e139bed8aa5d5f869cf054  2019-12-03 13:55:00 UTC   \n",
       "...                                         ...                      ...   \n",
       "43164  98427a2f54b32505d3f1a3e9dfe4acfb78346762  2019-12-31 18:00:00 UTC   \n",
       "43165  98427a2f54b32505d3f1a3e9dfe4acfb78346762  2019-12-08 16:10:00 UTC   \n",
       "43166  98427a2f54b32505d3f1a3e9dfe4acfb78346762  2019-12-08 16:35:00 UTC   \n",
       "43167  98427a2f54b32505d3f1a3e9dfe4acfb78346762  2019-12-05 17:40:00 UTC   \n",
       "43168  98427a2f54b32505d3f1a3e9dfe4acfb78346762  2019-12-21 19:45:00 UTC   \n",
       "\n",
       "      HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0         heat          auto               677                      689   \n",
       "1         heat          auto               715                      723   \n",
       "2         auto          hold               710                      775   \n",
       "3         heat          auto               695                      713   \n",
       "5         heat          hold               679                      730   \n",
       "...        ...           ...               ...                      ...   \n",
       "43164     heat          hold               765                      760   \n",
       "43165     heat          hold               746                      760   \n",
       "43166     heat          hold               754                      760   \n",
       "43167     heat          hold               752                      760   \n",
       "43168     heat          hold               757                      760   \n",
       "\n",
       "       TemperatureExpectedHeat ProvinceState         City  \\\n",
       "0                          689            MT  East Helena   \n",
       "1                          718            MT     Billings   \n",
       "2                          714            MT      Bozeman   \n",
       "3                          687            MT     Missoula   \n",
       "5                          678            MT  East Helena   \n",
       "...                        ...           ...          ...   \n",
       "43164                      760            MT  Great Falls   \n",
       "43165                      760            MT  Great Falls   \n",
       "43166                      760            MT  Great Falls   \n",
       "43167                      760            MT  Great Falls   \n",
       "43168                      760            MT  Great Falls   \n",
       "\n",
       "       Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                       40             False         False            False   \n",
       "1                       50              True         False            False   \n",
       "2                        7             False         False            False   \n",
       "3                       70             False         False            False   \n",
       "5                       40             False         False            False   \n",
       "...                    ...               ...           ...              ...   \n",
       "43164                   70             False         False            False   \n",
       "43165                   70             False         False            False   \n",
       "43166                   70             False         False            False   \n",
       "43167                   70             False         False            False   \n",
       "43168                   70             False         False            False   \n",
       "\n",
       "      Auxilliary_Heat_Fuel_Type  \n",
       "0                           Gas  \n",
       "1                           Gas  \n",
       "2                           Gas  \n",
       "3                           Gas  \n",
       "5                           Gas  \n",
       "...                         ...  \n",
       "43164                       Gas  \n",
       "43165                       Gas  \n",
       "43166                       Gas  \n",
       "43167                       Gas  \n",
       "43168                       Gas  \n",
       "\n",
       "[35875 rows x 14 columns]"
      ]
     },
     "execution_count": 174,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "dec_2019.drop(dec_2019[dec_2019['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "dec_2019.drop(dec_2019[dec_2019['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "dec_2019.drop(dec_2019[dec_2019['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "dec_2019.drop(dec_2019[dec_2019['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "dec_2019"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 175,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "dec_2019[\"Year\"] = \"2019\"\n",
    "dec_2019[\"Month\"] = \"dec\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 176,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "dec_2019 = dec_2019.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 177,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "dec_2019_ave = dec_2019.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# dec_2019_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 178,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "dec_2019_ave.to_csv(\"data/day/MT/dec/dec_2019_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2020 December Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 179,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "dec_2020 = pd.read_csv(\"../data_large/MT-day/2020-dec-day-MT.csv\")\n",
    "\n",
    "# dec_2020"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 180,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>7522a70780f2d3f66a683747547beccbafc53be2</td>\n",
       "      <td>2020-12-15 17:25:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>714</td>\n",
       "      <td>715</td>\n",
       "      <td>715</td>\n",
       "      <td>MT</td>\n",
       "      <td>Billings</td>\n",
       "      <td>30</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>77c344f9bb93b933ebc934a7ebba997d720726d5</td>\n",
       "      <td>2020-12-21 19:40:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>639</td>\n",
       "      <td>656</td>\n",
       "      <td>639</td>\n",
       "      <td>MT</td>\n",
       "      <td>Butte</td>\n",
       "      <td>90</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>f74a23b378011473f8b32b3fc2efbbb3b6a9e098</td>\n",
       "      <td>2020-12-21 19:30:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>704</td>\n",
       "      <td>709</td>\n",
       "      <td>709</td>\n",
       "      <td>MT</td>\n",
       "      <td>Billings</td>\n",
       "      <td>55</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>7522a70780f2d3f66a683747547beccbafc53be2</td>\n",
       "      <td>2020-12-15 19:55:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>712</td>\n",
       "      <td>715</td>\n",
       "      <td>715</td>\n",
       "      <td>MT</td>\n",
       "      <td>Billings</td>\n",
       "      <td>30</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>f74a23b378011473f8b32b3fc2efbbb3b6a9e098</td>\n",
       "      <td>2020-12-15 19:35:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>703</td>\n",
       "      <td>709</td>\n",
       "      <td>709</td>\n",
       "      <td>MT</td>\n",
       "      <td>Billings</td>\n",
       "      <td>55</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>37131</th>\n",
       "      <td>98427a2f54b32505d3f1a3e9dfe4acfb78346762</td>\n",
       "      <td>2020-12-25 16:55:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>745</td>\n",
       "      <td>750</td>\n",
       "      <td>750</td>\n",
       "      <td>MT</td>\n",
       "      <td>Great Falls</td>\n",
       "      <td>70</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>37132</th>\n",
       "      <td>98427a2f54b32505d3f1a3e9dfe4acfb78346762</td>\n",
       "      <td>2020-12-25 17:45:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>746</td>\n",
       "      <td>750</td>\n",
       "      <td>750</td>\n",
       "      <td>MT</td>\n",
       "      <td>Great Falls</td>\n",
       "      <td>70</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>37133</th>\n",
       "      <td>98427a2f54b32505d3f1a3e9dfe4acfb78346762</td>\n",
       "      <td>2020-12-30 16:00:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>750</td>\n",
       "      <td>750</td>\n",
       "      <td>750</td>\n",
       "      <td>MT</td>\n",
       "      <td>Great Falls</td>\n",
       "      <td>70</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>37134</th>\n",
       "      <td>98427a2f54b32505d3f1a3e9dfe4acfb78346762</td>\n",
       "      <td>2020-12-25 17:10:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>748</td>\n",
       "      <td>750</td>\n",
       "      <td>750</td>\n",
       "      <td>MT</td>\n",
       "      <td>Great Falls</td>\n",
       "      <td>70</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>37135</th>\n",
       "      <td>98427a2f54b32505d3f1a3e9dfe4acfb78346762</td>\n",
       "      <td>2020-12-25 19:05:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>744</td>\n",
       "      <td>750</td>\n",
       "      <td>750</td>\n",
       "      <td>MT</td>\n",
       "      <td>Great Falls</td>\n",
       "      <td>70</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>31015 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                     Identifier                date_time  \\\n",
       "0      7522a70780f2d3f66a683747547beccbafc53be2  2020-12-15 17:25:00 UTC   \n",
       "1      77c344f9bb93b933ebc934a7ebba997d720726d5  2020-12-21 19:40:00 UTC   \n",
       "2      f74a23b378011473f8b32b3fc2efbbb3b6a9e098  2020-12-21 19:30:00 UTC   \n",
       "3      7522a70780f2d3f66a683747547beccbafc53be2  2020-12-15 19:55:00 UTC   \n",
       "4      f74a23b378011473f8b32b3fc2efbbb3b6a9e098  2020-12-15 19:35:00 UTC   \n",
       "...                                         ...                      ...   \n",
       "37131  98427a2f54b32505d3f1a3e9dfe4acfb78346762  2020-12-25 16:55:00 UTC   \n",
       "37132  98427a2f54b32505d3f1a3e9dfe4acfb78346762  2020-12-25 17:45:00 UTC   \n",
       "37133  98427a2f54b32505d3f1a3e9dfe4acfb78346762  2020-12-30 16:00:00 UTC   \n",
       "37134  98427a2f54b32505d3f1a3e9dfe4acfb78346762  2020-12-25 17:10:00 UTC   \n",
       "37135  98427a2f54b32505d3f1a3e9dfe4acfb78346762  2020-12-25 19:05:00 UTC   \n",
       "\n",
       "      HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0         heat          hold               714                      715   \n",
       "1         heat          hold               639                      656   \n",
       "2         heat          hold               704                      709   \n",
       "3         heat          hold               712                      715   \n",
       "4         heat          hold               703                      709   \n",
       "...        ...           ...               ...                      ...   \n",
       "37131     heat          hold               745                      750   \n",
       "37132     heat          hold               746                      750   \n",
       "37133     heat          hold               750                      750   \n",
       "37134     heat          hold               748                      750   \n",
       "37135     heat          hold               744                      750   \n",
       "\n",
       "       TemperatureExpectedHeat ProvinceState         City  \\\n",
       "0                          715            MT     Billings   \n",
       "1                          639            MT        Butte   \n",
       "2                          709            MT     Billings   \n",
       "3                          715            MT     Billings   \n",
       "4                          709            MT     Billings   \n",
       "...                        ...           ...          ...   \n",
       "37131                      750            MT  Great Falls   \n",
       "37132                      750            MT  Great Falls   \n",
       "37133                      750            MT  Great Falls   \n",
       "37134                      750            MT  Great Falls   \n",
       "37135                      750            MT  Great Falls   \n",
       "\n",
       "       Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                       30              True         False            False   \n",
       "1                       90             False         False            False   \n",
       "2                       55             False         False            False   \n",
       "3                       30              True         False            False   \n",
       "4                       55             False         False            False   \n",
       "...                    ...               ...           ...              ...   \n",
       "37131                   70             False         False            False   \n",
       "37132                   70             False         False            False   \n",
       "37133                   70             False         False            False   \n",
       "37134                   70             False         False            False   \n",
       "37135                   70             False         False            False   \n",
       "\n",
       "      Auxilliary_Heat_Fuel_Type  \n",
       "0                           Gas  \n",
       "1                           Gas  \n",
       "2                           Gas  \n",
       "3                           Gas  \n",
       "4                           Gas  \n",
       "...                         ...  \n",
       "37131                       Gas  \n",
       "37132                       Gas  \n",
       "37133                       Gas  \n",
       "37134                       Gas  \n",
       "37135                       Gas  \n",
       "\n",
       "[31015 rows x 14 columns]"
      ]
     },
     "execution_count": 180,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "dec_2020.drop(dec_2020[dec_2020['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "dec_2020.drop(dec_2020[dec_2020['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "dec_2020.drop(dec_2020[dec_2020['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "dec_2020.drop(dec_2020[dec_2020['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "dec_2020"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 181,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "dec_2020[\"Year\"] = \"2020\"\n",
    "dec_2020[\"Month\"] = \"dec\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 182,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "dec_2020 = dec_2020.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 183,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "dec_2020_ave = dec_2020.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# dec_2020_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 184,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "dec_2020_ave.to_csv(\"data/day/MT/dec/dec_2020_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "\n",
    "### Combine month CSV Files \n",
    "1. Read in files in folders for each state\n",
    "2. Export as combined CSV"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 185,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create variable for files in directory\n",
    "files = [f for f in os.listdir(\"data/day/MT/dec/\") if f.endswith(\".csv\")]\n",
    "\n",
    "# files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 186,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>Month</th>\n",
       "      <th>Year</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>City</th>\n",
       "      <th>Temperature_ctrl_ave</th>\n",
       "      <th>TemperatureExpectedCool_ave</th>\n",
       "      <th>TemperatureExpectedHeat_ave</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>019764d7b48dd9d6f1f4ad95451d7d0da2669405</td>\n",
       "      <td>dec</td>\n",
       "      <td>2017</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>Billings</td>\n",
       "      <td>709.260870</td>\n",
       "      <td>740.000000</td>\n",
       "      <td>690.000000</td>\n",
       "      <td>5.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0728208cff5242993896da88a882e076094b765f</td>\n",
       "      <td>dec</td>\n",
       "      <td>2017</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>Billings</td>\n",
       "      <td>679.140000</td>\n",
       "      <td>681.865000</td>\n",
       "      <td>681.705000</td>\n",
       "      <td>50.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0bc51b95b2dc23713f611e182c689ad03dc2c300</td>\n",
       "      <td>dec</td>\n",
       "      <td>2017</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>Great Falls</td>\n",
       "      <td>656.906977</td>\n",
       "      <td>657.837209</td>\n",
       "      <td>646.697674</td>\n",
       "      <td>10.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0bc51b95b2dc23713f611e182c689ad03dc2c300</td>\n",
       "      <td>dec</td>\n",
       "      <td>2017</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>Great Falls</td>\n",
       "      <td>661.723810</td>\n",
       "      <td>668.552381</td>\n",
       "      <td>668.476190</td>\n",
       "      <td>10.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>23d834c190c3686c02c030ee0bf42003872bd591</td>\n",
       "      <td>dec</td>\n",
       "      <td>2017</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>Bozeman</td>\n",
       "      <td>669.857143</td>\n",
       "      <td>683.285714</td>\n",
       "      <td>671.142857</td>\n",
       "      <td>25.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>71</th>\n",
       "      <td>fd93662943beeb4603fd63aacbbbb7abb3183470</td>\n",
       "      <td>dec</td>\n",
       "      <td>2020</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>NASHUA</td>\n",
       "      <td>690.474684</td>\n",
       "      <td>728.227848</td>\n",
       "      <td>689.636076</td>\n",
       "      <td>30.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>72</th>\n",
       "      <td>fd93662943beeb4603fd63aacbbbb7abb3183470</td>\n",
       "      <td>dec</td>\n",
       "      <td>2020</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>NASHUA</td>\n",
       "      <td>703.777778</td>\n",
       "      <td>690.222222</td>\n",
       "      <td>690.000000</td>\n",
       "      <td>30.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>73</th>\n",
       "      <td>fec2ec7ae8c5ee7ae93fe16b022bc4c1fc6f2931</td>\n",
       "      <td>dec</td>\n",
       "      <td>2020</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>Kalispell</td>\n",
       "      <td>724.106481</td>\n",
       "      <td>740.000000</td>\n",
       "      <td>720.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>74</th>\n",
       "      <td>fec2ec7ae8c5ee7ae93fe16b022bc4c1fc6f2931</td>\n",
       "      <td>dec</td>\n",
       "      <td>2020</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>Kalispell</td>\n",
       "      <td>706.644195</td>\n",
       "      <td>721.093633</td>\n",
       "      <td>701.093633</td>\n",
       "      <td>0.0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75</th>\n",
       "      <td>fec2ec7ae8c5ee7ae93fe16b022bc4c1fc6f2931</td>\n",
       "      <td>dec</td>\n",
       "      <td>2020</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>Kalispell</td>\n",
       "      <td>702.008264</td>\n",
       "      <td>720.066116</td>\n",
       "      <td>700.198347</td>\n",
       "      <td>0.0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>297 rows × 13 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                  Identifier Month  Year HvacMode  \\\n",
       "0   019764d7b48dd9d6f1f4ad95451d7d0da2669405   dec  2017     auto   \n",
       "1   0728208cff5242993896da88a882e076094b765f   dec  2017     heat   \n",
       "2   0bc51b95b2dc23713f611e182c689ad03dc2c300   dec  2017     heat   \n",
       "3   0bc51b95b2dc23713f611e182c689ad03dc2c300   dec  2017     heat   \n",
       "4   23d834c190c3686c02c030ee0bf42003872bd591   dec  2017     heat   \n",
       "..                                       ...   ...   ...      ...   \n",
       "71  fd93662943beeb4603fd63aacbbbb7abb3183470   dec  2020     heat   \n",
       "72  fd93662943beeb4603fd63aacbbbb7abb3183470   dec  2020     heat   \n",
       "73  fec2ec7ae8c5ee7ae93fe16b022bc4c1fc6f2931   dec  2020     auto   \n",
       "74  fec2ec7ae8c5ee7ae93fe16b022bc4c1fc6f2931   dec  2020     auto   \n",
       "75  fec2ec7ae8c5ee7ae93fe16b022bc4c1fc6f2931   dec  2020     heat   \n",
       "\n",
       "   CalendarEvent         City  Temperature_ctrl_ave  \\\n",
       "0           auto     Billings            709.260870   \n",
       "1           hold     Billings            679.140000   \n",
       "2           auto  Great Falls            656.906977   \n",
       "3           hold  Great Falls            661.723810   \n",
       "4           auto      Bozeman            669.857143   \n",
       "..           ...          ...                   ...   \n",
       "71          auto       NASHUA            690.474684   \n",
       "72          hold       NASHUA            703.777778   \n",
       "73          auto    Kalispell            724.106481   \n",
       "74          hold    Kalispell            706.644195   \n",
       "75          hold    Kalispell            702.008264   \n",
       "\n",
       "    TemperatureExpectedCool_ave  TemperatureExpectedHeat_ave  \\\n",
       "0                    740.000000                   690.000000   \n",
       "1                    681.865000                   681.705000   \n",
       "2                    657.837209                   646.697674   \n",
       "3                    668.552381                   668.476190   \n",
       "4                    683.285714                   671.142857   \n",
       "..                          ...                          ...   \n",
       "71                   728.227848                   689.636076   \n",
       "72                   690.222222                   690.000000   \n",
       "73                   740.000000                   720.000000   \n",
       "74                   721.093633                   701.093633   \n",
       "75                   720.066116                   700.198347   \n",
       "\n",
       "    Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \n",
       "0                   5.0             False         False            False  \n",
       "1                  50.0             False         False            False  \n",
       "2                  10.0             False         False            False  \n",
       "3                  10.0             False         False            False  \n",
       "4                  25.0             False         False            False  \n",
       "..                  ...               ...           ...              ...  \n",
       "71                 30.0             False         False            False  \n",
       "72                 30.0             False         False            False  \n",
       "73                  0.0              True         False            False  \n",
       "74                  0.0              True         False            False  \n",
       "75                  0.0              True         False            False  \n",
       "\n",
       "[297 rows x 13 columns]"
      ]
     },
     "execution_count": 186,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# https://stacMToverflow.com/questions/63886787/how-to-create-a-dataframe-from-multiple-csv-files\n",
    "# Initialize an empty dataframe\n",
    "MT_dec = pd.DataFrame()\n",
    "\n",
    "# Iterate through files and contents, then concatenate the data into the dataframe\n",
    "for file in files:\n",
    "    df = pd.read_csv(\"data/day/MT/dec/\" + file)\n",
    "    MT_dec = pd.concat([MT_dec, df])\n",
    "    \n",
    "MT_dec"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 187,
   "metadata": {},
   "outputs": [],
   "source": [
    "MT_dec.to_csv(\"Scraper_Output/State_Month_Day/MT/MT_dec.csv\", header=True, index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "----"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "----"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "\n",
    "### Combine state CSV Files \n",
    "1. Read in files in folders for each state\n",
    "2. Export as combined CSV"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 188,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create variable for files in directory\n",
    "files = [f for f in os.listdir(\"Scraper_Output/State_Month_Day/MT/\") if f.endswith(\".csv\")]\n",
    "\n",
    "# files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 189,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>Month</th>\n",
       "      <th>Year</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>City</th>\n",
       "      <th>Temperature_ctrl_ave</th>\n",
       "      <th>TemperatureExpectedCool_ave</th>\n",
       "      <th>TemperatureExpectedHeat_ave</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>037988cc24eaeaa08f7002dbc926d02c8f9c0f1c</td>\n",
       "      <td>aug</td>\n",
       "      <td>2017</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>Eureka</td>\n",
       "      <td>698.184211</td>\n",
       "      <td>724.421053</td>\n",
       "      <td>734.736842</td>\n",
       "      <td>10.0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>037988cc24eaeaa08f7002dbc926d02c8f9c0f1c</td>\n",
       "      <td>aug</td>\n",
       "      <td>2017</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>Eureka</td>\n",
       "      <td>692.443119</td>\n",
       "      <td>719.359608</td>\n",
       "      <td>719.305922</td>\n",
       "      <td>10.0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0bc51b95b2dc23713f611e182c689ad03dc2c300</td>\n",
       "      <td>aug</td>\n",
       "      <td>2017</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>Great Falls</td>\n",
       "      <td>705.645833</td>\n",
       "      <td>750.000000</td>\n",
       "      <td>680.000000</td>\n",
       "      <td>10.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0bc51b95b2dc23713f611e182c689ad03dc2c300</td>\n",
       "      <td>aug</td>\n",
       "      <td>2017</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>Great Falls</td>\n",
       "      <td>709.226804</td>\n",
       "      <td>724.481100</td>\n",
       "      <td>720.621993</td>\n",
       "      <td>10.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1e4188c276ca852a75feef8c5e3fc27e1f409c4d</td>\n",
       "      <td>aug</td>\n",
       "      <td>2017</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>Helena</td>\n",
       "      <td>741.703704</td>\n",
       "      <td>740.000000</td>\n",
       "      <td>740.000000</td>\n",
       "      <td>5.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>299</th>\n",
       "      <td>f404081c89442a6b4699e93e43dc4bb72f9e2dc1</td>\n",
       "      <td>jun</td>\n",
       "      <td>2021</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>Great Falls</td>\n",
       "      <td>721.327862</td>\n",
       "      <td>730.421420</td>\n",
       "      <td>680.421420</td>\n",
       "      <td>55.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>300</th>\n",
       "      <td>f74a23b378011473f8b32b3fc2efbbb3b6a9e098</td>\n",
       "      <td>jun</td>\n",
       "      <td>2021</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>Billings</td>\n",
       "      <td>679.107143</td>\n",
       "      <td>667.160714</td>\n",
       "      <td>617.160714</td>\n",
       "      <td>55.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>301</th>\n",
       "      <td>f74a23b378011473f8b32b3fc2efbbb3b6a9e098</td>\n",
       "      <td>jun</td>\n",
       "      <td>2021</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>Billings</td>\n",
       "      <td>664.848837</td>\n",
       "      <td>655.116279</td>\n",
       "      <td>654.720930</td>\n",
       "      <td>55.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>302</th>\n",
       "      <td>fd93662943beeb4603fd63aacbbbb7abb3183470</td>\n",
       "      <td>jun</td>\n",
       "      <td>2021</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>NASHUA</td>\n",
       "      <td>745.318584</td>\n",
       "      <td>741.955752</td>\n",
       "      <td>740.814159</td>\n",
       "      <td>30.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>303</th>\n",
       "      <td>fec2ec7ae8c5ee7ae93fe16b022bc4c1fc6f2931</td>\n",
       "      <td>jun</td>\n",
       "      <td>2021</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>Kalispell</td>\n",
       "      <td>712.032132</td>\n",
       "      <td>709.202194</td>\n",
       "      <td>679.605016</td>\n",
       "      <td>0.0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1598 rows × 13 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                   Identifier Month  Year HvacMode  \\\n",
       "0    037988cc24eaeaa08f7002dbc926d02c8f9c0f1c   aug  2017     cool   \n",
       "1    037988cc24eaeaa08f7002dbc926d02c8f9c0f1c   aug  2017     cool   \n",
       "2    0bc51b95b2dc23713f611e182c689ad03dc2c300   aug  2017     cool   \n",
       "3    0bc51b95b2dc23713f611e182c689ad03dc2c300   aug  2017     cool   \n",
       "4    1e4188c276ca852a75feef8c5e3fc27e1f409c4d   aug  2017     cool   \n",
       "..                                        ...   ...   ...      ...   \n",
       "299  f404081c89442a6b4699e93e43dc4bb72f9e2dc1   jun  2021     auto   \n",
       "300  f74a23b378011473f8b32b3fc2efbbb3b6a9e098   jun  2021     auto   \n",
       "301  f74a23b378011473f8b32b3fc2efbbb3b6a9e098   jun  2021     cool   \n",
       "302  fd93662943beeb4603fd63aacbbbb7abb3183470   jun  2021     cool   \n",
       "303  fec2ec7ae8c5ee7ae93fe16b022bc4c1fc6f2931   jun  2021     auto   \n",
       "\n",
       "    CalendarEvent         City  Temperature_ctrl_ave  \\\n",
       "0            auto       Eureka            698.184211   \n",
       "1            hold       Eureka            692.443119   \n",
       "2            auto  Great Falls            705.645833   \n",
       "3            hold  Great Falls            709.226804   \n",
       "4            hold       Helena            741.703704   \n",
       "..            ...          ...                   ...   \n",
       "299          hold  Great Falls            721.327862   \n",
       "300          hold     Billings            679.107143   \n",
       "301          hold     Billings            664.848837   \n",
       "302          hold       NASHUA            745.318584   \n",
       "303          hold    Kalispell            712.032132   \n",
       "\n",
       "     TemperatureExpectedCool_ave  TemperatureExpectedHeat_ave  \\\n",
       "0                     724.421053                   734.736842   \n",
       "1                     719.359608                   719.305922   \n",
       "2                     750.000000                   680.000000   \n",
       "3                     724.481100                   720.621993   \n",
       "4                     740.000000                   740.000000   \n",
       "..                           ...                          ...   \n",
       "299                   730.421420                   680.421420   \n",
       "300                   667.160714                   617.160714   \n",
       "301                   655.116279                   654.720930   \n",
       "302                   741.955752                   740.814159   \n",
       "303                   709.202194                   679.605016   \n",
       "\n",
       "     Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \n",
       "0                   10.0              True         False             True  \n",
       "1                   10.0              True         False             True  \n",
       "2                   10.0             False         False            False  \n",
       "3                   10.0             False         False            False  \n",
       "4                    5.0             False         False            False  \n",
       "..                   ...               ...           ...              ...  \n",
       "299                 55.0             False         False            False  \n",
       "300                 55.0             False         False            False  \n",
       "301                 55.0             False         False            False  \n",
       "302                 30.0             False         False            False  \n",
       "303                  0.0              True         False            False  \n",
       "\n",
       "[1598 rows x 13 columns]"
      ]
     },
     "execution_count": 189,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# https://stacMToverflow.com/questions/63886787/how-to-create-a-dataframe-from-multiple-csv-files\n",
    "# Initialize an empty dataframe\n",
    "MT_all = pd.DataFrame()\n",
    "\n",
    "# Iterate through files and contents, then concatenate the data into the dataframe\n",
    "for file in files:\n",
    "    df = pd.read_csv(\"Scraper_Output/State_Month_Day/MT/\" + file)\n",
    "    MT_all = pd.concat([MT_all, df])\n",
    "    \n",
    "MT_all"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 190,
   "metadata": {},
   "outputs": [],
   "source": [
    "MT_all.to_csv(\"Scraper_Output/State_Month_Day/MT_all_day.csv\", header=True, index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 191,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Unique jan_2017: ['MT']\n",
      "Unique jan_2018: ['MT']\n",
      "Unique jan_2019: ['MT']\n",
      "Unique jan_2020: ['MT']\n",
      "Unique jan_2021: ['MT']\n",
      "Unique feb_2017: ['MT']\n",
      "Unique feb_2018: ['MT']\n",
      "Unique feb_2019: ['MT']\n",
      "Unique feb_2020: ['MT']\n",
      "Unique feb_2021: ['MT']\n",
      "Unique jun_2017: ['MT']\n",
      "Unique jun_2018: ['MT']\n",
      "Unique jun_2019: ['MT']\n",
      "Unique jun_2020: ['MT']\n",
      "Unique jun_2021: ['MT']\n",
      "Unique jul_2017: ['MT']\n",
      "Unique jul_2018: ['MT']\n",
      "Unique jul_2019: ['MT']\n",
      "Unique jul_2020: ['MT']\n",
      "Unique jul_2021: ['MT']\n",
      "Unique aug_2017: ['MT']\n",
      "Unique aug_2018: ['MT']\n",
      "Unique aug_2019: ['MT']\n",
      "Unique aug_2020: ['MT']\n",
      "Unique dec_2017: ['MT']\n",
      "Unique dec_2018: ['MT']\n",
      "Unique dec_2019: ['MT']\n",
      "Unique dec_2020: ['MT']\n"
     ]
    }
   ],
   "source": [
    "# DatachecMT to maMTe sure state was selected correctly in BQ sql queries\n",
    "\n",
    "print(f\"Unique jan_2017: {jan_2017['ProvinceState'].unique()}\")\n",
    "print(f\"Unique jan_2018: {jan_2018['ProvinceState'].unique()}\")\n",
    "print(f\"Unique jan_2019: {jan_2019['ProvinceState'].unique()}\")\n",
    "print(f\"Unique jan_2020: {jan_2020['ProvinceState'].unique()}\")\n",
    "print(f\"Unique jan_2021: {jan_2021['ProvinceState'].unique()}\")\n",
    "print(f\"Unique feb_2017: {feb_2017['ProvinceState'].unique()}\")\n",
    "print(f\"Unique feb_2018: {feb_2018['ProvinceState'].unique()}\")\n",
    "print(f\"Unique feb_2019: {feb_2019['ProvinceState'].unique()}\")\n",
    "print(f\"Unique feb_2020: {feb_2020['ProvinceState'].unique()}\")\n",
    "print(f\"Unique feb_2021: {feb_2021['ProvinceState'].unique()}\")\n",
    "print(f\"Unique jun_2017: {jun_2017['ProvinceState'].unique()}\")\n",
    "print(f\"Unique jun_2018: {jun_2018['ProvinceState'].unique()}\")\n",
    "print(f\"Unique jun_2019: {jun_2019['ProvinceState'].unique()}\")\n",
    "print(f\"Unique jun_2020: {jun_2020['ProvinceState'].unique()}\")\n",
    "print(f\"Unique jun_2021: {jun_2021['ProvinceState'].unique()}\")\n",
    "print(f\"Unique jul_2017: {jul_2017['ProvinceState'].unique()}\")\n",
    "print(f\"Unique jul_2018: {jul_2018['ProvinceState'].unique()}\")\n",
    "print(f\"Unique jul_2019: {jul_2019['ProvinceState'].unique()}\")\n",
    "print(f\"Unique jul_2020: {jul_2020['ProvinceState'].unique()}\")\n",
    "print(f\"Unique jul_2021: {jul_2021['ProvinceState'].unique()}\")\n",
    "print(f\"Unique aug_2017: {aug_2017['ProvinceState'].unique()}\")\n",
    "print(f\"Unique aug_2018: {aug_2018['ProvinceState'].unique()}\")\n",
    "print(f\"Unique aug_2019: {aug_2019['ProvinceState'].unique()}\")\n",
    "print(f\"Unique aug_2020: {aug_2020['ProvinceState'].unique()}\")\n",
    "print(f\"Unique dec_2017: {dec_2017['ProvinceState'].unique()}\")\n",
    "print(f\"Unique dec_2018: {dec_2018['ProvinceState'].unique()}\")\n",
    "print(f\"Unique dec_2019: {dec_2019['ProvinceState'].unique()}\")\n",
    "print(f\"Unique dec_2020: {dec_2020['ProvinceState'].unique()}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.10"
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}