diff --git a/docs/api/channel_methods.md b/docs/api/channel_methods.md index fc4cf65..f1cd1c0 100644 --- a/docs/api/channel_methods.md +++ b/docs/api/channel_methods.md @@ -117,13 +117,47 @@ The data is shaped like this: } ``` -## `get_historical(weeks_to_get: int, thingspeak_field: str, start_date: datetime = datetime.now()) -> pd.DataFrame` +## `get_historical(weeks_to_get: int, thingspeak_field: str, start_date: datetime = datetime.now(), thingspeak_args: Dict[str, Any] = None) -> pd.DataFrame` -Get data from the ThingSpeak API from field `thingspeak_field` one week at a time up to `weeks_to_get` weeks in the past. +Get either primary or secondary data from the ThingSpeak API from field `thingspeak_field` one week at a time up to `weeks_to_get` weeks in the past. `thingspeak_field` is one of `{'primary', 'secondary'}`. -`start_date` is an optional field to supply a start date. `weeks_to_get` is relative to this value. If not set, it defaults to `datetime.now()` +`start_date` is an optional field to supply a start date. `weeks_to_get` is relative (historical) to this value. If not set, `start_date` defaults to `datetime.now()` + +`thingspeak_args` are optional parameters to send to the thingspeak API. The available paramters are listed [here](https://www.mathworks.com/help/thingspeak/readdata.html). + +See [Channel Fields](#channel-fields) for a description of available data. + +## `get_historical_between(thingspeak_field: str, start_date: datetime, end_date: datetime = datetime.now(), thingspeak_args: Dict[str, Any] = None)` + +Get either primary or secondary data from the ThingSpeak API from `start_date` to `end_date`. If omitted, `end_date` defaults to the current date and time. + +`thingspeak_field` is one of `{'primary', 'secondary'}`. + +`thingspeak_args` are optional parameters to send to the thingspeak API. The available paramters are listed [here](https://www.mathworks.com/help/thingspeak/readdata.html). + +See [Channel Fields](#channel-fields) for a description of available data. + +## `get_all_historical(weeks_to_get: int, start_date: datetime = datetime.now(), thingspeak_args: Dict[str, Any] = None) -> pd.DataFrame` + +Get both primary and secondary data from the ThingSpeak API from field `thingspeak_field` one week at a time up to `weeks_to_get` weeks in the past. + +`start_date` is an optional field to supply a start date. `weeks_to_get` is relative (historical) to this value. If not set, `start_date` defaults to `datetime.now()` + +`thingspeak_args` are optional parameters to send to the thingspeak API. The available paramters are listed [here](https://www.mathworks.com/help/thingspeak/readdata.html). + +See [Channel Fields](#channel-fields) for a description of available data. + +## `get_all_historical_between(start_date: datetime, end_date: datetime = datetime.now(), thingspeak_args: Dict[str, Any] = None)` + +Get both primary and secondary data from the ThingSpeak API from `start_date` to `end_date`. If omitted, `end_date` defaults to the current date and time. + +`thingspeak_args` are optional parameters to send to the thingspeak API. The available paramters are listed [here](https://www.mathworks.com/help/thingspeak/readdata.html). + +See [Channel Fields](#channel-fields) for a description of available data. + +## Channel Fields Parent Primary: diff --git a/docs/documentation.md b/docs/documentation.md index bc15247..422cbc8 100644 --- a/docs/documentation.md +++ b/docs/documentation.md @@ -56,6 +56,8 @@ Initialize a new sensor. * The location type of the sensor, one of `{'indoor', 'outdoor', 'unknown'}` * `location` * Location string if `parse_location` was true, otherwise empty string + * `created_date` + * The date the sensor was created See [api/sensor_methods.md](api/sensor_methods.md) for method documentation. @@ -68,6 +70,8 @@ Representation of a sensor channel, either `a` or `b`. For channel `b` (child) s * Properties * `channel_data` * metadata in Python dictionary format about the channel + * `created_date` + * The date the channel was created * `lat` * Sensor latitude * `lon` diff --git a/maps/sensor_map.png b/maps/sensor_map.png index b3dae75..44e9502 100644 Binary files a/maps/sensor_map.png and b/maps/sensor_map.png differ diff --git a/purpleair/channel.py b/purpleair/channel.py index 684baef..7fc805f 100644 --- a/purpleair/channel.py +++ b/purpleair/channel.py @@ -4,20 +4,16 @@ import json from datetime import datetime, timedelta -from typing import Optional - +from typing import Any, Dict, Optional from urllib.parse import urlencode -from requests_cache import CachedSession import pandas as pd import thingspeak +from requests_cache import CachedSession -from .api_data import ( - PARENT_PRIMARY_COLS, - PARENT_SECONDARY_COLS, - CHILD_PRIMARY_COLS, - CHILD_SECONDARY_COLS, - THINGSPEAK_API_URL) +from .api_data import (CHILD_PRIMARY_COLS, CHILD_SECONDARY_COLS, + PARENT_PRIMARY_COLS, PARENT_SECONDARY_COLS, + THINGSPEAK_API_URL) class Channel(): @@ -27,9 +23,9 @@ class Channel(): def __init__(self, channel_data: dict): self.channel_data = channel_data - self.setup() + self._setup() - def safe_float(self, key: str) -> Optional[float]: + def _safe_float(self, key: str) -> Optional[float]: """ Convert to float if the item exists, otherwise return none """ @@ -43,13 +39,13 @@ def safe_float(self, key: str) -> Optional[float]: return None return result - def setup(self) -> None: + def _setup(self) -> None: """ Initialize metadata and real data for a sensor; for detailed info see docs """ # Meta - self.lat: Optional[float] = self.safe_float('Lat') - self.lon: Optional[float] = self.safe_float('Lon') + self.lat: Optional[float] = self._safe_float('Lat') + self.lon: Optional[float] = self._safe_float('Lon') self.identifier: Optional[int] = self.channel_data.get('ID') self.parent: Optional[int] = self.channel_data.get('ParentID') self.type: str = 'parent' if self.parent is None else 'child' @@ -59,27 +55,27 @@ def setup(self) -> None: 'DEVICE_LOCATIONTYPE') # Data, possible TODO: abstract to class - self.current_pm2_5: Optional[float] = self.safe_float('PM2_5Value') - self.current_temp_f: Optional[float] = self.safe_float('temp_f') + self.current_pm2_5: Optional[float] = self._safe_float('PM2_5Value') + self.current_temp_f: Optional[float] = self._safe_float('temp_f') self.current_temp_c = (self.current_temp_f - 32) * (5 / 9) \ if self.current_temp_f is not None else None - self.current_humidity: Optional[float] = self.safe_float('humidity') - self.current_pressure: Optional[float] = self.safe_float('pressure') - self.current_p_0_3_um: Optional[float] = self.safe_float('p_0_3_um') - self.current_p_0_5_um: Optional[float] = self.safe_float('p_0_5_um') - self.current_p_1_0_um: Optional[float] = self.safe_float('p_1_0_um') - self.current_p_2_5_um: Optional[float] = self.safe_float('p_2_5_um') - self.current_p_5_0_um: Optional[float] = self.safe_float('p_5_0_um') - self.current_p_10_0_um: Optional[float] = self.safe_float('p_10_0_um') - self.current_pm1_0_cf_1: Optional[float] = self.safe_float( + self.current_humidity: Optional[float] = self._safe_float('humidity') + self.current_pressure: Optional[float] = self._safe_float('pressure') + self.current_p_0_3_um: Optional[float] = self._safe_float('p_0_3_um') + self.current_p_0_5_um: Optional[float] = self._safe_float('p_0_5_um') + self.current_p_1_0_um: Optional[float] = self._safe_float('p_1_0_um') + self.current_p_2_5_um: Optional[float] = self._safe_float('p_2_5_um') + self.current_p_5_0_um: Optional[float] = self._safe_float('p_5_0_um') + self.current_p_10_0_um: Optional[float] = self._safe_float('p_10_0_um') + self.current_pm1_0_cf_1: Optional[float] = self._safe_float( 'pm1_0_cf_1') - self.current_pm2_5_cf_1: Optional[float] = self.safe_float( + self.current_pm2_5_cf_1: Optional[float] = self._safe_float( 'pm2_5_cf_1') - self.current_pm10_0_cf_1: Optional[float] = self.safe_float( + self.current_pm10_0_cf_1: Optional[float] = self._safe_float( 'pm10_0_cf_1') - self.current_pm1_0_atm: Optional[float] = self.safe_float('pm1_0_atm') - self.current_pm2_5_atm: Optional[float] = self.safe_float('pm2_5_atm') - self.current_pm10_0_atm: Optional[float] = self.safe_float( + self.current_pm1_0_atm: Optional[float] = self._safe_float('pm1_0_atm') + self.current_pm2_5_atm: Optional[float] = self._safe_float('pm2_5_atm') + self.current_pm10_0_atm: Optional[float] = self._safe_float( 'pm10_0_atm') # Statistics @@ -159,11 +155,12 @@ def setup(self) -> None: @property def created_date(self): - """Gets the date the channel was created + """ + Gets the date the channel was created Useful for finding out the earliest data point for a given channel """ - url = self.get_thingspeak_url( + url = self._get_thingspeak_url( 'primary', start=datetime( 1990, 1, 1), end=None, thingspeak_args={ 'results': 1}, dataformat='json') @@ -176,18 +173,19 @@ def created_date(self): '%Y-%m-%dT%H:%M:%SZ') return created_at - def get_thingspeak_url( + def _get_thingspeak_url( self, - thingspeak_field, - start, - end=None, - thingspeak_args=None, - dataformat='csv'): - """Build the URL to fetch the thingspeak data + thingspeak_field: str, + start: datetime, + end: Optional[datetime] = None, + thingspeak_args: Optional[Dict[str, Any]] = None, + dataformat: str = 'csv'): + """ + Build the URL to fetch the thingspeak data - thingspeak_args takes an optional list of additional arguments + `thingspeak_args` takes an optional list of additional arguments to send to the Thingspeak API. - See here for more details:https://ww2.mathworks.cn/help/thingspeak/readdata.html + See here for more details: https://www.mathworks.com/help/thingspeak/readdata.html """ if thingspeak_field not in {'primary', 'secondary'}: @@ -227,7 +225,7 @@ def get_thingspeak_url( channel=channel, dataformat=dataformat) return base_url + urlencode(thingspeak_args) - def clean_data(self, thingspeak_field, data): + def _clean_data(self, thingspeak_field: str, data: pd.DataFrame): """ Cleans up data from the thingspeak API @@ -259,7 +257,7 @@ def clean_data(self, thingspeak_field, data): def get_all_historical(self, weeks_to_get: int, start_date: datetime = datetime.now(), - thingspeak_args=None) -> pd.DataFrame: + thingspeak_args: Optional[Dict[str, Any]] = None) -> pd.DataFrame: """ Get all data (both primary and secondary) from the ThingSpeak API in weekly increments @@ -272,9 +270,10 @@ def get_all_historical(self, return pd.merge(primary, secondary, how='inner', on='created_at') def get_all_historical_between(self, - first_date: datetime, - last_date: datetime = datetime.now(), - thingspeak_args=None) -> pd.DataFrame: + start_date: datetime, + end_date: datetime = datetime.now(), + thingspeak_args: Optional[Dict[str, Any]] = None + ) -> pd.DataFrame: """ Get all data (both primary and secondary) from the ThingSpeak API between two dates @@ -284,15 +283,15 @@ def get_all_historical_between(self, """ primary = self.get_historical_between( - 'primary', first_date, last_date, thingspeak_args) + 'primary', start_date, end_date, thingspeak_args) secondary = self.get_historical_between( - 'secondary', first_date, last_date, thingspeak_args) + 'secondary', start_date, end_date, thingspeak_args) return pd.merge(primary, secondary, how='inner', on='created_at') def get_historical_between(self, thingspeak_field: str, - first_date: datetime, - last_date: datetime = datetime.now(), + start_date: datetime, + end_date: datetime = datetime.now(), thingspeak_args=None) -> pd.DataFrame: """ Get data from the ThingSpeak API in one go between two dates. @@ -302,21 +301,20 @@ def get_historical_between(self, may be a better option. """ - url = self.get_thingspeak_url( + url = self._get_thingspeak_url( thingspeak_field, - first_date, - last_date, + start_date, + end_date, thingspeak_args) - return self.clean_data(thingspeak_field, pd.read_csv(url)) + return self._clean_data(thingspeak_field, pd.read_csv(url)) def get_historical(self, weeks_to_get: int, thingspeak_field: str, start_date: datetime = datetime.now(), - thingspeak_args=None) -> pd.DataFrame: + thingspeak_args: Optional[Dict[str, Any]] = None) -> pd.DataFrame: """ Get data from the ThingSpeak API one week at a time up to weeks_to_get weeks in the past. - """ to_week = start_date - timedelta(weeks=1) weekly_data = [] @@ -324,15 +322,15 @@ def get_historical(self, for _ in range(weeks_to_get): start_date = to_week # DateTimes are immutable so this reference is not a problem to_week = to_week - timedelta(weeks=1) - url = self.get_thingspeak_url( + url = self._get_thingspeak_url( thingspeak_field, to_week, start_date, thingspeak_args) weekly_data.append(pd.read_csv(url)) weeks_to_get -= 1 - weekly_data = pd.concat(weekly_data) + weekly_data_df = pd.DataFrame(pd.concat(weekly_data)) # Handle formatting the DataFrame column names - return self.clean_data(thingspeak_field, weekly_data) + return self._clean_data(thingspeak_field, weekly_data_df) def as_dict(self) -> dict: """ diff --git a/purpleair/network.py b/purpleair/network.py index fdb826c..45ef0db 100644 --- a/purpleair/network.py +++ b/purpleair/network.py @@ -5,9 +5,9 @@ import json import time +from datetime import timedelta from json.decoder import JSONDecodeError from typing import List, Optional, Union -from datetime import timedelta import pandas as pd from requests_cache import CachedSession diff --git a/purpleair/sensor.py b/purpleair/sensor.py index ec6a7b1..0abec91 100644 --- a/purpleair/sensor.py +++ b/purpleair/sensor.py @@ -5,12 +5,12 @@ import json import os -from re import sub -from typing import Optional, List from datetime import timedelta +from re import sub +from typing import List, Optional -from requests_cache import CachedSession from geopy.geocoders import Nominatim +from requests_cache import CachedSession from .api_data import API_ROOT from .channel import Channel @@ -135,8 +135,7 @@ def is_useful(self) -> bool: if self.parent.current_pressure is None: return False if not self.parent.channel_data.get('Stats', None): - # Happens before stats because they will be missing if this is - # missing + # Happens before stats because they will be missing if this is missing return False if self.parent.last_modified_stats is None: return False diff --git a/tests/test_channel.py b/tests/test_channel.py index a760a76..fed39df 100644 --- a/tests/test_channel.py +++ b/tests/test_channel.py @@ -4,6 +4,7 @@ from purpleair import api_data import datetime + class TestChannelMethods(unittest.TestCase): """ Tests for Sensor class @@ -25,17 +26,17 @@ def test_get_historical(self): se.parent.get_historical(1, 'secondary') se.child.get_historical(1, 'primary') se.child.get_historical(1, 'secondary') - + def columns_for_channel(self, channel_type): if channel_type == 'child': columns = set(api_data.CHILD_PRIMARY_COLS.values()) - columns.update( api_data.CHILD_SECONDARY_COLS.values()) - columns.remove('entry_id') # remove entry_id + columns.update(api_data.CHILD_SECONDARY_COLS.values()) + columns.remove('entry_id') # remove entry_id else: columns = set(api_data.PARENT_PRIMARY_COLS.values()) - columns.update( api_data.PARENT_SECONDARY_COLS.values()) - columns.remove('entry_id') # remove entry_id - + columns.update(api_data.PARENT_SECONDARY_COLS.values()) + columns.remove('entry_id') # remove entry_id + return columns def test_get_all_historical(self): @@ -43,31 +44,30 @@ def test_get_all_historical(self): Test that we properly get both primary and secondary data in one go using the _all method """ se = sensor.Sensor(2891) - + # parent sensor parent_results = se.parent.get_all_historical(1) parent_columns = self.columns_for_channel('parent') for field in parent_columns: self.assertTrue(field in parent_results.columns) - - + # child sensor child_results = se.child.get_all_historical(1) child_columns = self.columns_for_channel('child') - + for field in child_columns: self.assertTrue(field in child_results.columns) - + def test_get_historical_between(self): """ Test getting the sensor's historical data between two dates """ - + se = sensor.Sensor(2891) start_date = datetime.datetime.today() - datetime.timedelta(weeks=1) se.parent.get_historical_between('primary', start_date) - + def test_get_all_historical_between(self): """ Test getting all the sensor's historical data (primary and secondary) between two dates @@ -81,8 +81,10 @@ def test_get_all_historical_between(self): def test_get_average_values(self): se = sensor.Sensor(2891) - results = se.parent.get_historical(1, 'primary', thingspeak_args={'average': 'daily'}) - self.assertTrue(len(results) in [7,8]) # either 7 or 8 results will be returned - depending on if we span 7 or 8 days with our 'week' time + results = se.parent.get_historical( + 1, 'primary', thingspeak_args={'average': 'daily'}) + # either 7 or 8 results will be returned - depending on if we span 7 or 8 days with our 'week' time + self.assertTrue(len(results) in [7, 8]) def test_as_dict(self): """