Merge pull request #99 from ReagentX/develop

Purple Air Client v1.3.0

.github/workflows/python-app.yml

@@ -28,7 +28,7 @@ jobs:
         pip install mypy pylint bandit
     - name: Lint with pylint
       run: |
-        pylint -d R0912 -d R0915 -d R0902 -d R0911 purpleair
+        pylint -d R0902 -d R0911 -d R0912 -d R0913 -d R0915 purpleair
     - name: Static Analysis with mypy
       run: |
         mypy . --ignore-missing-imports

docs/api/channel_methods.md

@@ -117,13 +117,47 @@ The data is shaped like this:
 }
 ```
 
-## `get_historical(weeks_to_get: int, thingspeak_field: str, start_date: datetime = datetime.now()) -> pd.DataFrame`
+## `get_historical(weeks_to_get: int, thingspeak_field: str, start_date: datetime = datetime.now(), thingspeak_args: Dict[str, Any] = None) -> pd.DataFrame`
 
-Get data from the ThingSpeak API from field `thingspeak_field` one week at a time up to `weeks_to_get` weeks in the past.
+Get either primary or secondary data from the ThingSpeak API from field `thingspeak_field` one week at a time up to `weeks_to_get` weeks in the past.
 
 `thingspeak_field` is one of `{'primary', 'secondary'}`.
 
-`start_date` is an optional field to supply a start date. `weeks_to_get` is relative to this value. If not set, it defaults to `datetime.now()`
+`start_date` is an optional field to supply a start date. `weeks_to_get` is relative (historical) to this value. If not set, `start_date` defaults to `datetime.now()`
+
+`thingspeak_args` are optional parameters to send to the thingspeak API. The available paramters are listed [here](https://www.mathworks.com/help/thingspeak/readdata.html).
+
+See [Channel Fields](#channel-fields) for a description of available data.
+
+## `get_historical_between(thingspeak_field: str, start_date: datetime, end_date: datetime = datetime.now(), thingspeak_args: Dict[str, Any] = None)`
+
+Get either primary or secondary data from the ThingSpeak API from `start_date` to `end_date`. If omitted, `end_date` defaults to the current date and time.
+
+`thingspeak_field` is one of `{'primary', 'secondary'}`.
+
+`thingspeak_args` are optional parameters to send to the thingspeak API. The available paramters are listed [here](https://www.mathworks.com/help/thingspeak/readdata.html).
+
+See [Channel Fields](#channel-fields) for a description of available data.
+
+## `get_all_historical(weeks_to_get: int, start_date: datetime = datetime.now(), thingspeak_args: Dict[str, Any] = None) -> pd.DataFrame`
+
+Get both primary and secondary data from the ThingSpeak API from field `thingspeak_field` one week at a time up to `weeks_to_get` weeks in the past.
+
+`start_date` is an optional field to supply a start date. `weeks_to_get` is relative (historical) to this value. If not set, `start_date` defaults to `datetime.now()`
+
+`thingspeak_args` are optional parameters to send to the thingspeak API. The available paramters are listed [here](https://www.mathworks.com/help/thingspeak/readdata.html).
+
+See [Channel Fields](#channel-fields) for a description of available data.
+
+## `get_all_historical_between(start_date: datetime, end_date: datetime = datetime.now(), thingspeak_args: Dict[str, Any] = None)`
+
+Get both primary and secondary data from the ThingSpeak API from `start_date` to `end_date`. If omitted, `end_date` defaults to the current date and time.
+
+`thingspeak_args` are optional parameters to send to the thingspeak API. The available paramters are listed [here](https://www.mathworks.com/help/thingspeak/readdata.html).
+
+See [Channel Fields](#channel-fields) for a description of available data.
+
+## Channel Fields
 
 Parent Primary:
 

docs/documentation.md

@@ -56,6 +56,8 @@ Initialize a new sensor.
     * The location type of the sensor, one of `{'indoor', 'outdoor', 'unknown'}`
   * `location`
     * Location string if `parse_location` was true, otherwise empty string
+  * `created_date`
+    * The date the sensor was created
 
 See [api/sensor_methods.md](api/sensor_methods.md) for method documentation.
 
@@ -68,6 +70,8 @@ Representation of a sensor channel, either `a` or `b`. For channel `b` (child) s
 * Properties
   * `channel_data`
     * metadata in Python dictionary format about the channel
+  * `created_date`
+    * The date the channel was created
   * `lat`
     * Sensor latitude
   * `lon`

purpleair/api_data.py

@@ -4,6 +4,7 @@
 
 
 API_ROOT = 'https://www.purpleair.com/json'
+THINGSPEAK_API_URL = "https://thingspeak.com/channels/{channel}/feed.{dataformat}?"
 
 PARENT_PRIMARY_COLS = {
     'created_at': 'created_at',

purpleair/channel.py

@@ -4,13 +4,16 @@
 
 import json
 from datetime import datetime, timedelta
-from typing import Optional
+from typing import Any, Dict, Optional
+from urllib.parse import urlencode
 
 import pandas as pd
 import thingspeak
+from requests_cache import CachedSession
 
-from .api_data import (PARENT_PRIMARY_COLS, PARENT_SECONDARY_COLS,
-                       CHILD_PRIMARY_COLS, CHILD_SECONDARY_COLS)
+from .api_data import (CHILD_PRIMARY_COLS, CHILD_SECONDARY_COLS,
+                       PARENT_PRIMARY_COLS, PARENT_SECONDARY_COLS,
+                       THINGSPEAK_API_URL)
 
 
 class Channel():
@@ -20,9 +23,9 @@ class Channel():
 
     def __init__(self, channel_data: dict):
         self.channel_data = channel_data
-        self.setup()
+        self._setup()
 
-    def safe_float(self, key: str) -> Optional[float]:
+    def _safe_float(self, key: str) -> Optional[float]:
         """
         Convert to float if the item exists, otherwise return none
         """
@@ -36,13 +39,13 @@ def safe_float(self, key: str) -> Optional[float]:
                 return None
         return result
 
-    def setup(self) -> None:
+    def _setup(self) -> None:
         """
         Initialize metadata and real data for a sensor; for detailed info see docs
         """
         # Meta
-        self.lat: Optional[float] = self.safe_float('Lat')
-        self.lon: Optional[float] = self.safe_float('Lon')
+        self.lat: Optional[float] = self._safe_float('Lat')
+        self.lon: Optional[float] = self._safe_float('Lon')
         self.identifier: Optional[int] = self.channel_data.get('ID')
         self.parent: Optional[int] = self.channel_data.get('ParentID')
         self.type: str = 'parent' if self.parent is None else 'child'
@@ -52,32 +55,32 @@ def setup(self) -> None:
             'DEVICE_LOCATIONTYPE')
 
         # Data, possible TODO: abstract to class
-        self.current_pm2_5: Optional[float] = self.safe_float('PM2_5Value')
-        self.current_temp_f: Optional[float] = self.safe_float('temp_f')
+        self.current_pm2_5: Optional[float] = self._safe_float('PM2_5Value')
+        self.current_temp_f: Optional[float] = self._safe_float('temp_f')
         self.current_temp_c = (self.current_temp_f - 32) * (5 / 9) \
             if self.current_temp_f is not None else None
-        self.current_humidity: Optional[float] = self.safe_float('humidity')
-        self.current_pressure: Optional[float] = self.safe_float('pressure')
-        self.current_p_0_3_um: Optional[float] = self.safe_float('p_0_3_um')
-        self.current_p_0_5_um: Optional[float] = self.safe_float('p_0_5_um')
-        self.current_p_1_0_um: Optional[float] = self.safe_float('p_1_0_um')
-        self.current_p_2_5_um: Optional[float] = self.safe_float('p_2_5_um')
-        self.current_p_5_0_um: Optional[float] = self.safe_float('p_5_0_um')
-        self.current_p_10_0_um: Optional[float] = self.safe_float('p_10_0_um')
-        self.current_pm1_0_cf_1: Optional[float] = self.safe_float(
+        self.current_humidity: Optional[float] = self._safe_float('humidity')
+        self.current_pressure: Optional[float] = self._safe_float('pressure')
+        self.current_p_0_3_um: Optional[float] = self._safe_float('p_0_3_um')
+        self.current_p_0_5_um: Optional[float] = self._safe_float('p_0_5_um')
+        self.current_p_1_0_um: Optional[float] = self._safe_float('p_1_0_um')
+        self.current_p_2_5_um: Optional[float] = self._safe_float('p_2_5_um')
+        self.current_p_5_0_um: Optional[float] = self._safe_float('p_5_0_um')
+        self.current_p_10_0_um: Optional[float] = self._safe_float('p_10_0_um')
+        self.current_pm1_0_cf_1: Optional[float] = self._safe_float(
             'pm1_0_cf_1')
-        self.current_pm2_5_cf_1: Optional[float] = self.safe_float(
+        self.current_pm2_5_cf_1: Optional[float] = self._safe_float(
             'pm2_5_cf_1')
-        self.current_pm10_0_cf_1: Optional[float] = self.safe_float(
+        self.current_pm10_0_cf_1: Optional[float] = self._safe_float(
             'pm10_0_cf_1')
-        self.current_pm1_0_atm: Optional[float] = self.safe_float('pm1_0_atm')
-        self.current_pm2_5_atm: Optional[float] = self.safe_float('pm2_5_atm')
-        self.current_pm10_0_atm: Optional[float] = self.safe_float(
+        self.current_pm1_0_atm: Optional[float] = self._safe_float('pm1_0_atm')
+        self.current_pm2_5_atm: Optional[float] = self._safe_float('pm2_5_atm')
+        self.current_pm10_0_atm: Optional[float] = self._safe_float(
             'pm10_0_atm')
 
         # Statistics
-        self.pm2_5stats: Optional[dict] = json.loads(self.channel_data['Stats']) \
-            if 'Stats' in self.channel_data else None
+        self.pm2_5stats: Optional[dict] = json.loads(
+            self.channel_data['Stats']) if 'Stats' in self.channel_data else None
         self.m10avg: Optional[float] = self.pm2_5stats.get(
             'v1') if self.pm2_5stats else None
         self.m30avg: Optional[float] = self.pm2_5stats.get(
@@ -150,48 +153,184 @@ def setup(self) -> None:
         self.uptime: Optional[int] = self.channel_data.get('Uptime')
         self.is_owner: Optional[bool] = bool(self.channel_data.get('isOwner'))
 
-    def get_historical(self,
-                       weeks_to_get: int,
-                       thingspeak_field: str,
-                       start_date: datetime = datetime.now()) -> pd.DataFrame:
+    @property
+    def created_date(self):
+        """
+        Gets the date the channel was created
+
+        Useful for finding out the earliest data point for a given channel
+        """
+        url = self._get_thingspeak_url(
+            'primary', start=datetime(
+                1990, 1, 1), end=None, thingspeak_args={
+                'results': 1}, dataformat='json')
+
+        session = CachedSession(expire_after=timedelta(hours=1))
+        response = session.get(url)
+        data = json.loads(response.content)
+        created_at = datetime.strptime(
+            data['channel']['created_at'],
+            '%Y-%m-%dT%H:%M:%SZ')
+        return created_at
+
+    def _get_thingspeak_url(
+            self,
+            thingspeak_field: str,
+            start: datetime,
+            end: Optional[datetime] = None,
+            thingspeak_args: Optional[Dict[str, Any]] = None,
+            dataformat: str = 'csv'):
         """
-        Get data from the ThingSpeak API one week at a time up to weeks_to_get weeks in the past
+        Build the URL to fetch the thingspeak data
+
+        `thingspeak_args` takes an optional list of additional arguments
+        to send to the Thingspeak API.
+        See here for more details: https://www.mathworks.com/help/thingspeak/readdata.html
         """
+
         if thingspeak_field not in {'primary', 'secondary'}:
             # pylint: disable=line-too-long
             raise ValueError(
                 f'Invalid ThingSpeak key: {thingspeak_field}. Must be in {{"primary", "secondary"}}')
 
-        # Determine channel and key
-        # pylint: disable=line-too-long
-        channel = self.tp_primary_channel if thingspeak_field == 'primary' else self.tp_secondary_channel
-        key = self.tp_primary_key if thingspeak_field == 'primary' else self.tp_secondary_key
+        if thingspeak_field == 'primary':
+            channel = self.tp_primary_channel
+            key = self.tp_primary_key
+        else:
+            channel = self.tp_secondary_channel
+            key = self.tp_secondary_key
 
-        # Determine column columns
-        # pylint: disable=line-too-long
-        parent_cols = PARENT_PRIMARY_COLS if thingspeak_field == 'primary' else PARENT_SECONDARY_COLS
-        # pylint: disable=line-too-long
-        child_cols = CHILD_PRIMARY_COLS if thingspeak_field == 'primary' else CHILD_SECONDARY_COLS
+        # copy args to a local variable
+        if thingspeak_args:
+            thingspeak_args = thingspeak_args.copy()
+        else:
+            thingspeak_args = {}
+
+        default_args = {
+            'api_key': key,
+            'offset': 0,
+            'average': '',
+            'round': 2,
+        }
+
+        for key, val in default_args.items():
+            if key not in thingspeak_args:
+                thingspeak_args[key] = val
+
+        thingspeak_args['start'] = start.strftime("%Y-%m-%d 00:00:00")
+        if end:
+            thingspeak_args['end'] = end.strftime("%Y-%m-%d 00:00:00")
+
+        base_url = THINGSPEAK_API_URL.format(
+            channel=channel, dataformat=dataformat)
+        return base_url + urlencode(thingspeak_args)
+
+    def _clean_data(self, thingspeak_field: str, data: pd.DataFrame):
+        """
+        Cleans up data from the thingspeak API
+
+        * Inserts the correct column names
+        * Sets the index to `entry_id` if it exists
+        """
+
+        if thingspeak_field == 'primary':
+            parent_cols = PARENT_PRIMARY_COLS
+            child_cols = CHILD_PRIMARY_COLS
+        else:
+            parent_cols = PARENT_SECONDARY_COLS
+            child_cols = CHILD_SECONDARY_COLS
 
         columns = parent_cols if self.type == 'parent' else child_cols
+
+        data.rename(columns=columns, inplace=True)
+        data['created_at'] = pd.to_datetime(
+            data['created_at'], format='%Y-%m-%d %H:%M:%S %Z')
+
+        try:
+            data.index = data.pop('entry_id')
+        except KeyError:
+            # entry_id isn't always present. E.g. if you use
+            # timescale='nonstandard'
+            pass
+        return data
+
+    def get_all_historical(self,
+                           weeks_to_get: int,
+                           start_date: datetime = datetime.now(),
+                           thingspeak_args: Optional[Dict[str, Any]] = None) -> pd.DataFrame:
+        """
+        Get all data (both primary and secondary) from the ThingSpeak API in weekly increments
+
+        """
+
+        primary = self.get_historical(
+            weeks_to_get, 'primary', start_date, thingspeak_args)
+        secondary = self.get_historical(
+            weeks_to_get, 'secondary', start_date, thingspeak_args)
+        return pd.merge(primary, secondary, how='inner', on='created_at')
+
+    def get_all_historical_between(self,
+                                   start_date: datetime,
+                                   end_date: datetime = datetime.now(),
+                                   thingspeak_args: Optional[Dict[str, Any]] = None
+                                   ) -> pd.DataFrame:
+        """
+        Get all data (both primary and secondary) from the ThingSpeak API between two dates
+
+        WARNING: For huge date ranges, this may be a large dataset, and take
+        a long time to download. In these situations, get_historical (by week)
+        may be a better option.
+
+        """
+        primary = self.get_historical_between(
+            'primary', start_date, end_date, thingspeak_args)
+        secondary = self.get_historical_between(
+            'secondary', start_date, end_date, thingspeak_args)
+        return pd.merge(primary, secondary, how='inner', on='created_at')
+
+    def get_historical_between(self,
+                               thingspeak_field: str,
+                               start_date: datetime,
+                               end_date: datetime = datetime.now(),
+                               thingspeak_args=None) -> pd.DataFrame:
+        """
+        Get data from the ThingSpeak API in one go between two dates.
+
+        WARNING: For huge date ranges, this may be a large dataset, and take
+        a long time to download. In these situations, get_historical (by week)
+        may be a better option.
+        """
+
+        url = self._get_thingspeak_url(
+            thingspeak_field,
+            start_date,
+            end_date,
+            thingspeak_args)
+        return self._clean_data(thingspeak_field, pd.read_csv(url))
+
+    def get_historical(self,
+                       weeks_to_get: int,
+                       thingspeak_field: str,
+                       start_date: datetime = datetime.now(),
+                       thingspeak_args: Optional[Dict[str, Any]] = None) -> pd.DataFrame:
+        """
+        Get data from the ThingSpeak API one week at a time up to weeks_to_get weeks in the past.
+        """
         to_week = start_date - timedelta(weeks=1)
-        # pylint: disable=line-too-long
-        url = f'https://thingspeak.com/channels/{channel}/feed.csv?api_key={key}&offset=0&average=&round=2&start={to_week.strftime("%Y-%m-%d")}%2000:00:00&end={start_date.strftime("%Y-%m-%d")}%2000:00:00'
-        weekly_data = pd.read_csv(url)
-        if weeks_to_get > 1:
+        weekly_data = []
+        while weeks_to_get > 0:
             for _ in range(weeks_to_get):
                 start_date = to_week  # DateTimes are immutable so this reference is not a problem
                 to_week = to_week - timedelta(weeks=1)
-                # pylint: disable=line-too-long
-                url = f'https://thingspeak.com/channels/{channel}/feed.csv?api_key={key}&offset=0&average=&round=2&start={to_week.strftime("%Y-%m-%d")}%2000:00:00&end={start_date.strftime("%Y-%m-%d")}%2000:00:00'
-                weekly_data = pd.concat([weekly_data, pd.read_csv(url)])
-
-        # Handle formatting the DataFrame
-        weekly_data.rename(columns=columns, inplace=True)
-        weekly_data['created_at'] = pd.to_datetime(
-            weekly_data['created_at'], format='%Y-%m-%d %H:%M:%S %Z')
-        weekly_data.index = weekly_data.pop('entry_id')
-        return weekly_data
+                url = self._get_thingspeak_url(
+                    thingspeak_field, to_week, start_date, thingspeak_args)
+                weekly_data.append(pd.read_csv(url))
+                weeks_to_get -= 1
+
+        weekly_data_df = pd.DataFrame(pd.concat(weekly_data))
+
+        # Handle formatting the DataFrame column names
+        return self._clean_data(thingspeak_field, weekly_data_df)
 
     def as_dict(self) -> dict:
         """