Lint and document base data structure

.github/CONTRIBUTING.rst

@@ -65,6 +65,6 @@ Because it is so strict, we have the following policy:
 
 **All new code should conform to the pylint standards for check categories of `FATAL` (F), `ERROR` (E), `WARNING` (W) and `REFACTOR` (R), when commited.**
 
-**All new code should confirm to the full pylint standards (including the additional `CONVENTION` (C) which covers documentation) at the time a PR is merged**
+**All new code should conform to the full pylint standards (including the additional `CONVENTION` (C) which covers documentation) at the time a PR is merged**
 
 In the meantime, we are trying to upgrade older code to meet these standards.

.github/workflows/codeql-analysis.yml

@@ -0,0 +1,71 @@
+# For most projects, this workflow file will not need changing; you simply need
+# to commit it to your repository.
+#
+# You may wish to alter this file to override the set of languages analyzed,
+# or to provide custom queries or build logic.
+#
+# ******** NOTE ********
+# We have attempted to detect the languages in your repository. Please check
+# the `language` matrix defined below to confirm you have the correct set of
+# supported CodeQL languages.
+#
+name: "CodeQL"
+
+on:
+  push:
+    branches: [ master ]
+  pull_request:
+    # The branches below must be a subset of the branches above
+    branches: [ master ]
+  schedule:
+    - cron: '17 17 * * 2'
+
+jobs:
+  analyze:
+    name: Analyze
+    runs-on: ubuntu-latest
+    permissions:
+      actions: read
+      contents: read
+      security-events: write
+
+    strategy:
+      fail-fast: false
+      matrix:
+        language: [ 'python' ]
+        # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python' ]
+        # Learn more:
+        # https://docs.github.com/en/free-pro-team@latest/github/finding-security-vulnerabilities-and-errors-in-your-code/configuring-code-scanning#changing-the-languages-that-are-analyzed
+
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v2
+
+    # Initializes the CodeQL tools for scanning.
+    - name: Initialize CodeQL
+      uses: github/codeql-action/init@v1
+      with:
+        languages: ${{ matrix.language }}
+        # If you wish to specify custom queries, you can do so here or in a config file.
+        # By default, queries listed here will override any specified in a config file.
+        # Prefix the list here with "+" to use these queries and those in the config file.
+        # queries: ./path/to/local/query, your-org/your-repo/queries@main
+
+    # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
+    # If this step fails, then you should remove it and run the build manually (see below)
+    - name: Autobuild
+      uses: github/codeql-action/autobuild@v1
+
+    # ℹ️ Command-line programs to run using the OS shell.
+    # 📚 https://git.io/JvXDl
+
+    # ✏️ If the Autobuild fails above, remove it and uncomment the following three lines
+    #    and modify them (or add more) to build your code if your project
+    #    uses a compiled language
+
+    #- run: |
+    #   make bootstrap
+    #   make release
+
+    - name: Perform CodeQL Analysis
+      uses: github/codeql-action/analyze@v1

.github/workflows/pylint.yml

@@ -20,4 +20,4 @@ jobs:
         poetry install
     - name: Analysing the code with pylint
       run: |
-        find . -name '*.py' -exec pylint {} \;
+         poetry run pylint winterdrp

.pre-commit-config.yaml

@@ -39,19 +39,6 @@ repos:
             "--exit-zero"
           ]
         verbose: true
-#-   repo: local
-#    hooks:
-#      - id: pylint
-#        name: pylint
-#        entry: pylint
-#        language: system
-#        types: [python]
-#        args:
-#          [
-##            "--exit-zero",
-#            "--fail-on=F,E"
-#          ]
-#        verbose: true
 -   repo: https://github.com/python-poetry/poetry
     rev: 1.2.2
     hooks:

docs/source/about.rst

@@ -2,3 +2,10 @@ About
 =====
 
 What is winterdrp? Good question!
+
+
+Data Structure
+--------------
+.. include:: ../../winterdrp/data/base_data.py
+    :start-line: 1
+    :end-line: 15

docs/source/index.rst

@@ -15,7 +15,6 @@ Contents
 --------
 
 .. toctree::
-    :maxdepth: 2
 
     about
     installation
@@ -24,8 +23,10 @@ Contents
     contributing-guide
     modules
 
+    winterdrp.data
+
 Indices and tables
 ==================
 
 * :ref:`genindex`
-* :ref:`modindex`
+* :ref:`modindex`

tests/test_errors.py

@@ -1,3 +1,5 @@
+"""Script for testing the error handling in ..module::winterdrp.errors
+"""
 import logging
 import unittest
 
@@ -20,14 +22,17 @@
 
 
 class TestErrors(unittest.TestCase):
+
+    """Class for testing errors in ..module::winterdrp.errors"""
+
     def setUp(self):
         self.logger = logging.getLogger(__name__)
         self.logger.setLevel(logging.INFO)
 
     def test_pipeline(self):
         self.logger.info("\n\n Testing summer pipeline \n\n")
 
-        res, errorstack = pipeline.reduce_images(
+        _, errorstack = pipeline.reduce_images(
             Dataset(ImageBatch()), catch_all_errors=True
         )
 

tests/test_monitor.py

@@ -1,6 +1,5 @@
-#!/usr/bin/env python
+"""Test suite for ..module::winterdrp.monitor module"""
 import logging
-import os
 import unittest
 
 from winterdrp.downloader.get_test_data import get_test_data_dir
@@ -19,12 +18,15 @@
 ]
 
 
-class TestErrors(unittest.TestCase):
+class TestMonitor(unittest.TestCase):
+    """Class for testing ..module::winterdrp.monitor"""
+
     def setUp(self):
         self.logger = logging.getLogger(__name__)
         self.logger.setLevel(logging.INFO)
 
-    def test_pipeline(self):
+    def test_monitor(self):
+        """Function to test ..class::Monitor realtime processing"""
         self.logger.info("Testing monitor")
 
         monitor = Monitor(

winterdrp/data/__init__.py

@@ -1,3 +1,5 @@
+"""Module to specify the input data classes for :module:`wintedrp.processors`
+"""
 from winterdrp.data.base_data import DataBatch, DataBlock, Dataset
 from winterdrp.data.image_data import Image, ImageBatch
 from winterdrp.data.source_data import SourceBatch, SourceTable

winterdrp/data/base_data.py

@@ -1,3 +1,19 @@
+"""
+This contains the base data classes for the :module:`wintedrp.processors`.
+
+The smallest unit is a :class:`~winterdrp.data.image_data.DataBlock` object,
+corresponding to a single image.
+These :class:`~winterdrp.data.image_data.DataBlock` objects are grouped into
+:class:`~winterdrp.data.image_data.DataBatch` objects.
+Each :class:`~wintedrp.processors.BaseProcessor` will operate on a individual
+:class:`~winterdrp.data.image_data.DataBatch` object.
+
+The :class:`~winterdrp.data.image_data.DataBatch` objects are stored within a larger
+:class:`~winterdrp.data.image_data.DataSet` object.
+A :class:`~wintedrp.processors.BaseProcessor` will iterate over each
+:class:`~winterdrp.data.image_data.DataBatch` in a
+:class:`~winterdrp.data.image_data.DataSet`.
+"""
 import logging
 from pathlib import Path
 from typing import Type
@@ -8,6 +24,8 @@
 
 
 class DataBlock:
+    """Base unit for processing, corresponding to a single image."""
+
     def __init__(self):
         self.raw_img_list = self[raw_img_key].split(",")
         self.base_name = self[base_name_key]
@@ -19,9 +37,20 @@ def __setitem__(self, key, value):
         raise NotImplementedError
 
     def get_name(self) -> str:
+        """Function to retrieve the :variable:`winterdrp.paths.base_name_key`
+        of the parent image
+
+        :return: Base name of parent image
+        """
         return self.base_name
 
     def get_raw_img_list(self) -> list[str]:
+        """Function to retrieve the paths of all raw images from
+        which this object is derived.
+        Because of stacking, this list may include multiple entries.
+
+        :return: List of path strings
+        """
         return self.raw_img_list
 
 
@@ -39,6 +68,10 @@ class PseudoList:
 
     @property
     def data_type(self):
+        """
+        Each list should take one specific data type.
+        This is where that type is defined.
+        """
         raise NotImplementedError()
 
     def __init__(self, data_list=None):
@@ -59,22 +92,45 @@ def __init__(self, data_list=None):
             self.append(item)
 
     def get_data_list(self):
+        """
+        Retrieve the data list
+
+        :return: The saved list of objects
+        """
         return self._datalist
 
     def append(self, item):
+        """
+        Function to append, list-style, new objects.
+
+        :param item: Object to be added
+        :return: None
+        """
         self._append(item)
 
     def _append(self, item):
+        """
+        Protected method to append, list-style, new objects.
+        This function also checks the data type to ensure they are correct.
+
+        :param item: Object to be added
+        :return: None
+        """
+
         if not isinstance(item, self.data_type):
-            err = f"Error appending item {item} of type {type(item)}. Expected a {self.data_type} item"
+            err = (
+                f"Error appending item {item} of type {type(item)}. "
+                f"Expected a {self.data_type} item"
+            )
             logger.error(err)
             raise ValueError(err)
 
         if len(self._datalist) > 0:
-            if not type(self._datalist[0]) == type(item):
+            if not isinstance(item, type(self._datalist[0])):
                 err = (
-                    f"Error appending item {item} of type {type(item)}. This {self.__class__.__name__} "
-                    f"object already contains data of type {type(self._datalist[0])}. "
+                    f"Error appending item {item} of type {type(item)}. "
+                    f"This {self.__class__.__name__} object already contains "
+                    f"data of type {type(self._datalist[0])}. "
                     f"Please ensure all data is of the same type."
                 )
                 logger.error(err)
@@ -108,29 +164,55 @@ def __iter__(self):
 
 
 class DataBatch(PseudoList):
+    """
+    Base class for a collection of individual
+    :class:`~winterdrp.data.image_data.DataBlock` objects.
+    Each :class:`~winterdrp.data.image_data.DataBatch` will be operated on
+    by a :class:`~wintedrp.processors.BaseProcessor`
+    """
+
     @property
     def data_type(self) -> Type[DataBlock]:
         raise NotImplementedError()
 
     def __init__(self, batch: list[DataBlock] | DataBlock = None):
-        super(DataBatch, self).__init__(data_list=batch)
+        super().__init__(data_list=batch)
 
     def get_batch(self) -> list[DataBlock]:
+        """Returns the :class:`~winterdrp.data.image_data.DataBlock`
+        items within the batch
+
+        :return: list of :class:`~winterdrp.data.image_data.DataBlock` objects
+        """
         return self.get_data_list()
 
-    def get_raw_image_names(self) -> list[str]:
+    def get_raw_image_names(self) -> list[Path]:
+        """Returns the name of each parent raw image
+
+        :return: list of raw image names
+        """
         img_list = []
         for data_block in self.get_batch():
             img_list += [Path(x).name for x in data_block.get_raw_img_list()]
         return img_list
 
 
 class Dataset(PseudoList):
+    """
+    Base class for a collection of individual
+    :class:`~winterdrp.data.image_data.DataBatch` objects.
+    A :class:`~wintedrp.processors.BaseProcessor` will iterate over these.
+    """
 
     data_type = DataBatch
 
     def get_batches(self):
+        """Returns the :class:`~winterdrp.data.image_data.DataBatch`
+        items within the batch
+
+        :return: list of :class:`~winterdrp.data.image_data.DataBatch` objects
+        """
         return self.get_data_list()
 
     def __init__(self, batches: list[DataBatch] | DataBatch = None):
-        super(Dataset, self).__init__(data_list=batches)
+        super().__init__(data_list=batches)

winterdrp/data/image_data.py

@@ -1,3 +1,8 @@
+"""
+
+Module to specify the input data classes for :class:`winterdrp.processors.base_processor.ImageHandler`
+
+"""
 import logging
 
 import numpy as np
@@ -75,8 +80,8 @@ class ImageBatch(DataBatch):
     def __init__(self, batch: list[Image] | Image = None):
         super().__init__(batch=batch)
 
-    def append(self, data: Image):
-        self._append(data)
+    def append(self, item: Image):
+        self._append(item)
 
     def __str__(self):
         return f"<An {self.__class__.__name__} object, containing {[x.get_name() for x in self.get_batch()]}>"