Data Objects

.gitignore

@@ -1,2 +1,21 @@
 winterdrp/processors/astromatic/config/sex.config
-.env
+.env
+__pycache__/
+*.pyc
+.DS_Store
+.idea/
+
+# Distribution / packaging
+bin/
+build/
+develop-eggs/
+dist/
+eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+*.egg-info/
+.installed.cfg
+*.egg

tests/test_errors.py

@@ -2,7 +2,7 @@
 import logging
 from winterdrp.pipelines import get_pipeline
 from winterdrp.errors import ImageNotFoundError
-from datetime import datetime
+from winterdrp.data import ImageBatch, Dataset
 
 logger = logging.getLogger(__name__)
 
@@ -14,7 +14,7 @@
 
 expected_error = {
     'processor_name': 'winterdrp.processors.utils.image_loader',
-    'contents': [''],
+    'contents': [],
     'known_error_bool': True,
     'non_critical_bool': False
 }
@@ -28,11 +28,11 @@ def setUp(self):
     def test_pipeline(self):
         self.logger.info("\n\n Testing summer pipeline \n\n")
 
-        res, errorstack = pipeline.reduce_images([[[], []]], catch_all_errors=True)
+        res, errorstack = pipeline.reduce_images(Dataset(ImageBatch()), catch_all_errors=True)
 
         errorstack.summarise_error_stack(verbose=True)
 
-        self.assertEqual(len(errorstack.failed_images), 1)
+        self.assertEqual(len(errorstack.failed_images), 0)
         self.assertEqual(len(errorstack.noncritical_reports), 0)
         self.assertEqual(len(errorstack.reports), 1)
 

tests/test_monitor.py

@@ -8,8 +8,6 @@
 
 logger = logging.getLogger(__name__)
 
-logging.getLogger("winterdrp").setLevel("DEBUG")
-
 summer_cal_requirements = [
     CalRequirement(target_name="bias", required_field="EXPTIME", required_values=["0.0"]),
     CalRequirement(target_name="flat", required_field="FILTERID", required_values=["r"]),

tests/test_summer_imsub_pipeline.py

@@ -1,16 +1,10 @@
 import unittest
 import logging
-from winterdrp.processors.utils.image_loader import ImageLoader
-from winterdrp.processors.utils.image_selector import ImageSelector
-from winterdrp.paths import base_name_key
 from winterdrp.pipelines.summer.summer_pipeline import SummerPipeline
-from winterdrp.downloader.get_test_data import get_test_data_dir
-from winterdrp.pipelines.summer.blocks import subtract
-from winterdrp.pipelines.summer.load_summer_image import load_proc_summer_image
+from winterdrp.data import Dataset, ImageBatch
 
 logger = logging.getLogger(__name__)
 
-
 expected_values = {
     'SCORSTD': 1.120988782614284,
     'SCORMED': 0.0010565268947477073,
@@ -28,11 +22,11 @@ def setUp(self):
     def test_pipeline(self):
         self.logger.info("\n\n Testing summer pipeline \n\n")
 
-        res, errorstack = pipeline.reduce_images([[[], []]], catch_all_errors=False)
+        res, errorstack = pipeline.reduce_images(Dataset([ImageBatch()]), catch_all_errors=False)
 
-        self.assertEqual(len(res[0][0]), 1)
+        self.assertEqual(len(res), 1)
 
-        header = res[0][1][0]
+        header = res[0][0].get_header()
 
         for key, value in expected_values.items():
             if isinstance(value, float):
@@ -42,22 +36,3 @@ def test_pipeline(self):
             else:
                 raise TypeError(f"Type for value ({type(value)} is neither float not int.")
 
-
-if __name__ == "__main__":
-
-    print("Calculating latest scorr metrics dictionary")
-
-    # Code to generate updated ZP dict of the results change
-
-    new_res, new_errorstack = pipeline.reduce_images([[[], []]], catch_all_errors=False)
-
-    new_header = new_res[0][1][0]
-
-    new_exp = "expected_values = { \n"
-    for header_key in new_header.keys():
-        if "SCOR" in header_key:
-            new_exp += f'    "{header_key}": {new_header[header_key]}, \n'
-    new_exp += "}"
-    print(new_exp)
-
-

tests/test_summer_pipeline.py

@@ -1,6 +1,7 @@
 import unittest
 import logging
 from winterdrp.pipelines import get_pipeline
+from winterdrp.data import Dataset, ImageBatch
 
 logger = logging.getLogger(__name__)
 
@@ -45,11 +46,11 @@ def setUp(self):
     def test_pipeline(self):
         self.logger.info("\n\n Testing summer pipeline \n\n")
 
-        res, errorstack = pipeline.reduce_images([[[], []]], catch_all_errors=False)
+        res, errorstack = pipeline.reduce_images(Dataset([ImageBatch()]), catch_all_errors=False)
 
-        self.assertEqual(len(res[0][0]), 1)
+        self.assertEqual(len(res[0]), 1)
 
-        header = res[0][1][0]
+        header = res[0][0].get_header()
 
         for key, value in expected_zp.items():
             if isinstance(value, float):
@@ -58,23 +59,3 @@ def test_pipeline(self):
                 self.assertEqual(value, header[key])
             else:
                 raise TypeError(f"Type for value ({type(value)} is neither float not int.")
-
-
-if __name__ == "__main__":
-
-    print("Calculating latest ZP dictionary")
-
-    # Code to generate updated ZP dict of the results change
-
-    new_res, new_errorstack = pipeline.reduce_images([[[], []]], catch_all_errors=False)
-
-    new_header = new_res[0][1][0]
-
-    new_exp = "expected_zp = { \n"
-    for header_key in new_header.keys():
-        if "ZP_" in header_key:
-            new_exp += f'    "{header_key}": {new_header[header_key]}, \n'
-    new_exp += "}"
-    print(new_exp)
-
-

tests/test_wirc_imsub_pipeline.py

@@ -1,10 +1,8 @@
 import unittest
 from winterdrp.processors.utils.image_loader import ImageLoader
-from winterdrp.processors.utils.image_selector import ImageSelector, ImageBatcher, ImageDebatcher
 import logging
 import os
 from winterdrp.downloader.get_test_data import get_test_data_dir
-from winterdrp.paths import base_name_key
 from winterdrp.pipelines.wirc.load_wirc_image import load_raw_wirc_image
 from winterdrp.pipelines.wirc.wirc_pipeline import WircPipeline
 from astropy.io import fits
@@ -13,13 +11,14 @@
 from winterdrp.pipelines.wirc.blocks import subtract
 from winterdrp.pipelines.wirc.generator import wirc_reference_image_resampler, wirc_reference_sextractor, \
     wirc_reference_psfex
+from winterdrp.data import Dataset, ImageBatch
+
 
 logger = logging.getLogger(__name__)
 
 test_data_dir = get_test_data_dir()
 
 ref_img_directory = os.path.join(test_data_dir, 'wirc/ref')
-print(ref_img_directory)
 
 
 def test_reference_image_generator(
@@ -49,8 +48,8 @@ def test_reference_image_generator(
     ),
     Reference(
         ref_image_generator=test_reference_image_generator,
-        ref_swarp_resampler=wirc_reference_image_resampler,
-        ref_sextractor=wirc_reference_sextractor,
+        swarp_resampler=wirc_reference_image_resampler,
+        sextractor=wirc_reference_sextractor,
         ref_psfex=wirc_reference_psfex
     ),
 ] + subtract
@@ -68,11 +67,11 @@ def setUp(self):
     def test_pipeline(self):
         self.logger.info("\n\n Testing wirc imsub pipeline \n\n")
 
-        res, errorstack = pipeline.reduce_images([[[], []]], catch_all_errors=False)
+        res, errorstack = pipeline.reduce_images(dataset=Dataset(ImageBatch()), catch_all_errors=False)
 
         self.assertEqual(len(res), 1)
 
-        header = res[0][1][0]
+        header = res[0][0].get_header()
 
         for key, value in expected_values.items():
             if isinstance(value, float):
@@ -89,7 +88,7 @@ def test_pipeline(self):
 
     # Code to generate updated ZP dict of the results change
 
-    new_res, new_errorstack = pipeline.reduce_images([[[], []]], catch_all_errors=False)
+    new_res, new_errorstack = pipeline.reduce_images(catch_all_errors=False)
 
     new_header = new_res[0][1][0]
 

tests/test_wirc_pipeline.py

@@ -18,6 +18,7 @@
 from winterdrp.pipelines.wirc.generator import wirc_astrometric_catalog_generator, wirc_photometric_catalog_generator
 from winterdrp.processors.csvlog import CSVLog
 from winterdrp.downloader.get_test_data import get_test_data_dir
+from winterdrp.data import Dataset, ImageBatch
 
 logger = logging.getLogger(__name__)
 
@@ -101,11 +102,11 @@ def setUp(self):
     def test_pipeline(self):
         self.logger.info("\n\n Testing wirc pipeline \n\n")
 
-        res, errorstack = pipeline.reduce_images([[[], []]], catch_all_errors=False)
+        res, errorstack = pipeline.reduce_images(Dataset([ImageBatch()]), catch_all_errors=False)
 
         self.assertEqual(len(res), 1)
 
-        header = res[0][1][0]
+        header = res[0][0].get_header()
 
         for key, value in expected_zp.items():
             if isinstance(value, float):
@@ -115,21 +116,3 @@ def test_pipeline(self):
             else:
                 raise TypeError(f"Type for value ({type(value)} is neither float not int.")
 
-
-if __name__ == "__main__":
-
-    print("Calculating latest ZP dictionary")
-
-    # Code to generate updated ZP dict of the results change
-
-    new_res, new_errorstack = pipeline.reduce_images([[[], []]], catch_all_errors=False)
-
-    new_header = new_res[0][1][0]
-
-    new_exp = "expected_zp = { \n"
-    for header_key in new_header.keys():
-        if "ZP_" in header_key:
-            new_exp += f'    "{header_key}": {new_header[header_key]}, \n'
-    new_exp += "}"
-    print(new_exp)
-

winterdrp/__main__.py

@@ -8,8 +8,8 @@
 from astropy.time import Time
 from astropy import units as u
 from winterdrp.monitor.base_monitor import Monitor
-from winterdrp.paths import base_raw_dir
 from datetime import datetime
+from winterdrp.data import Dataset, ImageBatch
 from winterdrp.processors.utils import ImageLoader
 
 logger = logging.getLogger(__name__)
@@ -167,7 +167,7 @@
         night=night,
     )
 
-    batches, errorstack = pipe.reduce_images([[[], []]], catch_all_errors=True)
+    batches, errorstack = pipe.reduce_images(Dataset([ImageBatch()]), catch_all_errors=True)
     if args.postprocessconfig is not None:
         post_config = [x for x in pipe.set_configuration(config) if isinstance(x, ImageLoader)][:1]
         post_config += pipe.postprocess_configuration(
@@ -181,7 +181,7 @@
         pipe.set_configuration(protected_key)
 
         _, new_errorstack = pipe.reduce_images(
-            batches=[[[], []]],
+            Dataset([ImageBatch()]),
             selected_configurations=protected_key,
             catch_all_errors=True
         )

winterdrp/data/__init__.py

@@ -0,0 +1,3 @@
+from winterdrp.data.base_data import DataBlock, DataBatch, Dataset
+from winterdrp.data.image_data import Image, ImageBatch
+from winterdrp.data.source_data import SourceTable, SourceBatch

winterdrp/data/base_data.py

@@ -0,0 +1,90 @@
+
+class DataBlock:
+
+    def __getitem__(self, item):
+        raise NotImplementedError
+
+    def __setitem__(self, key, value):
+        raise NotImplementedError
+
+
+class DataBatch:
+
+    def __init__(self, batch: list[DataBlock] | DataBlock = None):
+
+        if batch is None:
+            batch = []
+        elif isinstance(batch, DataBlock):
+            batch = [batch]
+
+        self._batch = batch
+
+    def get_batch(self) -> list[DataBlock]:
+        raise NotImplementedError
+
+    def append(self, item):
+        self._batch.append(item)
+
+    def __getitem__(self, item):
+        return self._batch.__getitem__(item)
+
+    def __setitem__(self, key, value):
+        return self._batch.__setitem__(key, value)
+
+    def __add__(self, other):
+        return DataBatch(self._batch + other.get_batch())
+
+    def __iadd__(self, other):
+        self._batch += other.get_batch()
+        return self
+
+    def __len__(self):
+        return self._batch.__len__()
+
+    def __iter__(self):
+        return self._batch.__iter__()
+
+
+class Dataset:
+
+    def __init__(
+            self,
+            batches: list[DataBatch] | DataBatch = None
+    ):
+
+        if batches is None:
+            batches = []
+        elif isinstance(batches, DataBatch):
+            batches = [batches]
+
+        self._batches = batches
+
+    def get_batches(self):
+        return self._batches
+
+    def append(self, batch: DataBatch):
+
+        if len(self._batches) > 0:
+            assert type(self._batches[0]) == type(batch)
+
+        self._batches.append(batch)
+
+    def __getitem__(self, item):
+        return self._batches.__getitem__(item)
+
+    def __setitem__(self, key, value):
+        return self._batches.__setitem__(key, value)
+
+    def __len__(self):
+        return self._batches.__len__()
+
+    def __add__(self, other):
+        return DataBatch(self._batches + other.get_batches())
+
+    def __iadd__(self, other):
+        self._batches += other.get_batches()
+        return self
+
+    def __iter__(self):
+        return self._batches.__iter__()
+