Merge pull request #145 from AstarVienna/144-run-notebook-tests-in-pa…

…rallel

Run notebook tests in parallel

.github/workflows/tests.yml

@@ -13,15 +13,16 @@ on:
   workflow_dispatch:
 
 jobs:
-  job_master:
+  tests_master:
+    name: Test against ScopeSim master
     runs-on: ${{ matrix.os }}
     # Run if our target is IRDB master, or when this is ran manually.
     if: github.base_ref == 'master' || github.base_ref == ''
     strategy:
       fail-fast: false
       matrix:
         os: [ubuntu-latest]
-        python-version: ['3.8', '3.9', '3.10', '3.11']
+        python-version: ['3.8', '3.9', '3.10', '3.11', '3.12']
 
     steps:
       - uses: actions/checkout@v3
@@ -38,16 +39,22 @@ jobs:
           pip install git+https://github.com/AstarVienna/ScopeSim_Templates.git
       - name: Run Pytest
         run: pytest
+      - name: Store badge report files
+        uses: actions/upload-artifact@v3
+        with:
+          name: badge-report
+          path: _REPORTS
 
-  job_devmaster:
+  tests_devmaster:
+    name: Test against ScopeSim dev_master
     runs-on: ${{ matrix.os }}
     # Run if our target is IRDB dev_master, or when this is ran manually.
     if: github.base_ref == 'dev_master' || github.base_ref == ''
     strategy:
       fail-fast: false
       matrix:
         os: [ubuntu-latest]
-        python-version: ['3.8', '3.9', '3.10', '3.11']
+        python-version: ['3.8', '3.9', '3.10', '3.11', '3.12']
 
     steps:
       - uses: actions/checkout@v3
@@ -65,7 +72,8 @@ jobs:
       - name: Run Pytest
         run: pytest
 
-  job_notebooks:
+  micado_notebooks:
+    name: Run MICADO Notebooks
     # Run the notebooks. This is a time-consuming step, but important to catch
     # errors. It is therefore only ran on the master and dev_master branches,
     # or when manually triggered. It is not ran on pull requests.
@@ -91,12 +99,63 @@ jobs:
           pip uninstall -y scopesim scopesim_templates
           pip install git+https://github.com/AstarVienna/ScopeSim.git@dev_master
           pip install git+https://github.com/AstarVienna/ScopeSim_Templates.git
-      - name: Run METIS Notebooks
-        run: |
-          for fn in METIS/docs/example_notebooks/*.ipynb; do echo "${fn}"; /usr/bin/time -v jupytext --execute --update "${fn}"; done
-          for fn in METIS/docs/example_notebooks/demos/*.ipynb; do echo "${fn}"; /usr/bin/time -v jupytext --execute --update "${fn}"; done
-      - name: Run MICADO Notebooks
+      - name: Run Notebooks
+        env:
+          PYDEVD_DISABLE_FILE_VALIDATION: 1
         run: |
+          echo "## MICADO Notebooks tested" >> $GITHUB_STEP_SUMMARY
           sed -i "s/USE_FULL_DETECTOR = True/USE_FULL_DETECTOR = False/g" MICADO/docs/example_notebooks/*.ipynb
           sed -i 's/# cmd\[\\"!SIM.spectral.spectral_bin_width/cmd\[\\"!SIM.spectral.spectral_bin_width/g'   MICADO/docs/example_notebooks/*.ipynb
-          for fn in MICADO/docs/example_notebooks/*.ipynb; do echo "${fn}"; /usr/bin/time -v jupytext --execute --update "${fn}"; done
+          for fn in MICADO/docs/example_notebooks/*.ipynb
+          do 
+            echo "${fn}"
+            echo "- ${fn}" >> $GITHUB_STEP_SUMMARY
+            /usr/bin/time -v jupytext --execute --update "${fn}"
+          done
+
+  metis_notebooks:
+    name: Run METIS Notebooks
+    # Run the notebooks. This is a time-consuming step, but important to catch
+    # errors. It is therefore only ran on the master and dev_master branches,
+    # or when manually triggered. It is not ran on pull requests.
+    runs-on: ubuntu-latest
+    # Run if we are on the master or dev_master branch, or if this job permissions:
+    # started manually.
+    # TODO: Do something smarter than github.base_ref == '', see
+    # https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows
+    if: github.ref == 'dev_master' || github.ref == 'master' || github.base_ref == ''
+    steps:
+      - uses: actions/checkout@v3
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          # No matrix is used since this is a time-consuming tosk.
+          python-version: 3.11
+      - name: Install dependencies
+        # TODO: Also test against ScopeSIM@master?
+        # TODO: And/or ScopeSIM_Templates@dev_master?
+        run: |
+          python -m pip install --upgrade pip
+          pip install -r requirements.github_actions.txt
+          pip uninstall -y scopesim scopesim_templates
+          pip install git+https://github.com/AstarVienna/ScopeSim.git@dev_master
+          pip install git+https://github.com/AstarVienna/ScopeSim_Templates.git
+      - name: Run Notebooks
+        env:
+          PYDEVD_DISABLE_FILE_VALIDATION: 1
+        run: |
+          echo "## METIS Notebooks tested" >> $GITHUB_STEP_SUMMARY
+          echo "### Example Notebooks" >> $GITHUB_STEP_SUMMARY
+          for fn in METIS/docs/example_notebooks/*.ipynb
+          do
+            echo "${fn}"
+            echo "- ${fn}" >> $GITHUB_STEP_SUMMARY
+            /usr/bin/time -v jupytext --execute --update "${fn}"
+          done
+          echo "## Demo Notebooks" >> $GITHUB_STEP_SUMMARY
+          for fn in METIS/docs/example_notebooks/demos/*.ipynb
+          do
+            echo "${fn}"
+            echo "- ${fn}" >> $GITHUB_STEP_SUMMARY
+            /usr/bin/time -v jupytext --execute --update "${fn}"
+          done

LFOA/tests/test_lfoa.py

@@ -47,6 +47,7 @@ def test_something_comes_out(self):
 
             plt.show()
 
+    @pytest.mark.slow
     def test_observes_from_scopesim_templates(self):
         src = sim_tp.stellar.cluster(mass=10000, distance=2000, core_radius=1)
 
@@ -64,6 +65,7 @@ def test_observes_from_scopesim_templates(self):
             plt.colorbar()
             plt.show()
 
+    @pytest.mark.slow
     def test_saves_readout_to_disc(self):
         src = sim_tp.stellar.cluster(mass=10000, distance=2000, core_radius=1)
         lfoa = scopesim.OpticalTrain("LFOA")

LFOA/tests/test_source_shift.py

@@ -52,6 +52,7 @@
 #         shutil.rmtree(rc_local_path)
 
 
+@pytest.mark.slow
 class TestShiftSource:
     def test_shift_lfao(self):
         # core_radius = 0.6 to ensure it fits the image after shifting

METIS/tests/test_metis.py

@@ -222,6 +222,7 @@ def test_tracelist_has_traces(self, trace_list):
                               scopesim.effects.SpectralTrace)
 
 
+@pytest.mark.slow
 class TestObserves:
     """Test basic observations for the main instrument modes"""
     def test_something_comes_out_img_lm(self):

OSIRIS/tests/test_compiles.py

@@ -1,4 +1,5 @@
 from pathlib import Path
+import pytest
 
 import numpy as np
 import scopesim as sim
@@ -38,6 +39,7 @@ def test_system_throughput_is_not_zero(self):
 
         assert trans.sum() > 0
 
+    @pytest.mark.slow
     def test_run_imaging_simulation(self):
         # n stars, mag_min, mag_max, width=[arcsec]
         src = star_field(n=1000, mmin=10, mmax=20, width=450, use_grid=False)
@@ -60,6 +62,7 @@ def test_everything_is_read_in_nicely(self):
         assert isinstance(cmds, sim.UserCommands)
         assert len(cmds.yaml_dicts) > 2
 
+    @pytest.mark.slow
     def test_run_lss_simulation(self):
         # n stars, mag_min, mag_max, width=[arcsec]
         src1 = star(x=-3.8, y=-3.8, flux=10*u.mag)
@@ -94,6 +97,7 @@ def test_everything_is_read_in_nicely(self):
         assert isinstance(cmds, sim.UserCommands)
         assert len(cmds.yaml_dicts) > 2
 
+    @pytest.mark.slow
     def test_run_maat_simulation(self):
         # n stars, mag_min, mag_max, width=[arcsec]
         src = star_field(5**2, 10, 10, 8, use_grid=True)

OSIRIS/tests/test_maat_daves_tests.py

@@ -1,4 +1,5 @@
 from pathlib import Path
+import pytest
 
 import scopesim as sim
 from astropy import units as u
@@ -15,6 +16,7 @@
 sim.rc.__config__["!SIM.file.local_packages_path"] = str(PATH_HERE.parent.parent)
 
 
+@pytest.mark.slow
 def test_maat_runs_with_point_source():
     g191 = SourceSpectrum.from_file(str(PATH_HERE / 'test_data' / 'fg191b2b.dat'))
     src = point_source(sed=g191, filter_curve='V',
@@ -36,6 +38,7 @@ def test_maat_runs_with_point_source():
         plt.show()
 
 
+@pytest.mark.slow
 def test_maat_runs_with_extended_source():
     src = galaxy("kc96/s0", z=0.1, amplitude=12*u.mag, filter_curve="V",
                  pixel_scale=0.1, r_eff=3.5, n=2, ellip=0.3, theta=45, extend=5)
@@ -56,6 +59,7 @@ def test_maat_runs_with_extended_source():
         plt.show()
 
 
+@pytest.mark.slow
 def test_maat_runs_with_line_list_source():
     arcspec = SourceSpectrum.from_file(str(PATH_HERE / 'test_data' / 'OSIRIS_stitchedArc.dat'))
     arc = uniform_source(sed=arcspec, filter_curve='V', amplitude=16*u.ABmag, extend=520)

OSIRIS/tests/test_maat_traces.py

@@ -1,4 +1,5 @@
 from pathlib import Path
+import pytest
 
 import numpy as np
 import scopesim as sim
@@ -15,6 +16,7 @@
 PLOTS = False
 
 
+@pytest.mark.slow
 class TestMaatTraces:
     def test_plot_traces(self):
         """
@@ -64,6 +66,7 @@ def test_plot_traces(self):
             plt.show()
 
 
+@pytest.mark.slow
 class TestMaatOperations:
     def test_basic_trace_image(self):
 

ViennaLT/tests/test_lvilt.py

@@ -22,6 +22,7 @@ def test_scopesim_loads_package(self):
         assert isinstance(lfoa, scopesim.OpticalTrain)
 
 
+@pytest.mark.slow
 class TestObserves:
     def test_something_comes_out(self):
         src = star_field(100, 0, 10, width=360, use_grid=True)

irdb/badges.py

@@ -222,14 +222,15 @@ def __init__(self,
 
     def __enter__(self):
         logging.debug("REPORT ENTER")
-        try:
-            # TODO: WHY do we actually load this first? It caused some issues
-            #       with 'old' badges that are not cleared. Is there any good
-            #       reason at all to load the previous yaml file???
-            with self.yamlpath.open(encoding="utf-8") as file:
-                self.update(yaml.full_load(file))
-        except FileNotFoundError:
-            logging.warning("%s not found, init empty dict", self.yamlpath)
+        # try:
+        #     # TODO: WHY do we actually load this first? It caused some issues
+        #     #       with 'old' badges that are not cleared. Is there any good
+        #     #       reason at all to load the previous yaml file???
+        #     with self.yamlpath.open(encoding="utf-8") as file:
+        #         self.update(yaml.full_load(file))
+        # except FileNotFoundError:
+        #     logging.warning("%s not found, init empty dict", self.yamlpath)
+        logging.debug("Init emtpy dict.")
         return self
 
     def __exit__(self, exc_type, exc_value, exc_traceback):

irdb/tests/test_publish.py

@@ -105,7 +105,7 @@ def test_missing_called(self):
         with mock.patch("irdb.publish._handle_missing_folder",
                         mock_obj) as mock_missing:
             assert pub._get_server_path("bogus", "foo") == "bar/foo"
-            assert mock_missing.called_once_with("bogus")
+            mock_missing.assert_called_once_with("bogus")
 
 
 def _fake_input(prompt: str, from_user: str) -> str:
@@ -144,6 +144,7 @@ def fixture_default_argv():
     return ["", "-l", "fake_username", "-p", "fake_password", "test_package"]
 
 
+@pytest.mark.webtest
 @pytest.mark.usefixtures("default_argv", "temp_zipfiles")
 @pytest.mark.parametrize("argv, called, response", [
     (["-u"], False, None),

pytest.ini

@@ -1,3 +1,7 @@
 [pytest]
 # Prevent recursion into MICADO/docs/example_notebooks/inst_pkgs
-addopts = --ignore-glob="*/inst_pkgs/*"
+addopts = --ignore-glob="*/inst_pkgs/*"  -p no:randomly
+# Badge report needs order (at least for now, should be solved by using astar-utils NestedMapping)
+markers =
+    webtest: mark a test as using network resources.
+    slow: mark test as slow.