Merge branch 'integration_times' of github.com:juanep97/iop4 into int…

…egration_times

.github/workflows/ci.yml

@@ -33,14 +33,14 @@ jobs:
       - name: Set up Python ${{ matrix.python-version }}
         uses: actions/setup-python@v4
         with:
-          python-version: "3.10.9"
+          python-version: "3.11"
 
   tests:
     runs-on: [self-hosted, x64, Linux, ubuntu, generic]
     strategy:
       max-parallel: 1
       matrix:
-        python-version: ["3.10.9", "3.11"]
+        python-version: ["3.11", "3.12"]
     steps:
       - uses: actions/checkout@v4
         with:
@@ -55,6 +55,10 @@ jobs:
         run: |
           python -m pip install --upgrade pip
 
+      - name: Install required packages
+        run: |
+          sudo apt-get -y install gcc
+
       - name: Build the package
         run: |
           pip install build
@@ -94,7 +98,7 @@ jobs:
     strategy:
       max-parallel: 1
       matrix:
-        python-version: ["3.10.9", "3.11"]
+        python-version: ["3.11", "3.12"]
     steps:
       - uses: actions/checkout@v4
         with:
@@ -111,7 +115,7 @@ jobs:
 
       - name: Install required packages
         run: |
-          sudo apt-get -y install pandoc make
+          sudo apt-get -y install pandoc make gcc
 
       - name: Install doc dependencies
         run: |

.github/workflows/deploy_pypi.yml

@@ -16,7 +16,7 @@ jobs:
       - name: Set up Python
         uses: actions/setup-python@v4
         with:
-          python-version: 3.10.9
+          python-version: 3.11
 
       - name: Install dependencies
         run: |

.github/workflows/docs.yml

@@ -39,15 +39,15 @@ jobs:
       - name: Set up Python
         uses: actions/setup-python@v4
         with:
-          python-version: "3.10.9"
+          python-version: "3.11"
 
       - name: Update pip
         run: |
           python -m pip install --upgrade pip
 
       - name: Install required packages
         run: |
-          sudo apt-get -y install pandoc make
+          sudo apt-get -y install pandoc make gcc
 
       - name: Install doc dependencies
         run: |

README.md

@@ -18,18 +18,18 @@ We recommend installing IOP4 in an isolated environment as described below. IOP4
 
 ### Option 1: Using a virtual environment
 
-**Note:** IOP4 requires Python 3.10 or later. You can check your Python version with `python --version`. If you have a compatible version, you can skip this step.
+**Note:** IOP4 requires Python 3.11 or later. You can check your Python version with `python --version`. If you have a compatible version, you can skip this step.
 
-If you don't have Python 3.10 or later, you can install [pyenv](https://github.com/pyenv/pyenv) and [pyenv-virtualenv](https://github.com/pyenv/pyenv-virtualenv), which will manage python versions for you. You can use the automatic installer [pyenv-installer](https://github.com/pyenv/pyenv-installer):
+If you don't have Python 3.11 or later, you can install [pyenv](https://github.com/pyenv/pyenv) and [pyenv-virtualenv](https://github.com/pyenv/pyenv-virtualenv), which will manage python versions for you. You can use the automatic installer [pyenv-installer](https://github.com/pyenv/pyenv-installer):
 
 ```bash
     $ curl https://pyenv.run | bash
 ```
 
 Follow the instructions that this command outputs to add `pyenv` to `PATH` (or copy the commands from https://github.com/pyenv/pyenv for your shell). Restart your terminal, or source the file (e.g. `. ~/.bashrc` or `. ~/.zshrc`) Then, run 
 ```bash 
-    $ pyenv install 3.10
-    $ pyenv virtualenv 3.10 iop4-venv
+    $ pyenv install 3.11
+    $ pyenv virtualenv 3.11 iop4-venv
     $ pyenv activate iop4-venv
 ```
 Now you will have a virtual environment with the right Python version, and you can continue with the next step. To deactivate, just run `pyenv deactivate`.
@@ -51,7 +51,7 @@ or `pip install -e .` if you want to install it in developer mode.
 ### Option 2: Using conda/mamba
 As the previous option, create and activate the environment as follows:
 ```bash
-    $ conda create -n iop4 python=3.10
+    $ conda create -n iop4 python=3.11
     $ conda activate iop4
 ```
 

iop4admin/modeladmins/astrosource.py

@@ -3,7 +3,7 @@
 iop4conf = iop4lib.Config(config_db=False)
 
 # django imports
-from django.contrib import admin
+from django.contrib import admin, messages
 from django.utils.html import format_html
 from django.urls import path, reverse 
 from django.db.models import Avg
@@ -12,6 +12,10 @@
 from iop4api.models import AstroSource, ReducedFit
 from iop4admin import views
 
+import numpy as np
+from astroquery.mast import Catalogs
+from astropy.coordinates import SkyCoord
+
 # iop4lib imports
 from iop4lib.enums import BANDS
 
@@ -27,6 +31,7 @@ class AdminAstroSource(admin.ModelAdmin):
                     'get_comment_firstline', 'get_details']
     search_fields = ['name', 'other_names', 'ra_hms', 'dec_dms', 'srctype', 'comment']
     list_filter = ('srctype',)
+    actions = ['add_field_stars_from_panstarrs', 'remove_field_stars_from_panstarrs']
 
     @admin.display(description='CALIBRATES')
     def get_calibrates(self, obj):
@@ -110,3 +115,129 @@ def get_texp_dipol(self, obj):
 
         return f"{texp} x {nreps}"
 
+    @admin.action(description='Automatically add field stars from PanSTARRS')
+    def add_field_stars_from_panstarrs(self, request, queryset):
+
+        if queryset.count() > 1:
+            messages.error(request, "Only one source at a time, please.")
+            return
+
+        main_src = queryset.first()
+
+        # quality constraints
+        MIN_R_MAG = 11 # minimum R magnitude
+        MAX_R_MAG = 16  # maximum R magnitude
+        MAX_MAG_STD = 0.01 # maximum standard deviation in the required bands (we want our calibrators to be stable)
+        MIN_N_OBS = 5 # minimum number of observations in the required bands
+
+        logger.info(f"Querying PanSTARRS around {main_src.name} ({main_src.coord.ra.deg} {main_src.coord.dec.deg})")
+
+        try:
+            catalog_data = Catalogs.query_criteria(coordinates=f"{main_src.coord.ra.deg} {main_src.coord.dec.deg}", catalog="PANSTARRS", version=1, radius="0.12 deg")
+        except Exception as e:
+            logger.error(f"Error querying PanSTARRS around {main_src.name} ({main_src.coord.ra.deg} {main_src.coord.dec.deg}): {e}")
+            messages.error(request, f"Error querying PanSTARRS around {main_src.name}: {e}")
+            return
+
+        logger.info(f"Got {len(catalog_data)} PanSTARRS field stars around {main_src.name}")
+
+
+        column_names = ['objName', 'raMean', 'decMean',
+                        'rMeanApMag', 'rMeanApMagErr', 'rMeanApMagStd', 'rMeanApMagNpt',
+                        'gMeanApMag', 'gMeanApMagErr', 'gMeanApMagStd', 'gMeanApMagNpt',
+                        'iMeanApMag', 'iMeanApMagErr', 'iMeanApMagStd', 'iMeanApMagNpt',
+                        'yMeanApMag', 'yMeanApMagErr', 'yMeanApMagStd', 'yMeanApMagNpt']
+
+        for col in column_names:
+
+            idx = np.isfinite(catalog_data[col])
+            idx = idx.data & idx.mask
+            catalog_data = catalog_data[~idx]
+
+        idx = (MIN_R_MAG <= catalog_data["rMeanApMag"]) & (catalog_data["rMeanApMag"] <= MAX_R_MAG)
+        idx = idx & (catalog_data["rMeanApMagStd"] < MAX_MAG_STD)
+        idx = idx & (catalog_data["rMeanApMagNpt"] > MIN_N_OBS)
+        idx = idx & (catalog_data["gMeanApMagStd"] < MAX_MAG_STD)
+        idx = idx & (catalog_data["gMeanApMagNpt"] > MIN_N_OBS)
+        idx = idx & (catalog_data["iMeanApMagStd"] < MAX_MAG_STD)
+        idx = idx & (catalog_data["iMeanApMagNpt"] > MIN_N_OBS)
+        idx = idx & (catalog_data["yMeanApMagStd"] < MAX_MAG_STD)
+        idx = idx & (catalog_data["yMeanApMagNpt"] > MIN_N_OBS)
+        catalog_data = catalog_data[idx]
+
+        catalog_data.remove_columns([col for col in catalog_data.columns if col not in column_names])
+
+        logger.info(f"Filtered down to {len(catalog_data)} PanSTARRS field stars with {MIN_R_MAG} <= R <= {MAX_R_MAG}, std < {MAX_MAG_STD} and n_obs > {MIN_N_OBS}")
+
+        if len(catalog_data) == 0:
+            logger.error(f"No PanSTARRS field stars found for {main_src.name}, skipping")
+
+        # sort by number of observations in R, take top 10 only
+        if len(catalog_data) > 10:
+            logger.info("Keeping only top 10 field stars by number of R observations")
+            catalog_data.sort('rMeanApMagNpt')
+            catalog_data = catalog_data[-10:]
+
+        field_stars = list()
+
+        for row in catalog_data:
+            try:
+                cat_ra, cat_dec = row["raMean"], row["decMean"]
+                ra_hms, dec_dms = SkyCoord(cat_ra, cat_dec, unit="deg").to_string("hmsdms").split()
+
+                # Transform SDSS to Johnson-Cousins (Lupton 2005)
+
+                B = row["gMeanApMag"] + 0.3130*(row["gMeanApMag"] - row["rMeanApMag"]) + 0.2271 
+                err_B = np.sqrt((1.313*row["gMeanApMagErr"])**2+(0.313*row["rMeanApMagErr"])**2+0.0107**2)
+
+                V = row["gMeanApMag"] - 0.5784*(row["gMeanApMag"] - row["rMeanApMag"]) - 0.0038  
+                err_V = np.sqrt((0.4216*row["gMeanApMagErr"])**2+(0.5784*row["rMeanApMagErr"])**2+0.0054**2)
+
+                R = row["rMeanApMag"] - 0.2936*(row["rMeanApMag"] - row["iMeanApMag"]) - 0.1439  
+                err_R = np.sqrt((0.7064*row["rMeanApMagErr"])**2+(0.2936*row["iMeanApMagErr"])**2+0.0072**2)
+
+                I = row["rMeanApMag"] - 1.2444*(row["rMeanApMag"] - row["iMeanApMag"]) - 0.3820;  
+                err_I = np.sqrt((0.2444*row["rMeanApMagErr"])**2+(1.2444*row["iMeanApMagErr"])**2+0.0078**2)
+
+                field_star = AstroSource(name=f"PanSTARRS {cat_ra:.5f} {cat_dec:.5f}", 
+                                        other_names=row["objName"],
+                                        ra_hms=ra_hms, dec_dms=dec_dms,
+                                        comment=(f"PanSTARRS field star for {main_src.name}.\n"
+                                                f"Object name: `{row['objName']}`\n"
+                                                f"Autogenerated from PanSTARRS catalog query.\n"
+                                                f"SDSS to Johnson-Cousins transformation by Lupton (2005).\n"
+                                                f"Field `other_names` corresponds to PanSTARRS `objName`.\n"),
+                                        srctype="star",
+                                        mag_B=B, mag_B_err=err_B,
+                                        mag_V=V, mag_V_err=err_V,
+                                        mag_R=R, mag_R_err=err_R,
+                                        mag_I=I, mag_I_err=err_I)
+
+                field_star.save()
+
+                field_stars.append(field_star)
+            except Exception as e:
+                logger.error(f"Error with {row['objName']}: {e}")
+                continue
+
+        logger.info(f"Added {len(field_stars)} PanSTARRS field stars for {main_src.name}")
+
+        main_src.calibrators.add(*field_stars)
+
+        messages.success(request, f"Added {len(field_stars)} PanSTARRS field stars for {main_src.name}")
+
+
+    @admin.action(description='Remove all field stars from PanSTARRS')
+    def remove_field_stars_from_panstarrs(self, request, queryset):
+
+        if queryset.count() > 1:
+            messages.error(request, "Only one source at a time, please.")
+            return
+
+        main_src = queryset.first()
+
+        field_stars = main_src.calibrators.filter(name__startswith="PanSTARRS")
+        field_stars.delete()
+        logger.info(f"Removed {len(field_stars)} PanSTARRS field stars for {main_src.name}")
+
+        messages.success(request, f"Removed {len(field_stars)} PanSTARRS field stars for {main_src.name}")

iop4api/templates/iop4api/index.html

@@ -29,12 +29,12 @@
     <script src="https://ajax.googleapis.com/ajax/libs/jqueryui/1.13.2/jquery-ui.min.js"></script> 
 
     <!-- Bokeh -->
-    <script src="https://cdn.bokeh.org/bokeh/release/bokeh-3.2.2.min.js"></script>
-    <script src="https://cdn.bokeh.org/bokeh/release/bokeh-widgets-3.2.2.min.js"></script>
-    <script src="https://cdn.bokeh.org/bokeh/release/bokeh-tables-3.2.2.min.js"></script>
-    <script src="https://cdn.bokeh.org/bokeh/release/bokeh-api-3.2.2.min.js"></script>
-    <script src="https://cdn.bokeh.org/bokeh/release/bokeh-gl-3.2.2.min.js"></script>
-    <script src="https://cdn.bokeh.org/bokeh/release/bokeh-mathjax-3.2.2.min.js"></script>
+    <script src="https://cdn.bokeh.org/bokeh/release/bokeh-3.6.0.min.js"></script>
+    <script src="https://cdn.bokeh.org/bokeh/release/bokeh-widgets-3.6.0.min.js"></script>
+    <script src="https://cdn.bokeh.org/bokeh/release/bokeh-tables-3.6.0.min.js"></script>
+    <script src="https://cdn.bokeh.org/bokeh/release/bokeh-api-3.6.0.min.js"></script>
+    <script src="https://cdn.bokeh.org/bokeh/release/bokeh-gl-3.6.0.min.js"></script>
+    <script src="https://cdn.bokeh.org/bokeh/release/bokeh-mathjax-3.6.0.min.js"></script>
 
     <!-- CSS styles and JS -->
     <link rel="stylesheet" href="{% static "iop4api/base.css" %}?datetime={% now "Y-m-d-Hi" %}">

iop4api/views/plot.py

@@ -375,7 +375,7 @@ def get_invariable_str(s):
         # alternatively: # figure(sizing_mode="stretch", width=800, height=200,
         p_main = figure(x_range=x1_lims, tools="", lod_threshold=lod_threshold, lod_factor=lod_factor, lod_timeout=lod_timeout, output_backend="webgl")  
         p_main.circle(x='x1', y='y1', source=source, view=view,
-                    size=3, 
+                    radius=3, 
                     line_color=bokeh.colors.named.navy, 
                     fill_color=bokeh.colors.named.navy, 
                     selection_line_color=bokeh.colors.named.navy.darken(0.1), 
@@ -633,7 +633,7 @@ def get_invariable_str(s):
         # Create Div elements for labels
         label = Div(text=f"""<label><span>{instrument}</span><input onclick="plot_hide_instrument(this);" data-instrument="{instrument}" type="checkbox" checked/></label>""", height=21, stylesheets=[label_stylesheet])
         # Create glyphs
-        circle_glyph = Circle(x='x', y='y', fill_color=color, line_color=color, size=8)
+        circle_glyph = Circle(x='x', y='y', fill_color=color, line_color=color, radius=8)
         # Create plots to hold the glyphs
         legend_p = figure(width=21, height=21, toolbar_location=None, min_border=0, tools="")
         legend_p.add_glyph(legend_source, circle_glyph)

iop4lib/db/masterbias.py

@@ -196,7 +196,7 @@ def build_file(self):
         header['EPOCH'] = self.epoch.epochname
         header['IMGSIZE'] = self.imgsize
         header['IMGTYPE'] = 'masterbias'
-        header['DATECREA'] = datetime.datetime.utcnow().isoformat(timespec="milliseconds")
+        header['DATECREA'] = datetime.datetime.now(datetime.UTC).isoformat(timespec="milliseconds")
         header['NRAWFITS'] = self.rawfits.count()
 
         logger.debug(f"Building HDU")

iop4lib/db/masterdark.py

@@ -209,7 +209,7 @@ def build_file(self):
         header['EPOCH'] = self.epoch.epochname
         header['IMGSIZE'] = self.imgsize
         header['IMGTYPE'] = 'masterdark'
-        header['DATECREA'] = datetime.datetime.utcnow().isoformat(timespec="milliseconds")
+        header['DATECREA'] = datetime.datetime.now(datetime.UTC).isoformat(timespec="milliseconds")
         header['NRAWFITS'] = self.rawfits.count()
         header['INSTRUME'] = self.instrument
         header['EXPTIME'] = self.exptime

iop4lib/db/masterflat.py

@@ -229,7 +229,7 @@ def build_file(self):
         header['EPOCH'] = self.epoch.epochname
         header['IMGSIZE'] = self.imgsize
         header['IMGTYPE'] = 'masterflat'
-        header['DATECREA'] = datetime.datetime.utcnow().isoformat(timespec="milliseconds")
+        header['DATECREA'] = datetime.datetime.now(datetime.UTC).isoformat(timespec="milliseconds")
         header['NRAWFITS'] = self.rawfits.count()
         header['BAND'] = self.band
         header['INSTRUME'] = self.instrument

iop4lib/telescopes/telescope.py

@@ -275,6 +275,9 @@ def list_remote_raw_fnames(cls, epoch):
     def download_rawfits(cls, rawfits: list['RawFit'] | list[str]):
         from iop4lib.db import RawFit
 
+        if cls.ftp_address is None or cls.ftp_user is None or cls.ftp_password is None:
+            raise ValueError(f"FTP credentials not set for {cls.name}.")
+
         try:
             ftp =  ftplib.FTP(cls.ftp_address, encoding=cls.ftp_encoding)
             ftp.login(cls.ftp_user, cls.ftp_password)
@@ -310,6 +313,9 @@ def download_rawfits(cls, rawfits: list['RawFit'] | list[str]):
     def list_remote_filelocs(cls, epochnames: list[str]) -> list[str]:
         from iop4lib.db import Epoch
 
+        if cls.ftp_address is None or cls.ftp_user is None or cls.ftp_password is None:
+            raise ValueError(f"FTP credentials not set for {cls.name}.")
+
         ftp =  ftplib.FTP(cls.ftp_address, encoding=cls.ftp_encoding)
         ftp.login(cls.ftp_user, cls.ftp_password)
 

pyproject.toml

@@ -15,23 +15,23 @@ classifiers = [
     "License :: OSI Approved :: BSD License",
     "Operating System :: OS Independent",
 ]
-requires-python = ">3.10,<=3.12"
+requires-python = ">=3.11,<3.13"
 dependencies = [
-    "numpy==1.24.2",
-    "matplotlib==3.7.1",
-    "bokeh==3.2.2",
-    "scipy==1.10.1",
-    "astropy==5.2.2",
+    "numpy>=1.26.4,<2",
+    "matplotlib>=3.7.1,<4",
+    "bokeh==3.6.0", # Bokeh version must match the one in iop4api/templates/iop4api/index.html.
+    "scipy>=1.10.1,<2",
+    "astropy>=5.2.2,<6",
     "astroquery",
-    "photutils==1.8.0",
+    "photutils>=1.8.0,<2",
     "pandas",
-    "scikit-learn==1.2.2",
+    "scikit-learn>=1.2.2,<2",
     "scikit-image",
     "coloredlogs",
     "ipython",
-    "django==4.1.7",
-    "astrometry==4.1.2",
-    "multiprocess==0.70.14",
+    "django<5",
+    "astrometry>=4.1.2,<5",
+    "multiprocess>=0.70.14,<1",
     "pypandoc",
     "termcolor",
     "pyyaml<5.4",
@@ -70,7 +70,8 @@ doc = [
     "sphinx<8; python_version<'3.11'", # See PR 127 (issue with new sphinx 8.0.2 and myst_parser 4.0.0 for Python 3.10.9)
     "sphinx-automodapi",
     "sphinx-mdinclude",
-    "pydata-sphinx-theme<0.15.2",
+    "pydata-sphinx-theme",
+    "pydata-sphinx-theme<0.15.2; python_version<'3.11'",
     "sphinxcontrib-bibtex",
     "docutils>=0.20",
     "nbsphinx",