Merge branch 'main' into try_actions_cache

MANIFEST.in

@@ -0,0 +1,3 @@
+prune iop4lib/_dev_version
+prune .github
+prune .gitignore

README.md

@@ -86,7 +86,7 @@ An utility script, `iop4site/resetdb.py`, is provided which will completely rese
 ### As A Program
 The pipeline script `iop4` can be invoked as
 ```bash
-    $ iop4 -l tel1/yymmdd tel2/yymmdd
+    $ iop4 --epoch-list tel1/yymmdd tel2/yymmdd
 ```
 to download and reduce the epoch `yymmdd` from telescopes `tel1` and `tel2` respectively. For example: `iop4 -l T090/230430`.
 
@@ -131,9 +131,9 @@ iop4conf = iop4lib.Config(config_db=True, gonogui=False, jupytermode=True)
 ```
 
 ### Tips
-You can get an IPython interactive terminal after running iop4 using the `-i` option as
+You can get an IPython interactive terminal after running iop4 using the `-i` option. You can override any config option using the `-o` option, e.g.:
 ```bash
-    $ iop4 -i -l T090/230313 T090/230317 T090/230319
+    $ iop4 -i -o nthreads=20 -o log_file=test.log --epoch-list T090/230313 T090/230317
 ```
 
 ## Documentation

config/config.example.yaml

@@ -9,9 +9,9 @@ datadir: ~/.iop4data/ # <str> Path to iop4data data folder.
 set_rawdata_readonly: False # <bool> True / False (sets raw fits file to readonly when downloading them or creating a RawFit object).
 db_path: ~/.iop4data/iop4.db # Path to iop4 sqlite database file.
 astrometry_cache_path: ~/.astrometry_cache/ # <tr> Path to store the astromery index files.
-max_concurrent_threads: 4 # <int> Number of threads / processes to use (e.g. 4).
+nthreads: 4 # <int> Number of threads / processes to use (e.g. 4).
 astrometry_timeout: 20 # <int> Timeout in minutes for astrometry solving.
-astrometry_allsky_allow: false # <bool> Whether to allow all sky searches in the astrometry.net solver. If there are many all-sky images and max_concurrent_threads is too high, it might cause
+astrometry_allsky_allow: false # <bool> Whether to allow all sky searches in the astrometry.net solver. If there are many all-sky images and nthreads is too high, it might cause
 # very high memory consumption.
 astrometry_allsky_septhreshold: 25 # <int> Threshold of detected poinint mismatch to trigger an all-sky search.
 
@@ -25,7 +25,7 @@ mplt_default_dpi: 100 # <int> dpi for matplotlib (e.g. 100)
 ### LOGGING ###
 ###############
 
-log_fname: ~/.iop4data/iop4.log # </path/to/iop4.log>
+log_file: ~/.iop4data/logs/iop4.log # <str> Path to log file.
 log_date_format: '%Y-%m-%d %H:%M:%S' # <log date format>
 log_format: '%(asctime)s - %(name)s [%(filename)s:%(lineno)d] - %(levelname)s - %(message)s' # <log format> You can also add '%(process)d (%(proc_memory)s)' to the format to show pid and memory usage per process.
 log_level: 20 # <int>. Possible values are: 10 (DEBUG), 20 (INFO), 30 (WARNING), 40 (ERROR), 50 (CRITICAL).
@@ -65,4 +65,12 @@ osn_source_list_path: null # <path/to/osn/source/list>
 
 osn_fnames_patterns: 
 - (^Flat.*\.fi?ts?$)
-- (^Bias.*\.fi?ts?$)
+- (^Bias.*\.fi?ts?$)
+
+
+#################
+### IOP4ADMIN ###
+#################
+
+iop4admin:
+  force_rebuild_finding_charts: true # <bool> Whether to force rebuild finding charts.

iop4admin/modeladmins/fitfile.py

@@ -9,6 +9,7 @@
 from django.urls import path, reverse 
 from django.http import HttpResponse, HttpResponseNotFound, FileResponse
 from django.apps import apps
+from django.utils.safestring import mark_safe
 
 # iop4lib imports
 from iop4api.filters import *
@@ -22,6 +23,21 @@
 import logging
 logger = logging.getLogger(__name__)
 
+
+@admin.action(description="Ignore these files in subsequent runs")
+def action_mark_ignore(modeladmin, request, queryset):
+    for obj in queryset:
+        logger.debug(f"Setting IGNORE flag on {obj}")
+        obj.set_flag(modeladmin.model.FLAGS.IGNORE)
+        obj.save()
+
+@admin.action(description="Do NOT ignore these files in subsequent runs")
+def action_unmark_ignore(modeladmin, request, queryset):
+    for obj in queryset:
+        logger.debug(f"Removing IGNORE flag on {obj}")
+        obj.unset_flag(modeladmin.model.FLAGS.IGNORE)
+        obj.save()
+
 class AdminFitFile(admin.ModelAdmin):
 
     list_per_page = 25
@@ -43,6 +59,15 @@ def filename(self, obj):
     def status(self, obj):
         return ", ".join(obj.flag_labels)
 
+    @admin.display(description="Built from")
+    def get_built_from(self, obj):
+        self.allow_tags = True
+
+        ids_str_L = [str(rf.id) for rf in obj.rawfits.all()]
+        a_href = reverse('iop4admin:%s_%s_changelist' % (RawFit._meta.app_label, RawFit._meta.model_name)) + "?id__in=%s" % ",".join(ids_str_L)
+        a_text = ", ".join(ids_str_L)
+        return mark_safe(f'<a href="{a_href}">{a_text}</a>')
+
     def get_urls(self):
         urls = super().get_urls()
         my_urls = [

iop4admin/modeladmins/masterbias.py

@@ -5,7 +5,7 @@
 from django.utils.safestring import mark_safe
 from iop4api.filters import *
 from iop4api.models import *
-from .fitfile import AdminFitFile
+from .fitfile import AdminFitFile, action_mark_ignore, action_unmark_ignore
 
 import logging
 logger = logging.getLogger(__name__)
@@ -14,7 +14,7 @@ class AdminMasterBias(AdminFitFile):
 
     model = MasterBias
 
-    list_display = ['id', 'telescope', 'night', 'instrument', 'imgsize', 'get_built_from', 'options']
+    list_display = ['id', 'telescope', 'night', 'instrument', 'imgsize', 'get_built_from', 'options', 'status']
 
     list_filter = (
             RawFitIdFilter,
@@ -25,6 +25,7 @@ class AdminMasterBias(AdminFitFile):
             "imgsize",
     )
 
+    actions = [action_mark_ignore, action_unmark_ignore]
 
     @admin.display(description='Options')
     def options(self, obj):
@@ -39,12 +40,4 @@ def telescope(self, obj):
     @admin.display(description='Night')
     def night(self, obj):
         return obj.epoch.night
-
-    @admin.display(description="Built from")
-    def get_built_from(self, obj):
-        self.allow_tags = True
-        link_L = list()
-        for rawfit in obj.rawfits.all():
-            url = reverse('iop4admin:%s_%s_changelist' % (RawFit._meta.app_label, RawFit._meta.model_name)) + f"?id={rawfit.id}"
-            link_L.append(rf'<a href="{url}">{rawfit.id}</a>')
-        return mark_safe(", ".join(link_L))
+

iop4admin/modeladmins/masterdark.py

@@ -6,7 +6,7 @@
 
 from iop4api.filters import *
 from iop4api.models import *
-from .fitfile import AdminFitFile
+from .fitfile import AdminFitFile, action_mark_ignore, action_unmark_ignore
 
 import logging
 logger = logging.getLogger(__name__)
@@ -15,7 +15,7 @@ class AdminMasterDark(AdminFitFile):
 
     model = MasterDark
 
-    list_display = ['id', 'telescope', 'night', 'instrument', 'imgsize', 'exptime', 'get_masterbias', 'get_built_from', 'options']
+    list_display = ['id', 'telescope', 'night', 'instrument', 'imgsize', 'exptime', 'get_masterbias', 'get_built_from', 'options', 'status']
 
     list_filter = (
             RawFitIdFilter,
@@ -26,6 +26,8 @@ class AdminMasterDark(AdminFitFile):
             "imgsize",
     )
 
+    actions = [action_mark_ignore, action_unmark_ignore]
+
     @admin.display(description='Options')
     def options(self, obj):
         url_details = reverse('iop4admin:iop4api_masterdark_details', args=[obj.id])
@@ -47,12 +49,4 @@ def get_masterbias(self, obj):
             return "-"
         url = reverse('iop4admin:%s_%s_changelist' % (MasterBias._meta.app_label, MasterBias._meta.model_name)) + f"?id={obj.masterbias.id}"
         return mark_safe(rf'<a href="{url}">{obj.masterbias.id}</a>')
-
-    @admin.display(description="Built from")
-    def get_built_from(self, obj):
-        self.allow_tags = True
-        link_L = list()
-        for rawfit in obj.rawfits.all():
-            url = reverse('iop4admin:%s_%s_changelist' % (RawFit._meta.app_label, RawFit._meta.model_name)) + f"?id={rawfit.id}"
-            link_L.append(rf'<a href="{url}">{rawfit.id}</a>')
-        return mark_safe(", ".join(link_L))
+

iop4admin/modeladmins/masterflat.py

@@ -6,7 +6,7 @@
 
 from iop4api.filters import *
 from iop4api.models import *
-from .fitfile import AdminFitFile
+from .fitfile import AdminFitFile, action_mark_ignore, action_unmark_ignore
 
 import logging
 logger = logging.getLogger(__name__)
@@ -15,7 +15,7 @@ class AdminMasterFlat(AdminFitFile):
 
     model = MasterFlat
 
-    list_display = ['id', 'telescope', 'night', 'instrument', 'imgsize', 'band', 'obsmode', 'rotangle', 'exptime', 'get_masterbias', 'get_built_from', 'options']
+    list_display = ['id', 'telescope', 'night', 'instrument', 'imgsize', 'band', 'obsmode', 'rotangle', 'exptime', 'get_masterbias', 'get_built_from', 'options', 'status']
 
     list_filter = (
             RawFitIdFilter,
@@ -26,6 +26,7 @@ class AdminMasterFlat(AdminFitFile):
             "imgsize",
     )
 
+    actions = [action_mark_ignore, action_unmark_ignore]
 
     @admin.display(description='Options')
     def options(self, obj):
@@ -48,12 +49,4 @@ def get_masterbias(self, obj):
             return "-"
         url = reverse('iop4admin:%s_%s_changelist' % (MasterBias._meta.app_label, MasterBias._meta.model_name)) + f"?id={obj.masterbias.id}"
         return mark_safe(rf'<a href="{url}">{obj.masterbias.id}</a>')
-
-    @admin.display(description="Built from")
-    def get_built_from(self, obj):
-        self.allow_tags = True
-        link_L = list()
-        for rawfit in obj.rawfits.all():
-            url = reverse('iop4admin:%s_%s_changelist' % (RawFit._meta.app_label, RawFit._meta.model_name)) + f"?id={rawfit.id}"
-            link_L.append(rf'<a href="{url}">{rawfit.id}</a>')
-        return mark_safe(", ".join(link_L))
+

iop4admin/views/astrosource.py

@@ -35,7 +35,7 @@ def get_context_data(self, **kwargs):
         # finding chart
         finding_char_path = Path(obj.filedpropdir) / "finding_chart.png"
 
-        if True: #not os.path.exists(finding_char_path):
+        if not os.path.exists(finding_char_path) or iop4conf.iop4admin['force_rebuild_finding_charts']:
             buf = io.BytesIO()
 
             width, height = 800, 800

iop4api/static/iop4api/base.css

@@ -438,8 +438,6 @@ form label {
 }
 
 .pipeline-log-entry {
-    line-height: 12px !important;
-    padding: 2px !important;
     font-size: 10px !important;
 }
 .pipeline-log-entry .highlight {
@@ -686,3 +684,24 @@ form label {
 .modal_body > label > span {
     padding-left: 0.6em;
 }
+
+/* table filters */
+
+.filter-field, .filter-type, .filter-value, .filter-add, .filter-remove {
+    padding: 2px;
+    margin: 2px;
+  }
+
+.filter-add, .filter-remove {
+    width: 30px;
+    height: 30px;
+    display: inline-flex;
+    justify-content: center;
+    align-content: center;
+    align-items: center;
+    border-radius: 15px;
+  }
+
+#FiltersTable *{
+    font-size: 14px;
+}