Merge pull request #12 from LSSTDESC/issue/375/introspection

Issue/375/introspection

.github/workflows/main.yml

@@ -32,7 +32,7 @@ jobs:
       run: |
         python -m pip install --upgrade pip
         pip install wheel numpy
-        pip install .
+        pip install .[dev]
         pip install flake8 pytest pytest-cov mockmpi pytest-timeout
         if [ -f requirements.txt ]; then pip install -r requirements.txt; fi			
     - name: Test with pytest

nb/estimation/inform_all_hsc.ipynb

@@ -0,0 +1,168 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "f9463120",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# usual imports\n",
+    "import os\n",
+    "import numpy as np\n",
+    "from rail.core.utils import RAILDIR\n",
+    "from rail.estimation.algos.bpz_version.utils import RAIL_BPZ_DIR\n",
+    "from rail.pipelines.estimation.inform_all import InformPipeline\n",
+    "from rail.core import common_params"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "fa7fa474",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "bands = 'grizy'\n",
+    "maglims = [27.66, 27.25, 26.6, 26.24, 25.35]\n",
+    "maglim_dict={}\n",
+    "for band,limx in zip(bands, maglims):\n",
+    "    maglim_dict[f\"HSC{band}_cmodel_dered\"] = limx"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "4f204e39",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "common_params.set_param_defaults(\n",
+    "    bands=[f'HSC{band}_cmodel_dered' for band in bands],\n",
+    "    err_bands=[f'{band}_cmodel_magerr' for band in bands],\n",
+    "    nondetect_val=np.nan,\n",
+    "    ref_band='HSCi_cmodel_dered',\n",
+    "    redshift_col='specz_redshift',\n",
+    "    mag_limits=maglim_dict,\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "36ab51b1",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "pipe = InformPipeline()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "013132ee",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "pipe.inform_bpz.config.update(\n",
+    "    zp_errors=[0.01, 0.01, 0.01, 0.01, 0.01],\n",
+    "    columns_file = os.path.join(\n",
+    "        RAIL_BPZ_DIR, 'rail', 'examples_data', 'estimation_data', 'configs', 'test_bpz_hsc.columns',\n",
+    "    ),\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "140b46c1",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "input_file = 'hsc_train/dered_223501_sz_match_pdr3_dud_NONDET.hdf5'\n",
+    "\n",
+    "input_dict = dict(\n",
+    "    input=input_file,\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "088b42d7",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "pipe_info = pipe.initialize(input_dict, dict(output_dir='.', log_dir='.', resume=True), None)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "298e80fa",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "pipe.save('tmp_inform_all.yml')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "97dd95f0",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import ceci\n",
+    "pr = ceci.Pipeline.read('tmp_inform_all.yml')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "d8a21466",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "pr.run()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "e4dab658",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "8f65a4fb",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.10.10"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

pyproject.toml

@@ -20,11 +20,20 @@ classifiers = [
 ]
 dynamic = ["version"]
 dependencies = [
-    "pz-rail-hub>=0.0.4",
+    "pz-rail>=0.98.5",
 ]
 
 [project.optional-dependencies]
+full = [
+    "pz-rail-bpz>=0.0.9",
+    "pz-rail-flexzboost>=0.0.9",
+    "pz-rail-gpz-v1>=0.1.1",
+]
+
 dev = [
+    "pz-rail-bpz>=0.0.9",
+    "pz-rail-flexzboost>=0.0.9",
+    "pz-rail-gpz-v1>=0.1.1",
     "coverage",
     "pylint",
     "pytest",

src/rail/pipelines/estimation/inform_all.py

@@ -0,0 +1,90 @@
+#!/usr/bin/env python
+# coding: utf-8
+
+# Prerquisites, os, and numpy
+import os
+import numpy as np
+
+# Various rail modules
+import rail.stages
+rail.stages.import_and_attach_all()
+from rail.stages import *
+
+from rail.pipelines.utils.name_factory import NameFactory, DataType, CatalogType, ModelType, PdfType
+from rail.core.stage import RailStage, RailPipeline
+
+import ceci
+
+
+namer = NameFactory()
+from rail.core.utils import RAILDIR
+
+input_file = 'rubin_dm_dc2_example.pq'
+
+
+class InformPipeline(RailPipeline):
+
+    def __init__(self):
+        RailPipeline.__init__(self)
+
+        DS = RailStage.data_store
+        DS.__class__.allow_overwrite = True
+        bands = ['u','g','r','i','z','y']
+        #band_list = [f'mag_{band}_lsst' for band in bands] + [f'mag_err_{band}_lsst' for band in bands]
+
+        self.inform_trainz = Inform_trainZ.build(
+            model=os.path.join(namer.get_data_dir(DataType.model, ModelType.estimator), "model_trainz.pkl"),
+            hdf5_groupname='',
+        )
+
+        self.inform_simplenn = Inform_SimpleNN.build(
+            model=os.path.join(namer.get_data_dir(DataType.model, ModelType.estimator), "model_simplenn.pkl"),
+            hdf5_groupname='',
+        )
+
+        self.inform_knn = Inform_KNearNeighPDF.build(
+            model=os.path.join(namer.get_data_dir(DataType.model, ModelType.estimator), "model_knn.pkl"),
+            hdf5_groupname='',
+        )
+
+        self.inform_simplesom = Inform_SimpleSOMSummarizer.build(
+            model=os.path.join(namer.get_data_dir(DataType.model, ModelType.estimator), "model_simplesom.pkl"),
+             hdf5_groupname='',
+        )
+
+        self.inform_somoclu = Inform_somocluSOMSummarizer.build(
+            model=os.path.join(namer.get_data_dir(DataType.model, ModelType.estimator), "model_somoclu.pkl"),
+            hdf5_groupname='',
+        )
+
+
+        self.inform_bpz = Inform_BPZ_lite.build(
+            model=os.path.join(namer.get_data_dir(DataType.model, ModelType.estimator), "model_bpz.hdf5"),
+             hdf5_groupname='',
+        )
+
+        """
+        self.inform_delight = Inform_DelightPZ.build(
+            model=os.path.join(namer.get_data_dir(DataType.model, ModelType.estimator), "model_delight.hdf5"),
+             hdf5_groupname='',
+        )
+        """
+        self.inform_fzboost = Inform_FZBoost.build(
+            model=os.path.join(namer.get_data_dir(DataType.model, ModelType.estimator), "model_FZBoost.hdf5"),
+            hdf5_groupname='',
+        )
+
+        self.inform_gpz = Inform_GPz_v1.build(
+            model=os.path.join(namer.get_data_dir(DataType.model, ModelType.estimator), "model_gpz.hdf5"),
+             hdf5_groupname='',
+        )
+
+
+
+if __name__ == '__main__':    
+    pipe = InformPipeline()
+    input_dict = dict(
+        input=input_file,
+    )
+    pipe.initialize(input_dict, dict(output_dir='.', log_dir='.', resume=False), None)
+    pipe.save('tmp_inform_all.yml')

src/rail/pipelines/examples/goldenspike/goldenspike.py

@@ -6,6 +6,8 @@
 import numpy as np
 
 # Various rail modules
+import rail.stages
+rail.stages.import_and_attach_all()
 from rail.stages import *
 
 from rail.pipelines.utils.name_factory import NameFactory, DataType, CatalogType, ModelType, PdfType