diff --git a/goodmatches.reg b/goodmatches.reg new file mode 100644 index 000000000..e987ae19a --- /dev/null +++ b/goodmatches.reg @@ -0,0 +1,11 @@ +wcs +point (1156.3772434381995,264.917696941145) #point=cross +point (136.6545795098512,257.3475994422151) #point=cross +point (386.3570084175888,304.73329817655275) #point=cross +point (658.7102203720341,1505.6381193758314) #point=cross +point (1403.3448595431144,1340.6170004758417) #point=cross +point (1543.3771582783136,476.2302901504253) #point=cross +point (1212.0844332063496,882.7325206860631) #point=cross +point (1230.0657672768502,955.2516539580486) #point=cross +point (1379.1515534147043,1253.575877501464) #point=cross +point (736.0425931878985,1032.1629949672376) #point=cross diff --git a/winterdrp.egg-info/PKG-INFO b/winterdrp.egg-info/PKG-INFO new file mode 100644 index 000000000..10d76db89 --- /dev/null +++ b/winterdrp.egg-info/PKG-INFO @@ -0,0 +1,34 @@ +Metadata-Version: 2.1 +Name: winterdrp +Version: 0.1.0 +Summary: Data reduction pipeline for WINTER +Home-page: https://github.com/winter-telescope/winter_drp +Author: Viraj Karambelkar, Robert Stein +Author-email: rdstein@caltech.edu +License: MIT +Keywords: astronomy image WINTER +Platform: UNKNOWN +Classifier: License :: OSI Approved :: MIT License +Classifier: Operating System :: OS Independent +Classifier: Programming Language :: Python :: 3 +Classifier: Programming Language :: Python :: 3.10 +Requires-Python: >=3.10 +Description-Content-Type: text/markdown + +# winter_drp +Requirements and installations - +Sextractor - +1. Download the source code from the repo https://github.com/astromatic/sextractor +2. Follow the instructions here https://sextractor.readthedocs.io/en/latest/Installing.html +3. It will ask you to install all sorts of weird libraries that may or may not be compatible with your device. The configure that worked best for me is without the atlas directories but by using the openblas files instead +./configure --enable-openblas --with-atlas-libdir=/usr/local/opt --with-atlas-incdir=/usr/local/opt/openblas/include + +Instructions for running the focusloop: +1. Download all images to a data directory +2. Download the log file (should be in the format of log_20210625.csv) +3. Run python fix_headers.py --d --l +4. Run python focusLoop.py --d --plot +This will go through all files in the data directory to check for files that are "focus" files and do the analysis on them, and plot the results in a file named "focusloop.pdf' + + + diff --git a/winterdrp.egg-info/SOURCES.txt b/winterdrp.egg-info/SOURCES.txt new file mode 100644 index 000000000..4becc2bdf --- /dev/null +++ b/winterdrp.egg-info/SOURCES.txt @@ -0,0 +1,98 @@ +README.md +setup.py +winterdrp/__init__.py +winterdrp/__main__.py +winterdrp/io.py +winterdrp/paths.py +winterdrp/read_IPAC.py +winterdrp/utils.py +winterdrp.egg-info/PKG-INFO +winterdrp.egg-info/SOURCES.txt +winterdrp.egg-info/dependency_links.txt +winterdrp.egg-info/requires.txt +winterdrp.egg-info/top_level.txt +winterdrp/catalog/__init__.py +winterdrp/catalog/base_catalog.py +winterdrp/catalog/gaia.py +winterdrp/catalog/ps1.py +winterdrp/catalog/sdss.py +winterdrp/catalog/kowalski/__init__.py +winterdrp/catalog/kowalski/ps1.py +winterdrp/catalog/kowalski/tmass.py +winterdrp/downloader/__init__.py +winterdrp/downloader/caltech.py +winterdrp/errors/__init__.py +winterdrp/pipelines/__init__.py +winterdrp/pipelines/base_pipeline.py +winterdrp/pipelines/summer/__init__.py +winterdrp/pipelines/summer/calibration.py +winterdrp/pipelines/summer/summer_pipeline.py +winterdrp/pipelines/summer/summer_files/__init__.py +winterdrp/pipelines/summer/summer_files/schema/__init__.py +winterdrp/pipelines/wirc/__init__.py +winterdrp/pipelines/wirc/wirc_pipeline.py +winterdrp/pipelines/wirc/wirc_files/__init__.py +winterdrp/pipelines/wirc_imsub/__init__.py +winterdrp/pipelines/wirc_imsub/wirc_imsub_pipeline.py +winterdrp/processors/__init__.py +winterdrp/processors/base_processor.py +winterdrp/processors/bias.py +winterdrp/processors/candidate_filters.py +winterdrp/processors/csvlog.py +winterdrp/processors/dark.py +winterdrp/processors/error.py +winterdrp/processors/flat.py +winterdrp/processors/mask.py +winterdrp/processors/photcal.py +winterdrp/processors/reduce.py +winterdrp/processors/reference.py +winterdrp/processors/send_to_fritz.py +winterdrp/processors/sky.py +winterdrp/processors/split.py +winterdrp/processors/xmatch.py +winterdrp/processors/astromatic/__init__.py +winterdrp/processors/astromatic/config/__init__.py +winterdrp/processors/astromatic/psfex/__init__.py +winterdrp/processors/astromatic/psfex/psfex.py +winterdrp/processors/astromatic/scamp/__init__.py +winterdrp/processors/astromatic/scamp/scamp.py +winterdrp/processors/astromatic/scamp/configs/__init__.py +winterdrp/processors/astromatic/sextractor/__init__.py +winterdrp/processors/astromatic/sextractor/settings.py +winterdrp/processors/astromatic/sextractor/sextractor.py +winterdrp/processors/astromatic/sextractor/sourceextractor.py +winterdrp/processors/astromatic/swarp/__init__.py +winterdrp/processors/astromatic/swarp/swarp.py +winterdrp/processors/astromatic/swarp/configs/__init__.py +winterdrp/processors/autoastrometry/__init__.py +winterdrp/processors/autoastrometry/autoastrometry.py +winterdrp/processors/autoastrometry/autoastrometry_processor.py +winterdrp/processors/candidates/__init__.py +winterdrp/processors/candidates/bright_star_mask.py +winterdrp/processors/candidates/candidate_detector.py +winterdrp/processors/candidates/candidate_filter.py +winterdrp/processors/candidates/edge_mask.py +winterdrp/processors/candidates/namer.py +winterdrp/processors/candidates/utils/__init__.py +winterdrp/processors/candidates/utils/dataframe_writer.py +winterdrp/processors/candidates/utils/regions_writer.py +winterdrp/processors/database/__init__.py +winterdrp/processors/database/base_database_processor.py +winterdrp/processors/database/database_exporter.py +winterdrp/processors/database/database_importer.py +winterdrp/processors/database/ingest_database.py +winterdrp/processors/database/postgres.py +winterdrp/processors/utils/__init__.py +winterdrp/processors/utils/header_annotate.py +winterdrp/processors/utils/header_reader.py +winterdrp/processors/utils/image_loader.py +winterdrp/processors/utils/image_saver.py +winterdrp/processors/utils/image_selector.py +winterdrp/references/__init__.py +winterdrp/references/base_reference_generator.py +winterdrp/references/wirc.py +winterdrp/utils/__init__.py +winterdrp/utils/dockerutil.py +winterdrp/utils/execute_cmd.py +winterdrp/utils/fits_tools.py +winterdrp/utils/ldac_tools.py \ No newline at end of file diff --git a/winterdrp.egg-info/dependency_links.txt b/winterdrp.egg-info/dependency_links.txt new file mode 100644 index 000000000..8b1378917 --- /dev/null +++ b/winterdrp.egg-info/dependency_links.txt @@ -0,0 +1 @@ + diff --git a/winterdrp.egg-info/requires.txt b/winterdrp.egg-info/requires.txt new file mode 100644 index 000000000..e309f4acc --- /dev/null +++ b/winterdrp.egg-info/requires.txt @@ -0,0 +1,11 @@ +astropy[recommended] +astroquery +docker +ephem +jupyter +matplotlib +numpy +pandas +psycopg[binary] +pyfftw +wget diff --git a/winterdrp.egg-info/top_level.txt b/winterdrp.egg-info/top_level.txt new file mode 100644 index 000000000..9d2124ca5 --- /dev/null +++ b/winterdrp.egg-info/top_level.txt @@ -0,0 +1 @@ +winterdrp diff --git a/winterdrp/__pycache__/__init__.cpython-310.pyc b/winterdrp/__pycache__/__init__.cpython-310.pyc new file mode 100644 index 000000000..a3751a677 Binary files /dev/null and b/winterdrp/__pycache__/__init__.cpython-310.pyc differ diff --git a/winterdrp/__pycache__/__main__.cpython-310.pyc b/winterdrp/__pycache__/__main__.cpython-310.pyc new file mode 100644 index 000000000..916e28aa6 Binary files /dev/null and b/winterdrp/__pycache__/__main__.cpython-310.pyc differ diff --git a/winterdrp/__pycache__/io.cpython-310.pyc b/winterdrp/__pycache__/io.cpython-310.pyc new file mode 100644 index 000000000..f3a08dd3f Binary files /dev/null and b/winterdrp/__pycache__/io.cpython-310.pyc differ diff --git a/winterdrp/__pycache__/paths.cpython-310.pyc b/winterdrp/__pycache__/paths.cpython-310.pyc new file mode 100644 index 000000000..78b0c8113 Binary files /dev/null and b/winterdrp/__pycache__/paths.cpython-310.pyc differ diff --git a/winterdrp/__pycache__/utils_kowalski.cpython-310.pyc b/winterdrp/__pycache__/utils_kowalski.cpython-310.pyc new file mode 100644 index 000000000..00250e3d6 Binary files /dev/null and b/winterdrp/__pycache__/utils_kowalski.cpython-310.pyc differ diff --git a/winterdrp/catalog/__pycache__/__init__.cpython-310.pyc b/winterdrp/catalog/__pycache__/__init__.cpython-310.pyc new file mode 100644 index 000000000..836b79ca6 Binary files /dev/null and b/winterdrp/catalog/__pycache__/__init__.cpython-310.pyc differ diff --git a/winterdrp/catalog/__pycache__/base_catalog.cpython-310.pyc b/winterdrp/catalog/__pycache__/base_catalog.cpython-310.pyc new file mode 100644 index 000000000..686df909e Binary files /dev/null and b/winterdrp/catalog/__pycache__/base_catalog.cpython-310.pyc differ diff --git a/winterdrp/catalog/__pycache__/gaia.cpython-310.pyc b/winterdrp/catalog/__pycache__/gaia.cpython-310.pyc new file mode 100644 index 000000000..895b48578 Binary files /dev/null and b/winterdrp/catalog/__pycache__/gaia.cpython-310.pyc differ diff --git a/winterdrp/catalog/__pycache__/ps1.cpython-310.pyc b/winterdrp/catalog/__pycache__/ps1.cpython-310.pyc new file mode 100644 index 000000000..d3d05ec49 Binary files /dev/null and b/winterdrp/catalog/__pycache__/ps1.cpython-310.pyc differ diff --git a/winterdrp/catalog/__pycache__/sdss.cpython-310.pyc b/winterdrp/catalog/__pycache__/sdss.cpython-310.pyc new file mode 100644 index 000000000..ade358f89 Binary files /dev/null and b/winterdrp/catalog/__pycache__/sdss.cpython-310.pyc differ diff --git a/winterdrp/catalog/kowalski/__pycache__/__init__.cpython-310.pyc b/winterdrp/catalog/kowalski/__pycache__/__init__.cpython-310.pyc new file mode 100644 index 000000000..218c42a4b Binary files /dev/null and b/winterdrp/catalog/kowalski/__pycache__/__init__.cpython-310.pyc differ diff --git a/winterdrp/catalog/kowalski/__pycache__/ps1.cpython-310.pyc b/winterdrp/catalog/kowalski/__pycache__/ps1.cpython-310.pyc new file mode 100644 index 000000000..3ee5218e1 Binary files /dev/null and b/winterdrp/catalog/kowalski/__pycache__/ps1.cpython-310.pyc differ diff --git a/winterdrp/catalog/kowalski/__pycache__/tmass.cpython-310.pyc b/winterdrp/catalog/kowalski/__pycache__/tmass.cpython-310.pyc new file mode 100644 index 000000000..ddef0b1b4 Binary files /dev/null and b/winterdrp/catalog/kowalski/__pycache__/tmass.cpython-310.pyc differ diff --git a/winterdrp/downloader/__pycache__/__init__.cpython-310.pyc b/winterdrp/downloader/__pycache__/__init__.cpython-310.pyc new file mode 100644 index 000000000..49d46f699 Binary files /dev/null and b/winterdrp/downloader/__pycache__/__init__.cpython-310.pyc differ diff --git a/winterdrp/downloader/__pycache__/caltech.cpython-310.pyc b/winterdrp/downloader/__pycache__/caltech.cpython-310.pyc new file mode 100644 index 000000000..6c5b6f184 Binary files /dev/null and b/winterdrp/downloader/__pycache__/caltech.cpython-310.pyc differ diff --git a/winterdrp/errors/__pycache__/__init__.cpython-310.pyc b/winterdrp/errors/__pycache__/__init__.cpython-310.pyc new file mode 100644 index 000000000..ee1c26c36 Binary files /dev/null and b/winterdrp/errors/__pycache__/__init__.cpython-310.pyc differ diff --git a/winterdrp/pipelines/__pycache__/__init__.cpython-310.pyc b/winterdrp/pipelines/__pycache__/__init__.cpython-310.pyc new file mode 100644 index 000000000..ed6b0a2e7 Binary files /dev/null and b/winterdrp/pipelines/__pycache__/__init__.cpython-310.pyc differ diff --git a/winterdrp/pipelines/__pycache__/base_pipeline.cpython-310.pyc b/winterdrp/pipelines/__pycache__/base_pipeline.cpython-310.pyc new file mode 100644 index 000000000..f48c2bdab Binary files /dev/null and b/winterdrp/pipelines/__pycache__/base_pipeline.cpython-310.pyc differ diff --git a/winterdrp/pipelines/summer/__pycache__/__init__.cpython-310.pyc b/winterdrp/pipelines/summer/__pycache__/__init__.cpython-310.pyc new file mode 100644 index 000000000..7cae2740a Binary files /dev/null and b/winterdrp/pipelines/summer/__pycache__/__init__.cpython-310.pyc differ diff --git a/winterdrp/pipelines/summer/__pycache__/summer_pipeline.cpython-310.pyc b/winterdrp/pipelines/summer/__pycache__/summer_pipeline.cpython-310.pyc new file mode 100644 index 000000000..8cfe05343 Binary files /dev/null and b/winterdrp/pipelines/summer/__pycache__/summer_pipeline.cpython-310.pyc differ diff --git a/winterdrp/pipelines/summer/summer_files/__pycache__/__init__.cpython-310.pyc b/winterdrp/pipelines/summer/summer_files/__pycache__/__init__.cpython-310.pyc new file mode 100644 index 000000000..836619164 Binary files /dev/null and b/winterdrp/pipelines/summer/summer_files/__pycache__/__init__.cpython-310.pyc differ diff --git a/winterdrp/pipelines/summer/summer_files/schema/__pycache__/__init__.cpython-310.pyc b/winterdrp/pipelines/summer/summer_files/schema/__pycache__/__init__.cpython-310.pyc new file mode 100644 index 000000000..cdf8c3beb Binary files /dev/null and b/winterdrp/pipelines/summer/summer_files/schema/__pycache__/__init__.cpython-310.pyc differ diff --git a/winterdrp/pipelines/wirc/__pycache__/__init__.cpython-310.pyc b/winterdrp/pipelines/wirc/__pycache__/__init__.cpython-310.pyc new file mode 100644 index 000000000..b7e96c59c Binary files /dev/null and b/winterdrp/pipelines/wirc/__pycache__/__init__.cpython-310.pyc differ diff --git a/winterdrp/pipelines/wirc/__pycache__/wirc_pipeline.cpython-310.pyc b/winterdrp/pipelines/wirc/__pycache__/wirc_pipeline.cpython-310.pyc new file mode 100644 index 000000000..0b945520e Binary files /dev/null and b/winterdrp/pipelines/wirc/__pycache__/wirc_pipeline.cpython-310.pyc differ diff --git a/winterdrp/pipelines/wirc/wirc_files/__pycache__/__init__.cpython-310.pyc b/winterdrp/pipelines/wirc/wirc_files/__pycache__/__init__.cpython-310.pyc new file mode 100644 index 000000000..deafa8b8a Binary files /dev/null and b/winterdrp/pipelines/wirc/wirc_files/__pycache__/__init__.cpython-310.pyc differ diff --git a/winterdrp/pipelines/wirc_imsub/__pycache__/__init__.cpython-310.pyc b/winterdrp/pipelines/wirc_imsub/__pycache__/__init__.cpython-310.pyc new file mode 100644 index 000000000..d9f595478 Binary files /dev/null and b/winterdrp/pipelines/wirc_imsub/__pycache__/__init__.cpython-310.pyc differ diff --git a/winterdrp/pipelines/wirc_imsub/__pycache__/wirc_imsub_pipeline.cpython-310.pyc b/winterdrp/pipelines/wirc_imsub/__pycache__/wirc_imsub_pipeline.cpython-310.pyc new file mode 100644 index 000000000..123359f5c Binary files /dev/null and b/winterdrp/pipelines/wirc_imsub/__pycache__/wirc_imsub_pipeline.cpython-310.pyc differ diff --git a/winterdrp/pipelines/wirc_imsub/wirc_imsub_pipeline.py b/winterdrp/pipelines/wirc_imsub/wirc_imsub_pipeline.py index 0c1fd69fa..fd29d1514 100644 --- a/winterdrp/pipelines/wirc_imsub/wirc_imsub_pipeline.py +++ b/winterdrp/pipelines/wirc_imsub/wirc_imsub_pipeline.py @@ -114,7 +114,7 @@ class WircImsubPipeline(Pipeline): load_image=load_raw_wirc_image ), # ImageBatcher(split_key='UTSHUT'), - ImageSelector((base_name_key, "ZTF21aagppzg_J_stack_1_20210702.fits")), + ImageSelector((base_name_key, "ZTF21aagppzg_J_stack_1_20210330.fits")), Reference( ref_image_generator=wirc_reference_image_generator, ref_swarp_resampler=wirc_reference_image_resampler, @@ -174,23 +174,18 @@ class WircImsubPipeline(Pipeline): name_start='aaaaa', xmatch_radius_arcsec=2 ), - DatabaseDataframeExporter( - db_name='wirc', - db_table='candidates', - schema_path='winterdrp/pipelines/wirc_imsub/wirc_imsub_files/schema/candidates.sql' - ), - DataframeWriter(output_dir_name='dbop') + # DatabaseDataframeExporter( + # db_name='wirc', + # db_table='candidates', + # schema_path='winterdrp/pipelines/wirc_imsub/wirc_imsub_files/schema/candidates.sql' + # ), + DataframeWriter(output_dir_name='dbop'), # EdgeCandidatesMask(edge_boundary_size=100) # FilterCandidates(), # AvroPacketMaker(output_sub_dir="avro", - # base_name="WNTR", - # broadcast=False, - # save_local=False), - # SendToFritz(output_sub_dir="test") - # base_name="WNTR", # broadcast=False, # save_local=False), - # SendToFritz(output_sub_dir="test") + # SendToFritz(update_thumbnails = True) ] } diff --git a/winterdrp/processors/__pycache__/__init__.cpython-310.pyc b/winterdrp/processors/__pycache__/__init__.cpython-310.pyc new file mode 100644 index 000000000..273e460aa Binary files /dev/null and b/winterdrp/processors/__pycache__/__init__.cpython-310.pyc differ diff --git a/winterdrp/processors/__pycache__/base_processor.cpython-310.pyc b/winterdrp/processors/__pycache__/base_processor.cpython-310.pyc new file mode 100644 index 000000000..ddde9eca1 Binary files /dev/null and b/winterdrp/processors/__pycache__/base_processor.cpython-310.pyc differ diff --git a/winterdrp/processors/__pycache__/bias.cpython-310.pyc b/winterdrp/processors/__pycache__/bias.cpython-310.pyc new file mode 100644 index 000000000..17cd34ed3 Binary files /dev/null and b/winterdrp/processors/__pycache__/bias.cpython-310.pyc differ diff --git a/winterdrp/processors/__pycache__/csvlog.cpython-310.pyc b/winterdrp/processors/__pycache__/csvlog.cpython-310.pyc new file mode 100644 index 000000000..e00f494f7 Binary files /dev/null and b/winterdrp/processors/__pycache__/csvlog.cpython-310.pyc differ diff --git a/winterdrp/processors/__pycache__/dark.cpython-310.pyc b/winterdrp/processors/__pycache__/dark.cpython-310.pyc new file mode 100644 index 000000000..3d02c242d Binary files /dev/null and b/winterdrp/processors/__pycache__/dark.cpython-310.pyc differ diff --git a/winterdrp/processors/__pycache__/flat.cpython-310.pyc b/winterdrp/processors/__pycache__/flat.cpython-310.pyc new file mode 100644 index 000000000..671bf145a Binary files /dev/null and b/winterdrp/processors/__pycache__/flat.cpython-310.pyc differ diff --git a/winterdrp/processors/__pycache__/mask.cpython-310.pyc b/winterdrp/processors/__pycache__/mask.cpython-310.pyc new file mode 100644 index 000000000..e0bb3b804 Binary files /dev/null and b/winterdrp/processors/__pycache__/mask.cpython-310.pyc differ diff --git a/winterdrp/processors/__pycache__/photcal.cpython-310.pyc b/winterdrp/processors/__pycache__/photcal.cpython-310.pyc new file mode 100644 index 000000000..4d5b67835 Binary files /dev/null and b/winterdrp/processors/__pycache__/photcal.cpython-310.pyc differ diff --git a/winterdrp/processors/__pycache__/reference.cpython-310.pyc b/winterdrp/processors/__pycache__/reference.cpython-310.pyc new file mode 100644 index 000000000..476773346 Binary files /dev/null and b/winterdrp/processors/__pycache__/reference.cpython-310.pyc differ diff --git a/winterdrp/processors/__pycache__/send_to_fritz.cpython-310.pyc b/winterdrp/processors/__pycache__/send_to_fritz.cpython-310.pyc new file mode 100644 index 000000000..c98d9fbc2 Binary files /dev/null and b/winterdrp/processors/__pycache__/send_to_fritz.cpython-310.pyc differ diff --git a/winterdrp/processors/__pycache__/sky.cpython-310.pyc b/winterdrp/processors/__pycache__/sky.cpython-310.pyc new file mode 100644 index 000000000..242aa6135 Binary files /dev/null and b/winterdrp/processors/__pycache__/sky.cpython-310.pyc differ diff --git a/winterdrp/processors/__pycache__/split.cpython-310.pyc b/winterdrp/processors/__pycache__/split.cpython-310.pyc new file mode 100644 index 000000000..c436dc121 Binary files /dev/null and b/winterdrp/processors/__pycache__/split.cpython-310.pyc differ diff --git a/winterdrp/processors/__pycache__/xmatch.cpython-310.pyc b/winterdrp/processors/__pycache__/xmatch.cpython-310.pyc new file mode 100644 index 000000000..468bda45c Binary files /dev/null and b/winterdrp/processors/__pycache__/xmatch.cpython-310.pyc differ diff --git a/winterdrp/processors/alert_packets/__pycache__/avro_alert.cpython-310.pyc b/winterdrp/processors/alert_packets/__pycache__/avro_alert.cpython-310.pyc new file mode 100644 index 000000000..a49e35792 Binary files /dev/null and b/winterdrp/processors/alert_packets/__pycache__/avro_alert.cpython-310.pyc differ diff --git a/winterdrp/processors/alert_packets/alert_schema/candidate_all.avsc b/winterdrp/processors/alert_packets/alert_schema/candidate_all.avsc deleted file mode 100644 index 05c301d76..000000000 --- a/winterdrp/processors/alert_packets/alert_schema/candidate_all.avsc +++ /dev/null @@ -1,123 +0,0 @@ -{ - "namespace": "winter.alert", - "name": "candidate", - "doc": "avro alert schema", - "version": "0.1", - "type": "record", - "fields": [ - {"name": "jd", "type": "double", "doc": "Observation Julian date at start of exposure [days]"}, - {"name": "fid", "type": "int", "doc": "Filter ID (1=g; 2=R; 3=i)"}, - // {"name": "pid", "type": "long", "doc": "Processing ID for science image to facilitate archive retrieval"}, - {"name": "diffmaglim", "type": ["float", "null"], "default": null, "doc": "Expected 5-sigma mag limit in difference image based on global noise estimate [mag]"}, - {"name": "pdiffimfilename", "type": ["string", "null"], "default": null, "doc": "filename of positive (sci minus ref) difference image"}, - {"name": "programpi", "type": ["string", "null"], "default": null, "doc": "Principal investigator attached to program ID"}, - {"name": "programid", "type": "int", "doc": "Program ID: encodes either public, collab, or caltech mode"}, - {"name": "candid", "type": "long", "doc": "Candidate ID from operations DB"}, - {"name": "isdiffpos", "type": "string", "doc": "t or 1 => candidate is from positive (sci minus ref) subtraction; f or 0 => candidate is from negative (ref minus sci) subtraction"}, - // {"name": "tblid", "type": ["long", "null"], "default": null, "doc": "Internal pipeline table extraction ID"}, - {"name": "nid", "type": ["int", "null"], "default": null, "doc": "Night ID"}, - // {"name": "rcid", "type": ["int", "null"], "default": null, "doc": "Readout channel ID [00 .. 63]"}, - {"name": "field", "type": ["int", "null"], "default": null, "doc": "ZTF field ID"}, - {"name": "xpos", "type": ["float", "null"], "default": null, "doc": "x-image position of candidate [pixels]"}, - {"name": "ypos", "type": ["float", "null"], "default": null, "doc": "y-image position of candidate [pixels]"}, - {"name": "ra", "type": "double", "doc": "Right Ascension of candidate; J2000 [deg]"}, - {"name": "dec", "type": "double", "doc": "Declination of candidate; J2000 [deg]"}, - {"name": "magpsf", "type": "float", "doc": "Magnitude from PSF-fit photometry [mag]"}, - {"name": "sigmapsf", "type": "float", "doc": "1-sigma uncertainty in magpsf [mag]"}, - {"name": "chipsf", "type": ["float", "null"], "default": null, "doc": "Reduced chi-square for PSF-fit"}, - {"name": "magap", "type": ["float", "null"], "default": null, "doc": "Aperture mag using 14 pixel diameter aperture [mag]"}, - {"name": "sigmagap", "type": ["float", "null"], "default": null, "doc": "1-sigma uncertainty in magap [mag]"}, - {"name": "distnr", "type": ["float", "null"], "default": null, "doc": "distance to nearest source in reference image PSF-catalog [pixels]"}, - // {"name": "magnr", "type": ["float", "null"], "default": null, "doc": "magnitude of nearest source in reference image PSF-catalog [mag]"}, - // {"name": "sigmagnr", "type": ["float", "null"], "default": null, "doc": "1-sigma uncertainty in magnr [mag]"}, - // {"name": "chinr", "type": ["float", "null"], "default": null, "doc": "DAOPhot chi parameter of nearest source in reference image PSF-catalog"}, - // {"name": "sharpnr", "type": ["float", "null"], "default": null, "doc": "DAOPhot sharp parameter of nearest source in reference image PSF-catalog"}, - // {"name": "sky", "type": ["float", "null"], "default": null, "doc": "Local sky background estimate [DN]"}, - {"name": "magdiff", "type": ["float", "null"], "default": null, "doc": "Difference: magap - magpsf [mag]"}, - {"name": "fwhm", "type": ["float", "null"], "default": null, "doc": "Full Width Half Max assuming a Gaussian core, from SExtractor [pixels]"}, - // {"name": "classtar", "type": ["float", "null"], "default": null, "doc": "Star/Galaxy classification score from SExtractor"}, - {"name": "mindtoedge", "type": ["float", "null"], "default": null, "doc": "Distance to nearest edge in image [pixels]"}, - // {"name": "magfromlim", "type": ["float", "null"], "default": null, "doc": "Difference: diffmaglim - magap [mag]"}, - // {"name": "seeratio", "type": ["float", "null"], "default": null, "doc": "Ratio: difffwhm / fwhm"}, - {"name": "aimage", "type": ["float", "null"], "default": null, "doc": "Windowed profile RMS afloat major axis from SExtractor [pixels]"}, - {"name": "bimage", "type": ["float", "null"], "default": null, "doc": "Windowed profile RMS afloat minor axis from SExtractor [pixels]"}, - // {"name": "aimagerat", "type": ["float", "null"], "default": null, "doc": "Ratio: aimage / fwhm"}, - // {"name": "bimagerat", "type": ["float", "null"], "default": null, "doc": "Ratio: bimage / fwhm"}, - // {"name": "elong", "type": ["float", "null"], "default": null, "doc": "Ratio: aimage / bimage"}, - {"name": "nneg", "type": ["int", "null"], "default": null, "doc": "number of negative pixels in a 5 x 5 pixel stamp"}, - // {"name": "nbad", "type": ["int", "null"], "default": null, "doc": "number of prior-tagged bad pixels in a 5 x 5 pixel stamp"}, - {"name": "rb", "type": ["float", "null"], "default": null, "doc": "RealBogus quality score from Random Forest classifier; range is 0 to 1 where closer to 1 is more reliable"}, - {"name": "rbversion", "type": "string", "doc": "version of Deep-Learning-based classifier model used to assign RealBogus (drb) quality score"}, - {"name": "ssdistnr", "type": ["float", "null"], "default": null, "doc": "distance to nearest known solar system object if exists within 30 arcsec [arcsec]"}, - {"name": "ssmagnr", "type": ["float", "null"], "default": null, "doc": "magnitude of nearest known solar system object if exists within 30 arcsec (usually V-band from MPC archive) [mag]"}, - {"name": "ssnamenr", "type": ["string", "null"], "default": null, "doc": "name of nearest known solar system object if exists within 30 arcsec (from MPC archive)"}, - {"name": "sumrat", "type": ["float", "null"], "default": null, "doc": "Ratio: sum(pixels) / sum(|pixels|) in a 5 x 5 pixel stamp where stamp is first median-filtered to mitigate outliers"}, - // {"name": "magapbig", "type": ["float", "null"], "default": null, "doc": "Aperture mag using 18 pixel diameter aperture [mag]"}, - // {"name": "sigmagapbig", "type": ["float", "null"], "default": null, "doc": "1-sigma uncertainty in magapbig [mag]"}, - // {"name": "ranr", "type": "double", "doc": "Right Ascension of nearest source in reference image PSF-catalog; J2000 [deg]"}, - // {"name": "decnr", "type": "double", "doc": "Declination of nearest source in reference image PSF-catalog; J2000 [deg]"}, - {"name": "ndethist", "type": "int", "doc": "Number of spatially-coincident detections falling within 1.5 arcsec going back to beginning of survey; only detections that fell on the same field and readout-channel ID where the input candidate was observed are counted. All raw detections down to a photometric S/N of ~ 3 are included."}, - // {"name": "ncovhist", "type": "int", "doc": "Number of times input candidate position fell on any field and readout-channel going back to beginning of survey"}, - {"name": "jdstarthist", "type": ["double", "null"], "default": null, "doc": "Earliest Julian date of epoch corresponding to ndethist [days]"}, - {"name": "jdendhist", "type": ["double", "null"], "default": null, "doc": "Latest Julian date of epoch corresponding to ndethist [days]"}, - {"name": "scorr", "type": ["double", "null"], "default": null, "doc": "Peak-pixel signal-to-noise ratio in point source matched-filtered detection image"}, - {"name": "tooflag", "type": ["int", "null"], "default": 0, "doc": "1 => candidate is from a Target-of-Opportunity (ToO) exposure; 0 => candidate is from a non-ToO exposure"}, - // {"name": "objectidps1", "type": ["long", "null"], "default": null, "doc": "Object ID of closest source from PS1 catalog; if exists within 30 arcsec"}, - {"name": "sgmag1", "type": ["float", "null"], "default": null, "doc": "g-band PSF-fit magnitude of closest source from PS1 catalog; if exists within 30 arcsec [mag]"}, - {"name": "srmag1", "type": ["float", "null"], "default": null, "doc": "r-band PSF-fit magnitude of closest source from PS1 catalog; if exists within 30 arcsec [mag]"}, - {"name": "simag1", "type": ["float", "null"], "default": null, "doc": "i-band PSF-fit magnitude of closest source from PS1 catalog; if exists within 30 arcsec [mag]"}, - {"name": "szmag1", "type": ["float", "null"], "default": null, "doc": "z-band PSF-fit magnitude of closest source from PS1 catalog; if exists within 30 arcsec [mag]"}, - {"name": "tmjmag1", "type": ["float", "null"], "default": null, "doc": "J-band PSF magnitude of closest source from 2MASS catalog; if exists within 30 arcsec [mag]"}, - {"name": "tmhmag1", "type": ["float", "null"], "default": null, "doc": "H-band PSF magnitude of closest source from 2MASS catalog; if exists within 30 arcsec [mag]"}, - {"name": "tmkmag1", "type": ["float", "null"], "default": null, "doc": "K-band PSF magnitude of closest source from 2MASS catalog; if exists within 30 arcsec [mag]"}, - {"name": "objectidtm1", "type": ["long", "null"], "default": null, "doc": "Object ID of closest source from 2MASS catalog; if exists within 30 arcsec"}, - {"name": "sgscore1", "type": ["float", "null"], "default": null, "doc": "Star/Galaxy score of closest source from PS1 catalog; if exists within 30 arcsec: 0 <= sgscore <= 1 where closer to 1 implies higher likelihood of being a star"}, - {"name": "distpsnr1", "type": ["float", "null"], "default": null, "doc": "Distance to closest source from PS1 catalog; if exists within 30 arcsec [arcsec]"}, - {"name": "objectidps2", "type": ["long", "null"], "default": null, "doc": "Object ID of second closest source from PS1 catalog; if exists within 30 arcsec"}, - {"name": "sgmag2", "type": ["float", "null"], "default": null, "doc": "g-band PSF-fit magnitude of second closest source from PS1 catalog; if exists within 30 arcsec [mag]"}, - {"name": "srmag2", "type": ["float", "null"], "default": null, "doc": "r-band PSF-fit magnitude of second closest source from PS1 catalog; if exists within 30 arcsec [mag]"}, - {"name": "simag2", "type": ["float", "null"], "default": null, "doc": "i-band PSF-fit magnitude of second closest source from PS1 catalog; if exists within 30 arcsec [mag]"}, - {"name": "szmag2", "type": ["float", "null"], "default": null, "doc": "z-band PSF-fit magnitude of second closest source from PS1 catalog; if exists within 30 arcsec [mag]"}, - {"name": "sgscore2", "type": ["float", "null"], "default": null, "doc": "Star/Galaxy score of second closest source from PS1 catalog; if exists within 30 arcsec: 0 <= sgscore <= 1 where closer to 1 implies higher likelihood of being a star"}, - {"name": "distpsnr2", "type": ["float", "null"], "default": null, "doc": "Distance to second closest source from PS1 catalog; if exists within 30 arcsec [arcsec]"}, - {"name": "tmjmag2", "type": ["float", "null"], "default": null, "doc": "J-band PSF magnitude of second source from 2MASS catalog; if exists within 30 arcsec [mag]"}, - {"name": "tmhmag2", "type": ["float", "null"], "default": null, "doc": "H-band PSF magnitude of second source from 2MASS catalog; if exists within 30 arcsec [mag]"}, - {"name": "tmkmag2", "type": ["float", "null"], "default": null, "doc": "K-band PSF magnitude of second source from 2MASS catalog; if exists within 30 arcsec [mag]"}, - {"name": "objectidtm2", "type": ["long", "null"], "default": null, "doc": "Object ID of second source from 2MASS catalog; if exists within 30 arcsec"}, - {"name": "objectidps3", "type": ["long", "null"], "default": null, "doc": "Object ID of third closest source from PS1 catalog; if exists within 30 arcsec"}, {"name": "sgmag3", "type": ["float", "null"], "default": null, "doc": "g-band PSF-fit magnitude of third closest source from PS1 catalog; if exists within 30 arcsec [mag]"}, - {"name": "srmag3", "type": ["float", "null"], "default": null, "doc": "r-band PSF-fit magnitude of third closest source from PS1 catalog; if exists within 30 arcsec [mag]"}, - {"name": "simag3", "type": ["float", "null"], "default": null, "doc": "i-band PSF-fit magnitude of third closest source from PS1 catalog; if exists within 30 arcsec [mag]"}, - {"name": "szmag3", "type": ["float", "null"], "default": null, "doc": "z-band PSF-fit magnitude of third closest source from PS1 catalog; if exists within 30 arcsec [mag]"}, - {"name": "sgscore3", "type": ["float", "null"], "default": null, "doc": "Star/Galaxy score of third closest source from PS1 catalog; if exists within 30 arcsec: 0 <= sgscore <= 1 where closer to 1 implies higher likelihood of being a star"}, - {"name": "distpsnr3", "type": ["float", "null"], "default": null, "doc": "Distance to third closest source from PS1 catalog; if exists within 30 arcsec [arcsec]"}, - {"name": "tmjmag3", "type": ["float", "null"], "default": null, "doc": "J-band PSF magnitude of third source from 2MASS catalog; if exists within 30 arcsec [mag]"}, - {"name": "tmhmag3", "type": ["float", "null"], "default": null, "doc": "H-band PSF magnitude of third source from 2MASS catalog; if exists within 30 arcsec [mag]"}, - {"name": "tmkmag3", "type": ["float", "null"], "default": null, "doc": "K-band PSF magnitude of third source from 2MASS catalog; if exists within 30 arcsec [mag]"}, - {"name": "objectidtm3", "type": ["long", "null"], "default": null, "doc": "Object ID of third source from 2MASS catalog; if exists within 30 arcsec"}, - // {"name": "nmtchps", "type": "int", "doc": "Number of source matches from PS1 catalog falling within 30 arcsec"}, - {"name": "nmtchtm", "type": "int", "doc": "Number of source matches from 2MASS catalog falling within 30 arcsec"}, - // {"name": "rfid", "type": "long", "doc": "Processing ID for reference image to facilitate archive retrieval"}, - {"name": "jdstartref", "type": "double", "doc": "Observation Julian date of earliest exposure used to generate reference image [days]"}, - {"name": "jdendref", "type": "double", "doc": "Observation Julian date of latest exposure used to generate reference image [days]"}, - {"name": "nframesref", "type": "int", "doc": "Number of frames (epochal images) used to generate reference image"}, - {"name": "rbversion", "type": "string", "doc": "version of Random Forest classifier model used to assign RealBogus (rb) quality score"}, - // {"name": "dsnrms", "type": ["float", "null"], "default": null, "doc": "Ratio: D/stddev(D) on event position where D = difference image"}, - // {"name": "ssnrms", "type": ["float", "null"], "default": null, "doc": "Ratio: S/stddev(S) on event position where S = image of convolution: D (x) PSF(D)"}, - // {"name": "dsdiff", "type": ["float", "null"], "default": null, "doc": "Difference of statistics: dsnrms - ssnrms"}, - {"name": "magzpsci", "type": ["float", "null"], "default": null, "doc": "Magnitude zero point for photometry estimates [mag]"}, - {"name": "magzpsciunc", "type": ["float", "null"], "default": null, "doc": "Magnitude zero point uncertainty (in magzpsci) [mag]"}, - {"name": "magzpscirms", "type": ["float", "null"], "default": null, "doc": "RMS (deviation from average) in all differences between instrumental photometry and matched photometric calibrators from science image processing [mag]"}, - {"name": "nmatches", "type": "int", "doc": "Number of PS1 photometric calibrators used to calibrate science image from science image processing"}, - {"name": "clrcoeff", "type": ["float", "null"], "default": null, "doc": "Color coefficient from linear fit from photometric calibration of science image"}, - {"name": "clrcounc", "type": ["float", "null"], "default": null, "doc": "Color coefficient uncertainty from linear fit (corresponding to clrcoeff)"}, - {"name": "zpclrcov", "type": ["float", "null"], "default": null, "doc": "Covariance in magzpsci and clrcoeff from science image processing [mag^2]"}, - {"name": "zpmed", "type": ["float", "null"], "default": null, "doc": "Magnitude zero point from median of all differences between instrumental photometry and matched photometric calibrators from science image processing [mag]"}, - {"name": "clrmed", "type": ["float", "null"], "default": null, "doc": "Median color of all PS1 photometric calibrators used from science image processing [mag]: for filter (fid) = 1, 2, 3, PS1 color used = g-r, g-r, r-i respectively"}, - {"name": "clrrms", "type": ["float", "null"], "default": null, "doc": "RMS color (deviation from average) of all PS1 photometric calibrators used from science image processing [mag]"}, - {"name": "neargaia", "type": ["float", "null"], "default": null, "doc": "Distance to closest source from Gaia DR1 catalog irrespective of magnitude; if exists within 90 arcsec [arcsec]"}, - {"name": "neargaiabright", "type": ["float", "null"], "default": null, "doc": "Distance to closest source from Gaia DR1 catalog brighter than magnitude 14; if exists within 90 arcsec [arcsec]"}, - {"name": "maggaia", "type": ["float", "null"], "default": null, "doc": "Gaia (G-band) magnitude of closest source from Gaia DR1 catalog irrespective of magnitude; if exists within 90 arcsec [mag]"}, - {"name": "maggaiabright", "type": ["float", "null"], "default": null, "doc": "Gaia (G-band) magnitude of closest source from Gaia DR1 catalog brighter than magnitude 14; if exists within 90 arcsec [mag]"}, - {"name": "exptime", "type": ["float", "null"], "default": null, "doc": "Integration time of camera exposure [sec]"} - ] -} \ No newline at end of file diff --git a/winterdrp/processors/alert_packets/alert_schema/candidate_custom_old.avsc b/winterdrp/processors/alert_packets/alert_schema/candidate_custom_old.avsc deleted file mode 100644 index 92b900e8a..000000000 --- a/winterdrp/processors/alert_packets/alert_schema/candidate_custom_old.avsc +++ /dev/null @@ -1,39 +0,0 @@ -{ - "namespace": "winter.alert", - "type": "record", - "name": "candidate", - "doc": "avro alert schema", - "version": "0.1", - "fields": [ - {"name": "NUMBER", "type": ["int","null"]}, - {"name": "FLUX_APER", "type": ["float", "null"]}, - {"name": "FLUXERR_APER", "type": ["float", "null"]}, - {"name": "FLUX_RADIUS", "type": ["float", "null"]}, - {"name": "FLUX_MAX", "type": ["float", "null"]}, - {"name": "X_IMAGE", "type": ["float", "null"]}, - {"name": "Y_IMAGE", "type": ["float", "null"]}, - {"name": "XPEAK_IMAGE", "type": ["int","null"]}, - {"name": "YPEAK_IMAGE", "type": ["int","null"]}, - {"name": "ERRX2_IMAGE", "type": ["float", "null"]}, - {"name": "ERRY2_IMAGE", "type": ["float", "null"]}, - {"name": "ALPHA_J2000", "type": ["float", "null"]}, - {"name": "DELTA_J2000", "type": ["float", "null"]}, - {"name": "A_IMAGE", "type": ["float", "null"]}, - {"name": "B_IMAGE", "type": ["float", "null"]}, - {"name": "ELONGATION", "type": ["float", "null"]}, - {"name": "FWHM_IMAGE", "type": ["float", "null"]}, - {"name": "FLAGS", "type": ["int","null"]}, - {"name": "Scorr_peak", "type": ["float", "null"]}, - {"name": "psf_flux", "type": ["float", "null"]}, - {"name": "psf_fluxunc", "type": ["float", "null"]}, - {"name": "minchi2", "type": ["float", "null"]}, - {"name": "xshift", "type": ["float", "null"]}, - {"name": "yshift", "type": ["float", "null"]}, - {"name": "SciBitIm", "type": ["bytes", "null"]}, - {"name": "RefBitIm", "type": ["bytes", "null"]}, - {"name": "DiffBitIm", "type": ["bytes", "null"]}, - {"name": "psf_mag", "type": ["float", "null"]}, - {"name": "psf_magerr", "type": ["float", "null"]}, - {"name": "ZP", "type": ["float", "null"]} - ] -} \ No newline at end of file diff --git a/winterdrp/processors/alert_packets/alert_schema/prv_candidate.avsc b/winterdrp/processors/alert_packets/alert_schema/prv_candidate.avsc index 55b49e105..50f1478a8 100644 --- a/winterdrp/processors/alert_packets/alert_schema/prv_candidate.avsc +++ b/winterdrp/processors/alert_packets/alert_schema/prv_candidate.avsc @@ -6,8 +6,6 @@ "type": "record", "fields": [ {"name": "jd", "type": "double", "doc": "Observation Julian date at start of exposure [days]"}, - {"name": "subid", "type": "long", "doc": "Processing ID for subtraction image"}, - {"name": "stackquadid", "type": "long", "doc": "Processing ID for science image"}, {"name": "diffmaglim", "type": ["float", "null"], "default": null, "doc": "Expected 5-sigma mag limit in difference image based on global noise estimate [mag]"}, {"name": "program", "type": ["string", "null"], "default": null, "doc": "Observing program associated with data"}, {"name": "candid", "type": ["long", "null"], "doc": "Candidate ID from operations DB"}, diff --git a/winterdrp/processors/alert_packets/alert_schema/prv_candidate_copied.avsc b/winterdrp/processors/alert_packets/alert_schema/prv_candidate_copied.avsc deleted file mode 100644 index 0a0eb52be..000000000 --- a/winterdrp/processors/alert_packets/alert_schema/prv_candidate_copied.avsc +++ /dev/null @@ -1,30 +0,0 @@ -{ - "namespace": "winter.alert", - "name": "prv_candidate", - "doc": "avro alert schema", - "version": "0.1", - "type": "record", - "fields": [ - {"name": "jd", "type": "double", "doc": "Observation Julian date at start of exposure [days]"}, - {"name": "subid", "type": "long", "doc": "Processing ID for subtraction image"}, - {"name": "stackquadid", "type": "long", "doc": "Processing ID for science image"}, - {"name": "diffmaglim", "type": ["float", "null"], "default": null, "doc": "Expected 5-sigma mag limit in difference image based on global noise estimate [mag]"}, - {"name": "program", "type": ["string", "null"], "default": null, "doc": "Observing program associated with data"}, - {"name": "candid", "type": ["long", "null"], "doc": "Candidate ID from operations DB"}, - {"name": "isdiffpos", "type": ["string", "null"], "doc": "t or 1 => candidate is from positive (sci minus ref) subtraction; f or 0 => candidate is from negative (ref minus sci) subtraction"}, - {"name": "nid", "type": ["int", "null"], "default": null, "doc": "Night ID"}, - {"name": "quadpos", "type": ["int", "null"], "default": null, "doc": "Quadrant ID [0..3]"}, - {"name": "subquadpos", "type": ["int", "null"], "default": null, "doc": "Sub-Quadrant ID [0..3]"}, - {"name": "field", "type": ["int", "null"], "default": null, "doc": "ZTF field ID"}, - {"name": "xpos", "type": ["float", "null"], "default": null, "doc": "x-image position of candidate [pixels]"}, - {"name": "ypos", "type": ["float", "null"], "default": null, "doc": "y-image position of candidate [pixels]"}, - {"name": "ra", "type": ["double", "null"], "doc": "Right Ascension of candidate; J2000 [deg]"}, - {"name": "dec", "type": ["double", "null"], "doc": "Declination of candidate; J2000 [deg]"}, - {"name": "magpsf", "type": ["float", "null"], "doc": "Magnitude from PSF-fit photometry [mag]"}, - {"name": "sigmapsf", "type": ["float", "null"], "doc": "1-sigma uncertainty in magpsf [mag]"}, - {"name": "fwhm", "type": ["float", "null"], "default": null, "doc": "Full Width Half Max assuming a Gaussian core, from SExtractor [pixels]"}, - {"name": "scorr", "type": ["double", "null"], "default": null, "doc": "Peak-pixel signal-to-noise ratio in point source matched-filtered detection image"}, - {"name": "drb", "type": ["float", "null"], "default": null, "doc": "RealBogus quality score from Deep-Learning-based classifier; range is 0 to 1 where closer to 1 is more reliable"}, - {"name": "drbversion", "type": ["string", "null"], "doc": "version of Deep-Learning-based classifier model used to assign RealBogus (drb) quality score"} - ] -} \ No newline at end of file diff --git a/winterdrp/processors/alert_packets/avro_alert.py b/winterdrp/processors/alert_packets/avro_alert.py index dd1967fdc..401646fce 100644 --- a/winterdrp/processors/alert_packets/avro_alert.py +++ b/winterdrp/processors/alert_packets/avro_alert.py @@ -5,6 +5,7 @@ from astropy.time import Time from astropy.io import ascii import pandas as pd +import datetime import io, gzip, os, sys import avro, fastavro @@ -32,7 +33,6 @@ class AvroPacketMaker(BaseDataframeProcessor): output_sub_dir (str): output data path. base_name (str): 4-letter code for telescope. save_local (bool): save avro packets to out_sub_dir. - use_database (bool): read candidate info from database. #TODO broadcast (bool): send to brokers at IPAC. """ @@ -40,7 +40,6 @@ def __init__(self, output_sub_dir: str, base_name: str, save_local=False, - use_database=False, broadcast=True, *args, **kwargs): @@ -48,7 +47,6 @@ def __init__(self, self.output_sub_dir = output_sub_dir self.base_name = base_name self.save_local = save_local - self.use_database = use_database self.broadcast = broadcast def _apply_to_candidates( @@ -75,14 +73,13 @@ def read_input_df(self, df): candidate = {} for key in df.keys(): try: - if type(df.iloc[i].get(key)) is str: + if type(df.iloc[i].get(key)) is str or type(df.iloc[i].get(key)) is list: candidate[key] = df.iloc[i].get(key) else: # change to native python type candidate[key] = df.iloc[i].get(key).item() except AttributeError: # for IOBytes objs candidate[key] = df.iloc[i].get(key).getvalue() - all_candidates.append(candidate) return all_candidates @@ -168,74 +165,6 @@ def read_avro_data(self, bytes_io, avro_schema): message = reader.read(decoder) return message - def get_next_name(self, lastname, candjd, bwfile = 'badwords.txt', begcount = 'aaaaaaa'): - """Creates candidate name following the naming format of 'WNTR22aaaaaaa' . - Modified from https://github.com/dekishalay/pgirdps - - Args: - lastname(str): last used candidate name. - candjd (str): candidate's JD. - bwfile (str): file name of .txt file of excluded words. - begcount (str): string to start naming convention. - - Returns: - (str): next candidate name in sequence. - """ - curyear = Time(candjd, format = 'jd').datetime.strftime('%Y')[2:4] - - if lastname is None: - #If this is the first source being named - newname = self.base_name + curyear + begcount - return newname - - lastyear = lastname[4:6] - - if curyear != lastyear: - #If this is the first candidate of the new year, start with aaaaa - newname = self.base_name + curyear + begcount - return newname - else: - lastcount = lastname[6:] - charpos = len(lastcount) - 1 - # will iteratively try to increment characters starting from the last - inctrue = False - usestring = '' - while charpos >= 0: - cref = lastcount[charpos] - if inctrue: - usestring = cref + usestring - charpos -= 1 - continue - creford = ord(cref) - #increment each character, if at 'z', increment the next one - if creford + 1 > 122: - usestring = 'a' + usestring - else: - nextchar = chr(creford+1) - usestring = nextchar + usestring - inctrue = True - charpos -= 1 - continue - - newname = self.base_name + curyear + usestring - - curdir = os.path.dirname(__file__) # /data/sulekha/winterdrp/winterdrp/processors/alert_packets - file_path = os.path.join(curdir, bwfile) - - bwlist = ascii.read(file_path, format = 'no_header') - isbw = False - # check for bad word - for i in range(len(bwlist)): - if usestring.find(str(bwlist['col1'][i])) != -1: - #Shame shame - isbw = True - break - if isbw: - # increment the name with a recursive call - return self.get_next_name(newname, candjd) - else: - return newname - def create_alert_packet(self, cand, scicut, refcut, diffcut, cm_radius=8.0, search_history=90.0): """Create top level avro packet from input candidate. Args: @@ -249,9 +178,7 @@ def create_alert_packet(self, cand, scicut, refcut, diffcut, cm_radius=8.0, sear Returns: (dict): schema in dictionary format. """ - # TODO populate the candidate history (prv_candidate) - prev_cands = [] - # logger.info(f' {len(prev_cands)} prev candidates found ########') + prev_cands = cand["prv_candidates"] alert = {"schemavsn": "0.1", "publisher": "winter_test", "cutoutScience": scicut, @@ -374,9 +301,9 @@ def _make_ind_packet(self, cand, schema, topic_name, cand_num, num_cands): -1 if candidate not sent or saved. """ # Cutouts are include in the top level alert schema - scicut = cand.pop('SciBitIm') - refcut = cand.pop('RefBitIm') - diffcut = cand.pop('DiffBitIm') + scicut = cand.pop('cutoutScience') + refcut = cand.pop('cutoutTemplate') + diffcut = cand.pop('cutoutDifference') packet = self.create_alert_packet(cand, scicut, refcut, diffcut) @@ -402,7 +329,7 @@ def _success_message(self): message += "broadcasted" return message - def make_alert(self, df=None): + def make_alert(self, df): """Top level method to make avro alert. Args: @@ -410,10 +337,9 @@ def make_alert(self, df=None): """ t0 = time.time() - if not self.use_database and df is not None: - # input dataframe to avro_creation processor - logger.info(f'Avro Packet Maker: parsing {len(df)} candidates from dataframe') - all_cands = self.read_input_df(df) + + logger.info(f'Avro Packet Maker: parsing {len(df)} candidates from dataframe') + all_cands = self.read_input_df(df) num_cands = len(all_cands) successes = 0 @@ -423,40 +349,24 @@ def make_alert(self, df=None): "alert_schema/prv_candidate.avsc", "alert_schema/alert.avsc"]) - # TODO: fake!! remove; lastname, cand_id needs to updated from database - last_name = None - cand_id = 700 cand_num = 1 # for keeping track of broadcast alerts for cand in all_cands: - cand_jd = cand['jd'] - cand_name = self.get_next_name(last_name, str(cand_jd)) # TODO create alert_date once (before loop)from list of jd_list # Calculate the alert_date for this night # alert_date = Time(cand_jd, format = 'jd').tt.datetime.strftime('%Y%m%d') # logger.info(alert_date) # topic_name = 'winter_%s'%alert_date - topic_name = 'winter_20220714' - - # TODO candid should be coming from naming database - cand['candid'] = cand_id - cand_id += 1 - - # TODO check if cand is new? - # cand_name, new_status = self.check_and_insert_source(cand_name, cand) - - #TODO remove once new_status is up-to-date: dummy line - new_status = True - - if new_status: - last_name = cand_name - cand['objectId'] = cand_name + topic_name = f"winter_{datetime.datetime.utcnow().strftime('%Y%m%d')}" + logger.info(f'topic name {topic_name}') flag = self._make_ind_packet(cand, schema, topic_name, cand_num, num_cands) cand_num += 1 # for counting num of processed candidates if flag > 0: successes += 1 + + cand_id = cand["candid"] # used for testing opening the avro packets t1 = time.time() logger.info('###########################################') @@ -466,7 +376,7 @@ def make_alert(self, df=None): # Read data from an avro file - last_file = str(cand_id - 1) + '.avro' + last_file = str(cand_id) + '.avro' last_packet_path = os.path.join(self.get_sub_output_dir(), last_file) with open(last_packet_path, 'rb') as f: reader = DataFileReader(f, DatumReader()) diff --git a/winterdrp/processors/astromatic/__pycache__/__init__.cpython-310.pyc b/winterdrp/processors/astromatic/__pycache__/__init__.cpython-310.pyc new file mode 100644 index 000000000..992eef4b3 Binary files /dev/null and b/winterdrp/processors/astromatic/__pycache__/__init__.cpython-310.pyc differ diff --git a/winterdrp/processors/astromatic/config/__pycache__/__init__.cpython-310.pyc b/winterdrp/processors/astromatic/config/__pycache__/__init__.cpython-310.pyc new file mode 100644 index 000000000..f6bd75f36 Binary files /dev/null and b/winterdrp/processors/astromatic/config/__pycache__/__init__.cpython-310.pyc differ diff --git a/winterdrp/processors/astromatic/psfex/__pycache__/__init__.cpython-310.pyc b/winterdrp/processors/astromatic/psfex/__pycache__/__init__.cpython-310.pyc new file mode 100644 index 000000000..06b473996 Binary files /dev/null and b/winterdrp/processors/astromatic/psfex/__pycache__/__init__.cpython-310.pyc differ diff --git a/winterdrp/processors/astromatic/psfex/__pycache__/psfex.cpython-310.pyc b/winterdrp/processors/astromatic/psfex/__pycache__/psfex.cpython-310.pyc new file mode 100644 index 000000000..c01dfe44c Binary files /dev/null and b/winterdrp/processors/astromatic/psfex/__pycache__/psfex.cpython-310.pyc differ diff --git a/winterdrp/processors/astromatic/scamp/__pycache__/__init__.cpython-310.pyc b/winterdrp/processors/astromatic/scamp/__pycache__/__init__.cpython-310.pyc new file mode 100644 index 000000000..85c6976ec Binary files /dev/null and b/winterdrp/processors/astromatic/scamp/__pycache__/__init__.cpython-310.pyc differ diff --git a/winterdrp/processors/astromatic/scamp/__pycache__/scamp.cpython-310.pyc b/winterdrp/processors/astromatic/scamp/__pycache__/scamp.cpython-310.pyc new file mode 100644 index 000000000..c625c1eb9 Binary files /dev/null and b/winterdrp/processors/astromatic/scamp/__pycache__/scamp.cpython-310.pyc differ diff --git a/winterdrp/processors/astromatic/sextractor/__pycache__/__init__.cpython-310.pyc b/winterdrp/processors/astromatic/sextractor/__pycache__/__init__.cpython-310.pyc new file mode 100644 index 000000000..e76415f11 Binary files /dev/null and b/winterdrp/processors/astromatic/sextractor/__pycache__/__init__.cpython-310.pyc differ diff --git a/winterdrp/processors/astromatic/sextractor/__pycache__/settings.cpython-310.pyc b/winterdrp/processors/astromatic/sextractor/__pycache__/settings.cpython-310.pyc new file mode 100644 index 000000000..b915a4929 Binary files /dev/null and b/winterdrp/processors/astromatic/sextractor/__pycache__/settings.cpython-310.pyc differ diff --git a/winterdrp/processors/astromatic/sextractor/__pycache__/sextractor.cpython-310.pyc b/winterdrp/processors/astromatic/sextractor/__pycache__/sextractor.cpython-310.pyc new file mode 100644 index 000000000..c36f13172 Binary files /dev/null and b/winterdrp/processors/astromatic/sextractor/__pycache__/sextractor.cpython-310.pyc differ diff --git a/winterdrp/processors/astromatic/sextractor/__pycache__/sourceextractor.cpython-310.pyc b/winterdrp/processors/astromatic/sextractor/__pycache__/sourceextractor.cpython-310.pyc new file mode 100644 index 000000000..aec74f174 Binary files /dev/null and b/winterdrp/processors/astromatic/sextractor/__pycache__/sourceextractor.cpython-310.pyc differ diff --git a/winterdrp/processors/astromatic/swarp/__pycache__/__init__.cpython-310.pyc b/winterdrp/processors/astromatic/swarp/__pycache__/__init__.cpython-310.pyc new file mode 100644 index 000000000..ffca0b6fd Binary files /dev/null and b/winterdrp/processors/astromatic/swarp/__pycache__/__init__.cpython-310.pyc differ diff --git a/winterdrp/processors/astromatic/swarp/__pycache__/swarp.cpython-310.pyc b/winterdrp/processors/astromatic/swarp/__pycache__/swarp.cpython-310.pyc new file mode 100644 index 000000000..5a9f062d2 Binary files /dev/null and b/winterdrp/processors/astromatic/swarp/__pycache__/swarp.cpython-310.pyc differ diff --git a/winterdrp/processors/autoastrometry/__pycache__/__init__.cpython-310.pyc b/winterdrp/processors/autoastrometry/__pycache__/__init__.cpython-310.pyc new file mode 100644 index 000000000..5bcc8cebf Binary files /dev/null and b/winterdrp/processors/autoastrometry/__pycache__/__init__.cpython-310.pyc differ diff --git a/winterdrp/processors/autoastrometry/__pycache__/autoastrometry.cpython-310.pyc b/winterdrp/processors/autoastrometry/__pycache__/autoastrometry.cpython-310.pyc new file mode 100644 index 000000000..c1ad182dc Binary files /dev/null and b/winterdrp/processors/autoastrometry/__pycache__/autoastrometry.cpython-310.pyc differ diff --git a/winterdrp/processors/autoastrometry/__pycache__/autoastrometry_processor.cpython-310.pyc b/winterdrp/processors/autoastrometry/__pycache__/autoastrometry_processor.cpython-310.pyc new file mode 100644 index 000000000..d03971803 Binary files /dev/null and b/winterdrp/processors/autoastrometry/__pycache__/autoastrometry_processor.cpython-310.pyc differ diff --git a/winterdrp/processors/candidates/__pycache__/__init__.cpython-310.pyc b/winterdrp/processors/candidates/__pycache__/__init__.cpython-310.pyc new file mode 100644 index 000000000..b1e341972 Binary files /dev/null and b/winterdrp/processors/candidates/__pycache__/__init__.cpython-310.pyc differ diff --git a/winterdrp/processors/candidates/__pycache__/candidate_detector.cpython-310.pyc b/winterdrp/processors/candidates/__pycache__/candidate_detector.cpython-310.pyc new file mode 100644 index 000000000..4c922629e Binary files /dev/null and b/winterdrp/processors/candidates/__pycache__/candidate_detector.cpython-310.pyc differ diff --git a/winterdrp/processors/candidates/__pycache__/namer.cpython-310.pyc b/winterdrp/processors/candidates/__pycache__/namer.cpython-310.pyc new file mode 100644 index 000000000..611d41c96 Binary files /dev/null and b/winterdrp/processors/candidates/__pycache__/namer.cpython-310.pyc differ diff --git a/winterdrp/processors/candidates/utils/__pycache__/__init__.cpython-310.pyc b/winterdrp/processors/candidates/utils/__pycache__/__init__.cpython-310.pyc new file mode 100644 index 000000000..884051fa7 Binary files /dev/null and b/winterdrp/processors/candidates/utils/__pycache__/__init__.cpython-310.pyc differ diff --git a/winterdrp/processors/candidates/utils/__pycache__/dataframe_writer.cpython-310.pyc b/winterdrp/processors/candidates/utils/__pycache__/dataframe_writer.cpython-310.pyc new file mode 100644 index 000000000..bf3592d0c Binary files /dev/null and b/winterdrp/processors/candidates/utils/__pycache__/dataframe_writer.cpython-310.pyc differ diff --git a/winterdrp/processors/candidates/utils/__pycache__/regions_writer.cpython-310.pyc b/winterdrp/processors/candidates/utils/__pycache__/regions_writer.cpython-310.pyc new file mode 100644 index 000000000..caf680fad Binary files /dev/null and b/winterdrp/processors/candidates/utils/__pycache__/regions_writer.cpython-310.pyc differ diff --git a/winterdrp/processors/database/__pycache__/__init__.cpython-310.pyc b/winterdrp/processors/database/__pycache__/__init__.cpython-310.pyc new file mode 100644 index 000000000..96e781a6b Binary files /dev/null and b/winterdrp/processors/database/__pycache__/__init__.cpython-310.pyc differ diff --git a/winterdrp/processors/database/__pycache__/base_database_processor.cpython-310.pyc b/winterdrp/processors/database/__pycache__/base_database_processor.cpython-310.pyc new file mode 100644 index 000000000..23f9d9afc Binary files /dev/null and b/winterdrp/processors/database/__pycache__/base_database_processor.cpython-310.pyc differ diff --git a/winterdrp/processors/database/__pycache__/database_exporter.cpython-310.pyc b/winterdrp/processors/database/__pycache__/database_exporter.cpython-310.pyc new file mode 100644 index 000000000..5ebcc157c Binary files /dev/null and b/winterdrp/processors/database/__pycache__/database_exporter.cpython-310.pyc differ diff --git a/winterdrp/processors/database/__pycache__/database_importer.cpython-310.pyc b/winterdrp/processors/database/__pycache__/database_importer.cpython-310.pyc new file mode 100644 index 000000000..e396ae2c2 Binary files /dev/null and b/winterdrp/processors/database/__pycache__/database_importer.cpython-310.pyc differ diff --git a/winterdrp/processors/database/__pycache__/postgres.cpython-310.pyc b/winterdrp/processors/database/__pycache__/postgres.cpython-310.pyc new file mode 100644 index 000000000..45e116621 Binary files /dev/null and b/winterdrp/processors/database/__pycache__/postgres.cpython-310.pyc differ diff --git a/winterdrp/processors/photometry/__pycache__/aperture_photometry.cpython-310.pyc b/winterdrp/processors/photometry/__pycache__/aperture_photometry.cpython-310.pyc new file mode 100644 index 000000000..9276a099a Binary files /dev/null and b/winterdrp/processors/photometry/__pycache__/aperture_photometry.cpython-310.pyc differ diff --git a/winterdrp/processors/photometry/__pycache__/psf_photometry.cpython-310.pyc b/winterdrp/processors/photometry/__pycache__/psf_photometry.cpython-310.pyc new file mode 100644 index 000000000..e892846d5 Binary files /dev/null and b/winterdrp/processors/photometry/__pycache__/psf_photometry.cpython-310.pyc differ diff --git a/winterdrp/processors/photometry/__pycache__/utils.cpython-310.pyc b/winterdrp/processors/photometry/__pycache__/utils.cpython-310.pyc new file mode 100644 index 000000000..c477b3561 Binary files /dev/null and b/winterdrp/processors/photometry/__pycache__/utils.cpython-310.pyc differ diff --git a/winterdrp/processors/send_to_fritz.py b/winterdrp/processors/send_to_fritz.py index f9011e712..7af9fbbfe 100644 --- a/winterdrp/processors/send_to_fritz.py +++ b/winterdrp/processors/send_to_fritz.py @@ -1,9 +1,12 @@ import logging import requests +from requests.packages.urllib3.util.retry import Retry +from requests.adapters import HTTPAdapter import pandas as pd import os, gzip, io import base64 +from contextlib import contextmanager import astropy import json, time from datetime import datetime @@ -11,8 +14,17 @@ from astropy.io import ascii from astropy.io import fits from astropy.stats import sigma_clipped_stats +from astropy.visualization import ( + AsymmetricPercentileInterval, + LinearStretch, + LogStretch, + ImageNormalize, +) + +from copy import deepcopy import matplotlib.pyplot as plt import numpy as np +from typing import Mapping, Optional, Sequence from winterdrp.processors.base_processor import BaseDataframeProcessor @@ -20,40 +32,82 @@ logger = logging.getLogger(__name__) +DEFAULT_TIMEOUT = 5 # seconds + +class TimeoutHTTPAdapter(HTTPAdapter): + def __init__(self, *args, **kwargs): + self.timeout = DEFAULT_TIMEOUT + if "timeout" in kwargs: + self.timeout = kwargs["timeout"] + del kwargs["timeout"] + super().__init__(*args, **kwargs) + + def send(self, request, **kwargs): + try: + timeout = kwargs.get("timeout") + if timeout is None: + kwargs["timeout"] = self.timeout + return super().send(request, **kwargs) + except AttributeError: + kwargs["timeout"] = DEFAULT_TIMEOUT + class SendToFritz(BaseDataframeProcessor): def __init__(self, - output_sub_dir: str, - token = None, + update_thumbnails = True, group_ids = [1431], base_name = 'WIRC', - *args, + filter_id = 1152, + instrument_id = 5, + stream_id = 1005, + protocol = 'http', **kwargs): - super(SendToFritz, self).__init__(*args, **kwargs) + super(SendToFritz, self).__init__(**kwargs) self.token = None self.group_ids = group_ids self.base_name = base_name - self.origin = base_name + self.filter_id = filter_id + self.instrument_id = instrument_id + self.origin = base_name # used for sending updates to Fritz + self.stream_id = stream_id + self.protocol = protocol + self.update_thumbnails = update_thumbnails + + def _get_fritz_token(): + token_fritz = os.getenv("FRITZ_TOKEN") + + if token_fritz is None: + err = "No Fritz token specified. Run 'export FRITZ_TOKEN=' to set. " \ + "The Fritz token will need to be specified manually for Fritz API queries." + logger.warning(err) + raise ValueError + + return token_fritz + + # session to talk to SkyPortal/Fritz + self.session = requests.Session() + self.session_headers = { + "Authorization": f"token {_get_fritz_token()}", + "User-Agent": f"winterdrp", + } + + retries = Retry( + total=5, + backoff_factor=2, + status_forcelist=[405, 429, 500, 502, 503, 504], + method_whitelist=["HEAD", "GET", "PUT", "POST", "PATCH"], + ) + adapter = TimeoutHTTPAdapter(timeout=5, max_retries=retries) + self.session.mount("https://", adapter) + self.session.mount("http://", adapter) def _apply_to_candidates( self, candidate_table: pd.DataFrame, ) -> pd.DataFrame: logger.info("In SendToFritz") - self.token = self._get_fritz_token() self.make_alert(candidate_table) return candidate_table - def _get_fritz_token(self): - token_fritz = os.getenv("FRITZ_TOKEN") - - if token_fritz is None: - err = "No Fritz token specified. Run 'export FRITZ_TOKEN=' to set. " \ - "The Fritz token will need to be specified manually for Fritz API queries." - logger.warning(err) - raise ValueError - - return token_fritz - def _get_author_id(self): """Fritz author id is used in update calls. Can be found """ @@ -67,25 +121,11 @@ def _get_author_id(self): return authid_fritz - def open_bytes_obj(self, bytes_obj): - """Return numpy array of bytes_obj - - Args: - bytes_obj (_io.BytesIO object in memory): BytesIO obj representing image. - - Returns: - numpy.ndarrary: representation of the image - """ - bytes_io = io.BytesIO(gzip.open(io.BytesIO(bytes_obj.getvalue())).read()) # io.BytesIO obj, ready to be read by fits.open - cutout = fits.open(bytes_io)[0].data - return cutout - def read_input_df(self, df): """Takes a DataFrame, which has multiple candidate and creates list of dictionaries, each dictionary representing a single candidate. - NOTE: saves images as np.arrays for thumbnail sending. Args: df (pandas.core.frame.DataFrame): dataframe of all candidates. @@ -99,104 +139,216 @@ def read_input_df(self, df): candidate = {} for key in df.keys(): try: - if type(df.iloc[i].get(key)) is str: + if type(df.iloc[i].get(key)) is str or type(df.iloc[i].get(key)) is list: candidate[key] = df.iloc[i].get(key) else: # change to native python type candidate[key] = df.iloc[i].get(key).item() except AttributeError: # for IOBytes objs - candidate[key] = self.open_bytes_obj(df.iloc[i].get(key)) - + candidate[key] = df.iloc[i].get(key).getvalue() + all_candidates.append(candidate) return all_candidates - def get_next_name(self, lastname, candjd, bwfile = 'badwords.txt', begcount = 'aaaaaaa'): - """Creates candidate name following the naming format of 'WNTR22aaaaaaa' . - Modified from https://github.com/dekishalay/pgirdps + def api(self, method: str, endpoint: str, data: Optional[Mapping] = None): + """Make an API call to a SkyPortal instance - Args: - lastname(str): last used candidate name. - candjd (str): candidate's JD. - bwfile (str): file name of .txt file of excluded words. - begcount (str): string to start naming convention. - - Returns: - (str): next candidate name in sequence. - """ - curyear = Time(candjd, format = 'jd').datetime.strftime('%Y')[2:4] - - if lastname is None: - #If this is the first source being named - newname = self.base_name + curyear + begcount - return newname - - lastyear = lastname[4:6] + headers = {'Authorization': f'token {self.token}'} + response = requests.request(method, endpoint, json=data, headers=headers) + + :param method: + :param endpoint: + :param data: + :return: + """ + method = method.lower() + methods = { + "head": self.session.head, + "get": self.session.get, + "post": self.session.post, + "put": self.session.put, + "patch": self.session.patch, + "delete": self.session.delete, + } - if curyear != lastyear: - #If this is the first candidate of the new year, start with aaaaa - newname = self.base_name + curyear + begcount - return newname + if endpoint is None: + raise ValueError("Endpoint not specified") + if method not in ["head", "get", "post", "put", "patch", "delete"]: + raise ValueError(f"Unsupported method: {method}") + + if method == "get": + response = methods[method]( + f"{endpoint}", + params=data, + headers=self.session_headers, + ) else: - lastcount = lastname[6:] - charpos = len(lastcount) - 1 - # will iteratively try to increment characters starting from the last - inctrue = False - usestring = '' - while charpos >= 0: - cref = lastcount[charpos] - if inctrue: - usestring = cref + usestring - charpos -= 1 - continue - creford = ord(cref) - #increment each character, if at 'z', increment the next one - if creford + 1 > 122: - usestring = 'a' + usestring - else: - nextchar = chr(creford+1) - usestring = nextchar + usestring - inctrue = True - charpos -= 1 - continue - - newname = self.base_name + curyear + usestring - - curdir = os.path.dirname(__file__) # /data/sulekha/winterdrp/winterdrp/processors/alert_packets - file_path = os.path.join(curdir, bwfile) - - bwlist = ascii.read(file_path, format = 'no_header') - isbw = False - # check for bad word - for i in range(len(bwlist)): - if usestring.find(str(bwlist['col1'][i])) != -1: - #Shame shame - isbw = True - break - if isbw: - # increment the name with a recursive call - return self.get_next_name(newname, candjd) - else: - return newname + response = methods[method]( + f"{endpoint}", + json=data, + headers=self.session_headers, + ) - def api(self, method, endpoint, data=None): - headers = {'Authorization': f'token {self.token}'} - response = requests.request(method, endpoint, json=data, headers=headers) return response - def add_new_source(self, cand): - data = {'ra': cand['ra'], - 'dec': cand['dec'], - 'id': cand['objectId'], - 'group_ids': self.group_ids + def alert_post_source(self, alert, group_ids = None): + """Add a new source to SkyPortal + + :param alert: dict of source info + :param group_ids: list of group_ids to post source to, defaults to None + """ + if group_ids is None: + group_ids = self.group_ids + + + data = {'ra': alert['ra'], + 'dec': alert['dec'], + 'id': alert['objectId'], + 'group_ids': group_ids, + 'origin': self.origin } - logger.info(f"dict:{data}") + + logger.info(f"Saving {alert['objectId']} {alert['candid']} as a Source on SkyPortal") response = self.api('POST', 'https://fritz.science/api/sources', data) - return response + + if response.json()["status"] == "success": + logger.info(f"Saved {alert['objectId']} {alert['candid']} as a Source on SkyPortal") + else: + logger.info( + f"Failed to save {alert['objectId']} {alert['candid']} as a Source on SkyPortal" + ) + logger.info(response.json()) - def upload_thumbnail(self, cand): + def alert_post_candidate(self, alert): + """ + Post a candidate on SkyPortal. Creates new candidate(s) (one per filter) + """ + data = { "id": alert["objectId"], + "ra": alert["ra"], + "dec": alert["dec"], + "filter_ids": [self.filter_id], + "passing_alert_id": self.filter_id, + "passed_at": Time(datetime.utcnow()).isot, + "origin": "WINTERdrp", + } + + logger.info(f"Posting metadata of {alert['objectId']} {alert['candid']} to SkyPortal") + response = self.api("POST", "https://fritz.science/api/candidates", data) + + if response.json()["status"] == "success": + logger.info(f"Posted {alert['objectId']} {alert['candid']} metadata to SkyPortal") + else: + logger.info(f"Failed to post {alert['objectId']} {alert['candid']} metadata to SkyPortal") + logger.info(response.json()) + + def make_thumbnail( + self, alert, skyportal_type: str, alert_packet_type: str + ): + """ + Convert lossless FITS cutouts from ZTF-like alerts into PNGs. + Make thumbnail for pushing to SkyPortal. + + :param alert: ZTF-like alert packet/dict + :param skyportal_type: thumbnail type expected by SkyPortal + :param alert_packet_type: survey naming + :return: + """ + alert = deepcopy(alert) + cutout_data = alert[f"cutout{alert_packet_type}"] + + with gzip.open(io.BytesIO(cutout_data), "rb") as f: + with fits.open(io.BytesIO(f.read()), ignore_missing_simple=True) as hdu: + image_data = hdu[0].data + + # Survey-specific transformations to get North up and West on the right + # if self.instrument == "ZTF": + # image_data = np.flipud(image_data) + # elif self.instrument == "PGIR": + # image_data = np.rot90(np.fliplr(image_data), 3) + + buff = io.BytesIO() + plt.close("all") + fig = plt.figure() + fig.set_size_inches(4, 4, forward=False) + ax = plt.Axes(fig, [0.0, 0.0, 1.0, 1.0]) + ax.set_axis_off() + fig.add_axes(ax) + + # replace nans with median: + img = np.array(image_data) + # replace dubiously large values + xl = np.greater(np.abs(img), 1e20, where=~np.isnan(img)) + if img[xl].any(): + img[xl] = np.nan + if np.isnan(img).any(): + median = float(np.nanmean(img.flatten())) + img = np.nan_to_num(img, nan=median) + + norm = ImageNormalize( + img, + stretch=LinearStretch() + if alert_packet_type == "Difference" + else LogStretch(), + ) + img_norm = norm(img) + normalizer = AsymmetricPercentileInterval( + lower_percentile=1, upper_percentile=100 + ) + vmin, vmax = normalizer.get_limits(img_norm) + ax.imshow(img_norm, cmap="bone", origin="lower", vmin=vmin, vmax=vmax) + plt.savefig(buff, dpi=42) + + buff.seek(0) + plt.close("all") + + thumbnail_dict = { + "obj_id": alert["objectId"], + "data": base64.b64encode(buff.read()).decode("utf-8"), + "ttype": skyportal_type, + } + + return thumbnail_dict + + + def alert_post_thumbnails(self, alert): + """Post alert Science, Reference, and Subtraction thumbnails to SkyPortal + + :param alert: dict of source/candidate information + :return: + """ + for ttype, instrument_type in [ + ("new", "Science"), + ("ref", "Template"), + ("sub", "Difference"), + ]: + logger.info( + f"Making {instrument_type} thumbnail for {alert['objectId']} {alert['candid']}", + + ) + thumb = self.make_thumbnail(alert, ttype, instrument_type) + + logger.info( + f"Posting {instrument_type} thumbnail for {alert['objectId']} {alert['candid']} to SkyPortal", + ) + response = self.api("POST", "https://fritz.science/api/thumbnail", thumb) + + if response.json()["status"] == "success": + logger.info( + f"Posted {alert['objectId']} {alert['candid']} {instrument_type} cutout to SkyPortal" + ) + else: + logger.info( + f"Failed to post {alert['objectId']} {alert['candid']} {instrument_type} cutout to SkyPortal" + ) + logger.info(response.json()) + + def upload_thumbnail(self, alert): """Post new thumbnail to Fritz. + NOTE: this is the original WINTER method for sending thumbnails, + not full sized but higher contrast, similar to alert_make_thumbnail + Format of thumbnail payload: { "obj_id": "string", "data": "string", "ttype": "string"} """ @@ -204,7 +356,7 @@ def upload_thumbnail(self, cand): for fritz_key in fritz_to_cand.keys(): cand_key = fritz_to_cand[fritz_key] - cutout = cand[cand_key] + cutout = alert[cand_key] buffer = io.BytesIO() plt.figure(figsize=(3,3)) @@ -218,144 +370,327 @@ def upload_thumbnail(self, cand): cutoutb64 = base64.b64encode(buffer.getvalue()) cutoutb64_string = cutoutb64.decode('utf8') - data_payload = {'obj_id':cand['objectId'], + data_payload = {'obj_id':alert['objectId'], 'data':cutoutb64_string, 'ttype':fritz_key } response = self.api('POST', 'https://fritz.science/api/thumbnail', data=data_payload) - # logger.info(f'candid {data_payload["obj_id"]}: {data_payload["ttype"]}, thumbnail response:{response}') - return response - def update_photometry(self, cand): - """Send photometry to Fritz.""" - data_payload = {"filter": "cspjs", - "magsys": "vega", - "obj_id": cand["objectId"], - "mag": cand["magpsf"], - "magerr": cand["sigmapsf"], - "instrument_id": 5, - "mjd": cand['jd'] - 2400000.5, - "limiting_mag": 99, - "group_ids": self.group_ids - } - # response = self.api('PATCH', 'https://fritz.science/api/photometry/photometry_id', data=data_payload) - response = self.api('PUT', 'https://fritz.science/api/photometry', data=data_payload) + if response.json()["status"] == "success": + logger.info( + f"Posted {alert['objectId']} {alert['candid']} {cand_key} cutout to SkyPortal" + ) + else: + logger.info( + f"Failed to post {alert['objectId']} {alert['candid']} {cand_key} cutout to SkyPortal" + ) + logger.info(response.json()) - # logger.info(f'candid {data_payload["obj_id"]} photo response:{response.text}') - return response + def make_photometry(self, alert, jd_start: float = None): + """ + Make a de-duplicated pandas.DataFrame with photometry of alert['objectId'] + Modified from Kowalksi (https://github.com/dmitryduev/kowalski) - def create_new_cand(self, cand, id): - """Create new candidate(s) (one per filter)""" - data = { "id": cand["objectId"], - "filter_ids": [1152], - "passing_alert_id": 1152, - "passed_at": Time(datetime.utcnow()).isot, - "ra": cand["ra"], - "dec": cand["dec"], - } - response = self.api('POST','https://fritz.science/api/candidates',data=data) - # response = self.api('POST', 'https://fritz.science/#tag/candidates/paths/~1api~1candidates/get/api/candidates',data=data) - # logger.info(f'new create response {response.text}') - return response + :param alert: candidate dictionary + :param jd_start: date from which to start photometry from + """ + # print(alert.keys()) + alert = deepcopy(alert) + # df_candidate = pd.DataFrame(alert["candidate"], index=[0]) + top_level = ["schemavsn","publisher","objectId", + "candid","candidate","prv_candidates", + "cutoutScience","cutoutTemplate","cutoutDifference"] + alert["candidate"] = {} + + # (keys having value in 3.) + delete = [key for key in alert.keys() if key not in top_level] + + # delete the key/s + for key in delete: + alert["candidate"][key] = alert[key] + del alert[key] + + alert["candidate"] = [alert["candidate"]] + df_candidate = pd.DataFrame(alert["candidate"], index=[0]) - def retrieve_cand(self, cand): - """Checks whether a candidate already exist.""" - path = 'https://fritz.science/api/candidates/' + str(cand["objectId"]) - response = self.api('GET', path) - def update_annotation(self, cand, annotation_id): - """Update an annotation.""" - # https://fritz.science/api/associated_resource_type/resource_id/annotations/annotation_id - path_parms = os.path.join(cand["objectId"], "annotations", str(annotation_id)) - path = 'https://fritz.science/api/sources/' + path_parms + df_prv_candidates = pd.DataFrame(alert["prv_candidates"]) - data = {"fwhm":cand["fwhm"], - "scorr": cand["scorr"], - "chipsf": cand["chipsf"] - } - authid = self._get_author_id() - payload = {"data": data, - "origin": self.origin, - "author_id": authid, - "obj_id": cand["objectId"], - "group_ids": self.group_ids + df_light_curve = pd.concat( + [df_candidate, df_prv_candidates], ignore_index=True, sort=False + ) + + # note: WNTR (like PGIR) uses 2massj, which is not in sncosmo as of + # 20210803, cspjs seems to be close/good enough as an approximation + df_light_curve["filter"] = "cspjs" + + df_light_curve["magsys"] = "ab" + df_light_curve["mjd"] = df_light_curve["jd"] - 2400000.5 + + df_light_curve["mjd"] = df_light_curve["mjd"].apply(lambda x: np.float64(x)) + df_light_curve["magpsf"] = df_light_curve["magpsf"].apply( + lambda x: np.float32(x) + ) + df_light_curve["sigmapsf"] = df_light_curve["sigmapsf"].apply( + lambda x: np.float32(x) + ) + + df_light_curve = ( + df_light_curve.drop_duplicates(subset=["mjd", "magpsf"]) + .reset_index(drop=True) + .sort_values(by=["mjd"]) + ) + + # TODO add back once diffmaglim in df + # # filter out bad data: + # mask_good_diffmaglim = df_light_curve["diffmaglim"] > 0 + # df_light_curve = df_light_curve.loc[mask_good_diffmaglim] + + # # convert from mag to flux + + # # step 1: calculate the coefficient that determines whether the + # # flux should be negative or positive + # coeff = df_light_curve["isdiffpos"].apply( + # lambda x: 1.0 if x in [True, 1, "y", "Y", "t", "1"] else -1.0 + # ) + + + # # step 2: calculate the flux normalized to an arbitrary AB zeropoint of + # # 23.9 (results in flux in uJy) + # df_light_curve["flux"] = coeff * 10 ** ( + # -0.4 * (df_light_curve["magpsf"] - 23.9) + # ) + + # # step 3: separate detections from non detections + # detected = np.isfinite(df_light_curve["magpsf"]) + # undetected = ~detected + + # # step 4: calculate the flux error + # df_light_curve["fluxerr"] = None # initialize the column + + # # step 4a: calculate fluxerr for detections using sigmapsf + # df_light_curve.loc[detected, "fluxerr"] = np.abs( + # df_light_curve.loc[detected, "sigmapsf"] + # * df_light_curve.loc[detected, "flux"] + # * np.log(10) + # / 2.5 + # ) + + # # step 4b: calculate fluxerr for non detections using diffmaglim + # df_light_curve.loc[undetected, "fluxerr"] = ( + # 10 ** (-0.4 * (df_light_curve.loc[undetected, "diffmaglim"] - 23.9)) / 5.0 + # ) # as diffmaglim is the 5-sigma depth + + # # step 5: set the zeropoint and magnitude system + # df_light_curve["zp"] = 23.9 + # df_light_curve["zpsys"] = "ab" + + # # only "new" photometry requested? + # if jd_start is not None: + # w_after_jd = df_light_curve["jd"] > jd_start + # df_light_curve = df_light_curve.loc[w_after_jd] + + return df_light_curve + + def alert_put_photometry(self, alert): + """Send photometry to Fritz.""" + logger.info(f"Making alert photometry of {alert['objectId']} {alert['candid']}") + df_photometry = self.make_photometry(alert) + + # post photometry + photometry = { + "obj_id": alert["objectId"], + "stream_ids": [int(self.stream_id)], + "instrument_id": self.instrument_id, + "mjd": df_photometry["mjd"].tolist(), + # TODO uncomment once added (see make_photometry()) + # "flux": df_photometry["flux"].tolist(), + # "fluxerr": df_photometry["fluxerr"].tolist(), + # "zp": df_photometry["zp"].tolist(), + # "magsys": df_photometry["zpsys"].tolist() + # ## hardcoded ##, + "magsys": "vega", + "limiting_mag": 99, + "mag": alert["magpsf"], + "magerr": alert["sigmapsf"], + # ## end of hard coding ## + "filter": df_photometry["filter"].tolist(), + "ra": df_photometry["ra"].tolist(), + "dec": df_photometry["dec"].tolist(), } - response = self.api('PUT', path, payload) - logger.info(f'update annotation status: {response.json()["status"]}') - # logger.info(f'update message: {response.text}') - return response + # TODO uncomment + # if (len(photometry.get("flux", ())) > 0) or ( + # len(photometry.get("fluxerr", ())) > 0 + # ): + logger.info(f"Posting photometry of {alert['objectId']} {alert['candid']}, " + f"stream_id={self.stream_id} to SkyPortal") + response = self.api("PUT", "https://fritz.science/api/photometry", photometry) + if response.json()["status"] == "success": + logger.info( + f"Posted {alert['objectId']} photometry stream_id={self.stream_id} to SkyPortal" + ) + else: + logger.info( + f"Failed to post {alert['objectId']} photometry stream_id={self.stream_id} to SkyPortal" + ) + logger.info(response.json()) - def post_annotation(self, cand): - """Post an annotation. For new cand? + def alert_post_annotation(self, alert): + """Post an annotation. Works for both candidates and sources. """ - data = {"chipsf": cand["chipsf"], - "fwhm": cand["fwhm"], - "scorr": cand["scorr"]} + data = {"chipsf": alert["chipsf"], + "fwhm": alert["fwhm"], + "scorr": alert["scorr"]} payload = {"origin": self.origin, "data": data, "group_ids": self.group_ids } - path = 'https://fritz.science/api/sources/' + str(cand["objectId"]) + '/annotations' + path = f'https://fritz.science/api/sources/{str(alert["objectId"])}/annotations' response = self.api('POST', path, payload) - logger.info(f'candid {cand["objectId"]} annotation response:{response.text}') - def retrieve_annotation_specified_source(self, cand): + if response.json()["status"] == "success": + logger.info(f"Posted {alert['objectId']} annotation to SkyPortal") + else: + logger.info(f"Failed to post {alert['objectId']} annotation to SkyPortal") + logger.info(response.json()) + + def alert_put_annotation(self, alert): """Retrieve an annotation to check if it exists already.""" - resource_id = str(cand["objectId"]) - path = 'https://fritz.science/api/sources/' + resource_id+ '/annotations' - response = self.api('GET', path) - - # logger.info(f'json {response}') - json_response= response.json() - - annotation_posted = False - if json_response["status"] == "success": - logger.info(f'Annotation for {cand["objectId"]} already exists...updating') - origins = np.array([x["origin"] for x in json_response["data"]]) - if self.origin in origins: - annotation_ids = np.array([x["id"] for x in json_response["data"]]) - annotation_id = annotation_ids[origins == self.origin][0] - self.update_annotation(cand, annotation_id) - annotation_posted = True + response = self.api('GET', f'https://fritz.science/api/sources/{str(alert["objectId"])}/annotations') + + if response.json()["status"] == "success": + logger.info(f"Got {alert['objectId']} annotations from SkyPortal") + else: + logger.info(f"Failed to get {alert['objectId']} annotations from SkyPortal") + logger.info(response.json()) + return False + + existing_annotations = { + annotation["origin"]: { + "annotation_id": annotation["id"], + "author_id": annotation["author_id"], + } + for annotation in response.json()["data"] + } + + # no previous annotation exists on SkyPortal for this object? just post then + if self.origin not in existing_annotations: + self.alert_post_annotation(alert) + # annotation from this(WNTR) origin exists + else: + # annotation data + data = {"fwhm":alert["fwhm"], + "scorr": alert["scorr"], + "chipsf": alert["chipsf"] + } + new_annotation = { + "author_id": existing_annotations[self.origin]["author_id"], + "obj_id": alert["objectId"], + "origin": self.origin, + "data": data, + "group_ids": self.group_ids, + } - if not annotation_posted: - self.post_annotation(cand) + logger.info( + f"Putting annotation for {alert['objectId']} {alert['candid']} to SkyPortal", + ) + response = self.api( + "PUT", + f"https://fritz.science/api/sources/{alert['objectId']}" + f"/annotations/{existing_annotations[self.origin]['annotation_id']}", + new_annotation, + ) + if response.json()["status"] == "success": + logger.info(f"Posted updated {alert['objectId']} annotation to SkyPortal") + else: + logger.info(f"Failed to post updated {alert['objectId']} annotation to SkyPortal") + logger.info(response.json()) + + def alert_skyportal_manager(self, alert): + """Posts alerts to SkyPortal if criteria is met + + :param alert: _description_ + :type alert: _type_ + """ + # check if candidate exists in SkyPortal + logger.info(f"Checking if {alert['objectId']} is candidate in SkyPortal") + response = self.api("HEAD", f"https://fritz.science/api/candidates/{alert['objectId']}") + is_candidate = response.status_code == 200 + logger.info(f"{alert['objectId']} {'is' if is_candidate else 'is not'} candidate in SkyPortal") + + # check if source exists in SkyPortal + logger.info(f"Checking if {alert['objectId']} is source in SkyPortal") + response = self.api("HEAD", f"https://fritz.science/api/sources/{alert['objectId']}") + is_source = response.status_code == 200 + logger.info(f"{alert['objectId']} {'is' if is_source else 'is not'} source in SkyPortal") + + # object does not exist in SkyPortal: neither cand nor source + if (not is_candidate) and (not is_source): + # post candidate + self.alert_post_candidate(alert) + + # post annotations + self.alert_post_annotation(alert) + + # post full light curve + self.alert_put_photometry(alert) + + # post thumbnails + self.alert_post_thumbnails(alert) + + # TODO autosave stuff, necessary? + + # obj already exists in SkyPortal + else: + # TODO passed_filters logic + + # post candidate with new filter ids + self.alert_post_candidate(alert) + + # put (*not* post) annotations + self.alert_put_annotation(alert) + + # exists in SkyPortal & already saved as a source + if is_source: + # get info on the corresponding groups: + logger.info( + f"Getting source groups info on {alert['objectId']} from SkyPortal", + ) + response = self.api( + "GET", f"https://fritz.science/api/sources/{alert['objectId']}/groups" + ) + if response.json()["status"] == "success": + existing_groups = response.json()["data"] + existing_group_ids = [g["id"] for g in existing_groups] + + for existing_gid in existing_group_ids: + if existing_gid in self.group_ids: + self.alert_post_source(alert, str(existing_gid)) + else: + logger.info(f"Failed to get source groups info on {alert['objectId']}") + else: # exists in SkyPortal but NOT saved as a source + self.alert_post_source(alert) + + # post alert photometry in single call to /api/photometry + self.alert_put_photometry(alert) + + if self.update_thumbnails: + self.alert_post_thumbnails(alert) + + logger.info(f'======== SendToFritz Manager complete for {alert["objectId"]} =======') def make_alert(self, cand_table): + t0 = time.time() all_cands = self.read_input_df(cand_table) - - last_name = 'WIRC21aaaaaao' - cand_id = 700 + num_cands = len(all_cands) - for cand in all_cands: - cand_jd = cand['jd'] # float - cand_name = self.get_next_name(last_name, str(cand_jd)) + for cand in all_cands: + self.alert_skyportal_manager(cand) - #TODO candid should be coming from naming database - cand['candid'] = cand_id - cand_id += 1 - - # TODO check if cand is new? - # cand_name, new_status = self.check_and_insert_source(cand_name, cand) - - #TODO remove once new_status is up-to-date: dummy line - new_status = True - - if new_status: - last_name = cand_name - cand['objectId'] = cand_name - - # source_response = self.add_new_source(cand) - # source_response = self.create_new_cand(cand, id) - # logger.info(f'Add source {cand["objectId"]}: {source_response}') - # thumbnail_response = self.upload_thumbnail(cand) - # logger.info(f'Upload thumbnail {cand["objectId"]}: {thumbnail_response}') - # photometry_response = self.update_photometry(cand) - # logger.info(f'Photometry {cand["objectId"]}: {photometry_response}') - - self.retrieve_annotation_specified_source(cand) - # annotation_response = self.post_annotation(cand) - \ No newline at end of file + t1 = time.time() + logger.info('###########################################') + logger.info(f"Took {(t1 - t0):.2f} seconds to Fritz process {num_cands} candidates.") + logger.info('###########################################') \ No newline at end of file diff --git a/winterdrp/processors/utils/__pycache__/__init__.cpython-310.pyc b/winterdrp/processors/utils/__pycache__/__init__.cpython-310.pyc new file mode 100644 index 000000000..fad576529 Binary files /dev/null and b/winterdrp/processors/utils/__pycache__/__init__.cpython-310.pyc differ diff --git a/winterdrp/processors/utils/__pycache__/header_annotate.cpython-310.pyc b/winterdrp/processors/utils/__pycache__/header_annotate.cpython-310.pyc new file mode 100644 index 000000000..5bbea8bc5 Binary files /dev/null and b/winterdrp/processors/utils/__pycache__/header_annotate.cpython-310.pyc differ diff --git a/winterdrp/processors/utils/__pycache__/header_reader.cpython-310.pyc b/winterdrp/processors/utils/__pycache__/header_reader.cpython-310.pyc new file mode 100644 index 000000000..f32fa0160 Binary files /dev/null and b/winterdrp/processors/utils/__pycache__/header_reader.cpython-310.pyc differ diff --git a/winterdrp/processors/utils/__pycache__/image_loader.cpython-310.pyc b/winterdrp/processors/utils/__pycache__/image_loader.cpython-310.pyc new file mode 100644 index 000000000..d1735adaa Binary files /dev/null and b/winterdrp/processors/utils/__pycache__/image_loader.cpython-310.pyc differ diff --git a/winterdrp/processors/utils/__pycache__/image_rejector.cpython-310.pyc b/winterdrp/processors/utils/__pycache__/image_rejector.cpython-310.pyc new file mode 100644 index 000000000..f5fb8926d Binary files /dev/null and b/winterdrp/processors/utils/__pycache__/image_rejector.cpython-310.pyc differ diff --git a/winterdrp/processors/utils/__pycache__/image_saver.cpython-310.pyc b/winterdrp/processors/utils/__pycache__/image_saver.cpython-310.pyc new file mode 100644 index 000000000..32001247f Binary files /dev/null and b/winterdrp/processors/utils/__pycache__/image_saver.cpython-310.pyc differ diff --git a/winterdrp/processors/utils/__pycache__/image_selector.cpython-310.pyc b/winterdrp/processors/utils/__pycache__/image_selector.cpython-310.pyc new file mode 100644 index 000000000..54cbd8b5a Binary files /dev/null and b/winterdrp/processors/utils/__pycache__/image_selector.cpython-310.pyc differ diff --git a/winterdrp/processors/zogy/__pycache__/py_zogy.cpython-310.pyc b/winterdrp/processors/zogy/__pycache__/py_zogy.cpython-310.pyc new file mode 100644 index 000000000..356e3386f Binary files /dev/null and b/winterdrp/processors/zogy/__pycache__/py_zogy.cpython-310.pyc differ diff --git a/winterdrp/processors/zogy/__pycache__/zogy.cpython-310.pyc b/winterdrp/processors/zogy/__pycache__/zogy.cpython-310.pyc new file mode 100644 index 000000000..05b9201b6 Binary files /dev/null and b/winterdrp/processors/zogy/__pycache__/zogy.cpython-310.pyc differ diff --git a/winterdrp/references/__pycache__/__init__.cpython-310.pyc b/winterdrp/references/__pycache__/__init__.cpython-310.pyc new file mode 100644 index 000000000..041ed5866 Binary files /dev/null and b/winterdrp/references/__pycache__/__init__.cpython-310.pyc differ diff --git a/winterdrp/references/__pycache__/base_reference_generator.cpython-310.pyc b/winterdrp/references/__pycache__/base_reference_generator.cpython-310.pyc new file mode 100644 index 000000000..113fb678b Binary files /dev/null and b/winterdrp/references/__pycache__/base_reference_generator.cpython-310.pyc differ diff --git a/winterdrp/references/__pycache__/wirc.cpython-310.pyc b/winterdrp/references/__pycache__/wirc.cpython-310.pyc new file mode 100644 index 000000000..5ca6d9348 Binary files /dev/null and b/winterdrp/references/__pycache__/wirc.cpython-310.pyc differ diff --git a/winterdrp/swarp.xml b/winterdrp/swarp.xml new file mode 100644 index 000000000..983f05a57 --- /dev/null +++ b/winterdrp/swarp.xml @@ -0,0 +1,139 @@ + + + +produced by SWarp + + + Data related to SWarp + + + + SWarp meta-data + + + + + + + + + + + + + + + + + + + + Data gathered by SWarp for every FITS input image + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
1/data/sulekha/test/wirc_imsub/20220420/subtract/temp_ZTF21aagppzg_J_stack_1_20210330.fits/data/sulekha/test/wirc_imsub/20220420/subtract/temp_ZTF21aagppzg_J_stack_1_20210330.mask.fitsFZTF21aagppzg02022-07-2811:59:250.000.001586920.813738TAUTO12830MAP_WEIGHTT00.813738F120.57456.09931511160.6101024 34.397699380.2487432021.24261120002000ICRS
+ + SWarp warnings (limited to the last 1000) + + + + + + + +
2022-07-2811:59:25winterdrp/pipelines/wirc_imsub/wirc_imsub_files/config/config.swarp not found, using internal defaults
2022-07-2811:59:25GAIN keyword unknown
+ + SWarp configuration + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/winterdrp/utils/__pycache__/__init__.cpython-310.pyc b/winterdrp/utils/__pycache__/__init__.cpython-310.pyc new file mode 100644 index 000000000..5f4ac7030 Binary files /dev/null and b/winterdrp/utils/__pycache__/__init__.cpython-310.pyc differ diff --git a/winterdrp/utils/__pycache__/dockerutil.cpython-310.pyc b/winterdrp/utils/__pycache__/dockerutil.cpython-310.pyc new file mode 100644 index 000000000..7d1322e01 Binary files /dev/null and b/winterdrp/utils/__pycache__/dockerutil.cpython-310.pyc differ diff --git a/winterdrp/utils/__pycache__/execute_cmd.cpython-310.pyc b/winterdrp/utils/__pycache__/execute_cmd.cpython-310.pyc new file mode 100644 index 000000000..27f07152a Binary files /dev/null and b/winterdrp/utils/__pycache__/execute_cmd.cpython-310.pyc differ diff --git a/winterdrp/utils/__pycache__/fits_tools.cpython-310.pyc b/winterdrp/utils/__pycache__/fits_tools.cpython-310.pyc new file mode 100644 index 000000000..ca7bbae69 Binary files /dev/null and b/winterdrp/utils/__pycache__/fits_tools.cpython-310.pyc differ diff --git a/winterdrp/utils/__pycache__/ldac_tools.cpython-310.pyc b/winterdrp/utils/__pycache__/ldac_tools.cpython-310.pyc new file mode 100644 index 000000000..3b1cbb8e8 Binary files /dev/null and b/winterdrp/utils/__pycache__/ldac_tools.cpython-310.pyc differ