From 2df3c3c4110214ae38b36388e45c922d544deb70 Mon Sep 17 00:00:00 2001 From: Robert Stein Date: Mon, 10 Oct 2022 14:26:22 -0700 Subject: [PATCH 1/8] basename will be comma-joined list --- .../processors/astromatic/swarp/swarp.py | 42 +++++++++++++------ 1 file changed, 29 insertions(+), 13 deletions(-) diff --git a/winterdrp/processors/astromatic/swarp/swarp.py b/winterdrp/processors/astromatic/swarp/swarp.py index 5d39c7267..b62f5a77f 100644 --- a/winterdrp/processors/astromatic/swarp/swarp.py +++ b/winterdrp/processors/astromatic/swarp/swarp.py @@ -104,8 +104,8 @@ def __init__( center_dec: float = None, gain: float = None, include_scamp: bool = True, - combine: bool =False, - cache: bool =False, + combine: bool = False, + cache: bool = False, subtract_bkg: bool = False, *args, **kwargs @@ -167,8 +167,6 @@ def _apply_to_images( ) logger.debug(f"Saving to {output_image_path}") - out_files = [] - all_pixscales = [] all_imgpixsizes = [] all_ras = [] @@ -177,9 +175,19 @@ def _apply_to_images( for i, data in enumerate(images): header = headers[i] - pixscale_to_use, x_imgpixsize_to_use, y_imgpixsize_to_use, center_ra_to_use, center_dec_to_use = None, None, None, None, None - - if not ((self.pixscale is None) | (self.x_imgpixsize is None) | (self.y_imgpixsize is None) | (self.center_ra is None) | (self.center_dec is None)): + pixscale_to_use = None + x_imgpixsize_to_use = None + y_imgpixsize_to_use = None + center_ra_to_use = None + center_dec_to_use = None + + if not ( + (self.pixscale is None) | + (self.x_imgpixsize is None) | + (self.y_imgpixsize is None) | + (self.center_ra is None) | + (self.center_dec is None) + ): pixscale_to_use = self.pixscale x_imgpixsize_to_use = self.x_imgpixsize y_imgpixsize_to_use = self.y_imgpixsize @@ -187,25 +195,32 @@ def _apply_to_images( center_dec_to_use = self.center_dec else: w = WCS(header) + cd11 = header['CD1_1'] cd21 = header['CD2_1'] cd12 = header['CD1_2'] cd22 = header['CD2_2'] + nxpix = header['NAXIS1'] nypix = header['NAXIS2'] + image_x_cen = nxpix / 2 image_y_cen = nypix / 2 + [ra, dec] = w.all_pix2world(image_x_cen, image_y_cen, 1) + xscale = np.sqrt(cd11 ** 2 + cd21 ** 2) yscale = np.sqrt(cd12 ** 2 + cd22 ** 2) pixscale = xscale * 3600 imgpixsize = max(nxpix, nypix) + all_pixscales.append(pixscale) all_imgpixsizes.append(imgpixsize) all_ras.append(ra) all_decs.append(dec) - logger.info(f"{all_ras}, {all_decs}") + + logger.debug(f"{all_ras}, {all_decs}") if self.include_scamp: temp_head_path = copy_temp_file( @@ -213,8 +228,10 @@ def _apply_to_images( file_path=str(header[scamp_header_key]).replace('\n', '') ) - temp_img_path = get_temp_path(swarp_output_dir, header["BASENAME"]) + temp_img_path = get_temp_path(swarp_output_dir, header[base_name_key]) + self.save_fits(data, header, temp_img_path) + temp_mask_path = self.save_mask(data, header, temp_img_path) f.write(f"{temp_img_path}\n") @@ -225,8 +242,6 @@ def _apply_to_images( else: temp_files += [temp_img_path, temp_mask_path] - # out_files.append(out_path) - if pixscale_to_use is None: pixscale_to_use = np.max(all_pixscales) if x_imgpixsize_to_use is None: @@ -264,9 +279,10 @@ def _apply_to_images( temp_output_image_path = get_temp_path( swarp_output_dir, - os.path.splitext(headers[0]["BASENAME"])[0] + ".resamp.fits" + os.path.splitext(headers[0][base_name_key])[0] + ".resamp.fits" ) temp_output_image_weight_path = temp_output_image_path.replace(".fits", ".weight.fits") + if os.path.exists(temp_output_image_path): os.rename(temp_output_image_path, output_image_path) os.rename(temp_output_image_weight_path, output_image_weight_path) @@ -289,7 +305,7 @@ def _apply_to_images( os.remove(temp_file) logger.debug(f"Deleted temporary file {temp_file}") - new_header[base_name_key] = os.path.basename(output_image_path) + new_header[base_name_key] = ",".join([x[base_name_key] for x in headers]) new_header[latest_mask_save_key] = os.path.basename(output_image_weight_path) return [image], [new_header] From fa622adf478dd62f7c9560b88255b132399d6435 Mon Sep 17 00:00:00 2001 From: Robert Stein Date: Mon, 10 Oct 2022 14:26:52 -0700 Subject: [PATCH 2/8] basename split to include multiple images --- winterdrp/processors/base_processor.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/winterdrp/processors/base_processor.py b/winterdrp/processors/base_processor.py index ca845fcad..421849b74 100644 --- a/winterdrp/processors/base_processor.py +++ b/winterdrp/processors/base_processor.py @@ -138,7 +138,7 @@ def get_hash(headers: list[astropy.io.fits.Header]): return hashlib.sha1(key.encode()).hexdigest() def image_batch_error_report(self, exception: Exception, batch): - contents = [x[base_name_key] for x in batch[1]] + contents = ",".join([x[base_name_key] for x in batch[1]]).split(",") return ErrorReport(exception, self.__module__, contents) From 3b96ca16c85aca96b1292355d1a2012109ff91ba Mon Sep 17 00:00:00 2001 From: Robert Stein Date: Mon, 10 Oct 2022 14:31:10 -0700 Subject: [PATCH 3/8] Be more explicit --- winterdrp/processors/base_processor.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/winterdrp/processors/base_processor.py b/winterdrp/processors/base_processor.py index 421849b74..df2ebb706 100644 --- a/winterdrp/processors/base_processor.py +++ b/winterdrp/processors/base_processor.py @@ -138,7 +138,7 @@ def get_hash(headers: list[astropy.io.fits.Header]): return hashlib.sha1(key.encode()).hexdigest() def image_batch_error_report(self, exception: Exception, batch): - contents = ",".join([x[base_name_key] for x in batch[1]]).split(",") + contents = (",".join([x[base_name_key] for x in batch[1]])).split(",") return ErrorReport(exception, self.__module__, contents) From 26fbe5216eefd50572e8eccc9769d02b96385334 Mon Sep 17 00:00:00 2001 From: Robert Stein Date: Mon, 10 Oct 2022 15:05:15 -0700 Subject: [PATCH 4/8] Swap to raw image --- winterdrp/processors/astromatic/swarp/swarp.py | 6 ++++-- winterdrp/processors/base_processor.py | 4 ++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/winterdrp/processors/astromatic/swarp/swarp.py b/winterdrp/processors/astromatic/swarp/swarp.py index b62f5a77f..041495a77 100644 --- a/winterdrp/processors/astromatic/swarp/swarp.py +++ b/winterdrp/processors/astromatic/swarp/swarp.py @@ -3,7 +3,8 @@ import numpy as np import astropy.io.fits from winterdrp.processors.base_processor import BaseImageProcessor -from winterdrp.paths import get_output_dir, copy_temp_file, get_temp_path, base_name_key, latest_mask_save_key +from winterdrp.paths import get_output_dir, copy_temp_file, get_temp_path, base_name_key, latest_mask_save_key, \ + raw_img_key from winterdrp.utils import execute from winterdrp.processors.astromatic.scamp.scamp import Scamp, scamp_header_key from astropy.wcs import WCS @@ -305,7 +306,8 @@ def _apply_to_images( os.remove(temp_file) logger.debug(f"Deleted temporary file {temp_file}") - new_header[base_name_key] = ",".join([x[base_name_key] for x in headers]) + new_header[raw_img_key] = ",".join([x[raw_img_key] for x in headers]) + new_header[base_name_key] = os.path.basename(output_image_path) new_header[latest_mask_save_key] = os.path.basename(output_image_weight_path) return [image], [new_header] diff --git a/winterdrp/processors/base_processor.py b/winterdrp/processors/base_processor.py index df2ebb706..fa9a52486 100644 --- a/winterdrp/processors/base_processor.py +++ b/winterdrp/processors/base_processor.py @@ -11,7 +11,7 @@ from winterdrp.io import save_to_path, open_fits from winterdrp.paths import cal_output_sub_dir, get_mask_path, latest_save_key, latest_mask_save_key, get_output_path,\ - base_name_key, proc_history_key + base_name_key, proc_history_key, raw_img_key from winterdrp.errors import ErrorReport, ErrorStack, ProcessorError, NoncriticalProcessingError logger = logging.getLogger(__name__) @@ -138,7 +138,7 @@ def get_hash(headers: list[astropy.io.fits.Header]): return hashlib.sha1(key.encode()).hexdigest() def image_batch_error_report(self, exception: Exception, batch): - contents = (",".join([x[base_name_key] for x in batch[1]])).split(",") + contents = (",".join([x[raw_img_key] for x in batch[1]])).split(",") return ErrorReport(exception, self.__module__, contents) From 56204a1a993a4aae6c62f5349de28af9cba2be74 Mon Sep 17 00:00:00 2001 From: Robert Stein Date: Mon, 10 Oct 2022 15:05:33 -0700 Subject: [PATCH 5/8] Base name for raw path --- winterdrp/processors/base_processor.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/winterdrp/processors/base_processor.py b/winterdrp/processors/base_processor.py index fa9a52486..5cb02fff8 100644 --- a/winterdrp/processors/base_processor.py +++ b/winterdrp/processors/base_processor.py @@ -138,7 +138,7 @@ def get_hash(headers: list[astropy.io.fits.Header]): return hashlib.sha1(key.encode()).hexdigest() def image_batch_error_report(self, exception: Exception, batch): - contents = (",".join([x[raw_img_key] for x in batch[1]])).split(",") + contents = [os.path.basename(x) for x in ",".join([x[raw_img_key] for x in batch[1]]).split(",")] return ErrorReport(exception, self.__module__, contents) From 78ac83fef4277862335590ad0282ea35f59fd48f Mon Sep 17 00:00:00 2001 From: Robert Stein Date: Mon, 10 Oct 2022 15:09:49 -0700 Subject: [PATCH 6/8] Use pathlib --- winterdrp/processors/base_processor.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/winterdrp/processors/base_processor.py b/winterdrp/processors/base_processor.py index 5cb02fff8..1cce22525 100644 --- a/winterdrp/processors/base_processor.py +++ b/winterdrp/processors/base_processor.py @@ -8,6 +8,7 @@ import datetime import hashlib import pandas as pd +from pathlib import Path from winterdrp.io import save_to_path, open_fits from winterdrp.paths import cal_output_sub_dir, get_mask_path, latest_save_key, latest_mask_save_key, get_output_path,\ @@ -138,7 +139,7 @@ def get_hash(headers: list[astropy.io.fits.Header]): return hashlib.sha1(key.encode()).hexdigest() def image_batch_error_report(self, exception: Exception, batch): - contents = [os.path.basename(x) for x in ",".join([x[raw_img_key] for x in batch[1]]).split(",")] + contents = [Path(x).name for x in ",".join([x[raw_img_key] for x in batch[1]]).split(",")] return ErrorReport(exception, self.__module__, contents) From 97c142eaa340651e332530ffe7bab1f86962136b Mon Sep 17 00:00:00 2001 From: Robert Stein Date: Mon, 10 Oct 2022 15:24:58 -0700 Subject: [PATCH 7/8] Open fits with inbuilt function --- winterdrp/processors/reference.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/winterdrp/processors/reference.py b/winterdrp/processors/reference.py index 36b6a33ec..c429a0297 100644 --- a/winterdrp/processors/reference.py +++ b/winterdrp/processors/reference.py @@ -67,7 +67,7 @@ def _apply_to_images( ref_image = self.ref_image_generator(header) ref_image_path = ref_image.write_reference(header=header, output_dir=self.get_sub_output_dir()) - ref_data, ref_header = open_fits(ref_image_path) + ref_data, ref_header = self.open_fits(ref_image_path) if not (base_name_key in ref_header.keys()): logger.debug(os.path.basename(ref_image_path)) From 5be0dd927460acdcf9ae4c8c3faf577b37b409f6 Mon Sep 17 00:00:00 2001 From: Robert Stein Date: Thu, 3 Nov 2022 12:55:46 -0700 Subject: [PATCH 8/8] Write image path into ref image --- winterdrp/processors/reference.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/winterdrp/processors/reference.py b/winterdrp/processors/reference.py index c429a0297..a692fedef 100644 --- a/winterdrp/processors/reference.py +++ b/winterdrp/processors/reference.py @@ -5,7 +5,7 @@ from winterdrp.processors.base_processor import BaseImageProcessor from astropy.io import fits from winterdrp.paths import get_output_dir, copy_temp_file, base_name_key, sextractor_header_key, latest_mask_save_key, \ - raw_img_dir, psfex_header_key, norm_psfex_header_key + raw_img_dir, psfex_header_key, norm_psfex_header_key, raw_img_key from winterdrp.references.base_reference_generator import BaseReferenceGenerator from collections.abc import Callable from winterdrp.processors.astromatic.swarp.swarp import Swarp @@ -67,12 +67,15 @@ def _apply_to_images( ref_image = self.ref_image_generator(header) ref_image_path = ref_image.write_reference(header=header, output_dir=self.get_sub_output_dir()) - ref_data, ref_header = self.open_fits(ref_image_path) + ref_data, ref_header = open_fits(ref_image_path) if not (base_name_key in ref_header.keys()): logger.debug(os.path.basename(ref_image_path)) ref_header[base_name_key] = os.path.basename(ref_image_path) + if not (raw_img_key in ref_header.keys()): + ref_header[raw_img_key] = ref_image_path + # ref_header[base_name_key] = ref_header[base_name_key] + '_ref' ref_gain = ref_header['GAIN']