From 5b3ad0be26da8429a14db6d445e4d8d69073727d Mon Sep 17 00:00:00 2001 From: Gerrit Holl Date: Tue, 16 Aug 2022 17:05:35 +0200 Subject: [PATCH 01/33] First attempt at true color FCI RGB --- satpy/etc/composites/fci.yaml | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/satpy/etc/composites/fci.yaml b/satpy/etc/composites/fci.yaml index a3e558e455..8caa48b9c4 100644 --- a/satpy/etc/composites/fci.yaml +++ b/satpy/etc/composites/fci.yaml @@ -11,3 +11,14 @@ composites: - name: 'cloud_state' lut: [.nan, 0, 1, 1, 1, 1, 1, 1, 0, .nan] standard_name: binary_cloud_mask + + true_color: + compositor: !!python/name:satpy.composites.SelfSharpenedRGB + prerequisites: + - name: vis_06 + modifiers: [sunz_corrected, rayleigh_corrected] + - name: vis_05 + modifiers: [sunz_corrected, rayleigh_corrected] + - name: vis_04 + modifiers: [sunz_corrected, rayleigh_corrected] + standard_name: true_color From 9e719d4719b4f2ba9d90653dab4e788e003a023f Mon Sep 17 00:00:00 2001 From: Gerrit Holl Date: Tue, 16 Aug 2022 18:19:20 +0200 Subject: [PATCH 02/33] Add a corrected green composite for FCI and use this Add a corrected green composite for FCI and use this in the true color RGB. The problem is the same as for AHI, in that the "green" channel misses the chlorophyl peak making our green planet look rather brown as if all the forests have burnt :( --- satpy/etc/composites/fci.yaml | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/satpy/etc/composites/fci.yaml b/satpy/etc/composites/fci.yaml index 8caa48b9c4..309bec89ac 100644 --- a/satpy/etc/composites/fci.yaml +++ b/satpy/etc/composites/fci.yaml @@ -2,6 +2,20 @@ sensor_name: visir/fci composites: + corrected_green: + description: > + The FCI green band at 0.51 µm deliberately misses the chlorophyl band, such that + the signal comes rather from aerosols and ash rather than vegetation. An effect + is that vegetation in a true colour RGB looks rather brown than green. Mixing in + some part of the NIR 0.8 channel reduced this effect. + compositor: !!python/name:satpy.composites.ahi.GreenCorrector + fractions: [0.93, 0.07] + prerequisites: + - name: vis_05 + modifiers: [sunz_corrected, rayleigh_corrected] + - name: vis_08 + modifiers: [sunz_corrected, rayleigh_corrected] + standard_name: toa_bidirectional_reflectance binary_cloud_mask: # This will set all clear pixels to '0', all pixles with cloudy features (meteorological/dust/ash clouds) to '1' and @@ -17,8 +31,7 @@ composites: prerequisites: - name: vis_06 modifiers: [sunz_corrected, rayleigh_corrected] - - name: vis_05 - modifiers: [sunz_corrected, rayleigh_corrected] + - name: corrected_green - name: vis_04 modifiers: [sunz_corrected, rayleigh_corrected] standard_name: true_color From 06231e90026f66435d606e3f399188312b4fdd32 Mon Sep 17 00:00:00 2001 From: Gerrit Holl Date: Tue, 16 Aug 2022 18:31:12 +0200 Subject: [PATCH 03/33] Move green corrector to new module Move the GreenCorrector to the new module satpy.composites.spectral, as it applies to FCI as well as to AHI. Adapt AHI and FCI composites using this corrector. Improve documentation for the corrector. --- satpy/composites/{ahi.py => spectral.py} | 32 ++++++++++++++++++++++-- satpy/etc/composites/ahi.yaml | 6 ++--- satpy/etc/composites/fci.yaml | 2 +- 3 files changed, 34 insertions(+), 6 deletions(-) rename satpy/composites/{ahi.py => spectral.py} (55%) diff --git a/satpy/composites/ahi.py b/satpy/composites/spectral.py similarity index 55% rename from satpy/composites/ahi.py rename to satpy/composites/spectral.py index edd195bbcf..eafa7e6df2 100644 --- a/satpy/composites/ahi.py +++ b/satpy/composites/spectral.py @@ -15,7 +15,7 @@ # # You should have received a copy of the GNU General Public License along with # satpy. If not, see . -"""Composite classes for the AHI instrument.""" +"""Composite classes for spectral adjustments.""" import logging @@ -26,7 +26,35 @@ class GreenCorrector(GenericCompositor): - """Corrector of the AHI green band to compensate for the deficit of chlorophyll signal.""" + """Corrector of the FCI or AHI green band. + + The green band in FCI and AHI deliberately misses the chlorophyll peak + in order to focus on aerosol and ash rather than on vegetation. This + makes true colour RGBs look like all the forests have burnt down, which + they haven't yet. To make them look a bit greener, this corrector allows + to simulate the green band as a fraction of two or more other channels. + + To be used, the composite takes two re more input channels and a parameter + ``fractions`` that should be a list of floats with the same length as the + number of channels. + + For example, to simulate an FCI corrected green composite, one could use + a combination of 93% from the green band (vis_05) and 7% from the + near-infrared 0.8 µm band (vis_08):: + + corrected_green: + compositor: !!python/name:satpy.composites.ahi.GreenCorrector + fractions: [0.93, 0.07] + prerequisites: + - name: vis_05 + modifiers: [sunz_corrected, rayleigh_corrected] + - name: vis_08 + modifiers: [sunz_corrected, rayleigh_corrected] + standard_name: toa_bidirectional_reflectance + + Other examples can be found in the ``fci.yaml`` and ``ahi.yaml`` composite + files in the satpy distribution. + """ def __init__(self, *args, fractions=(0.85, 0.15), **kwargs): """Set default keyword argument values.""" diff --git a/satpy/etc/composites/ahi.yaml b/satpy/etc/composites/ahi.yaml index 6c3d1860e6..ad77eeab50 100644 --- a/satpy/etc/composites/ahi.yaml +++ b/satpy/etc/composites/ahi.yaml @@ -16,7 +16,7 @@ modifiers: composites: green: - compositor: !!python/name:satpy.composites.ahi.GreenCorrector + compositor: !!python/name:satpy.composites.spectral.GreenCorrector # FUTURE: Set a wavelength...see what happens. Dependency finding # probably wouldn't work. prerequisites: @@ -31,7 +31,7 @@ composites: green_true_color_reproduction: # JMA True Color Reproduction green band # http://www.jma.go.jp/jma/jma-eng/satellite/introduction/TCR.html - compositor: !!python/name:satpy.composites.ahi.GreenCorrector + compositor: !!python/name:satpy.composites.spectral.GreenCorrector fractions: [0.6321, 0.2928, 0.0751] prerequisites: - name: B02 @@ -43,7 +43,7 @@ composites: standard_name: none green_nocorr: - compositor: !!python/name:satpy.composites.ahi.GreenCorrector + compositor: !!python/name:satpy.composites.spectral.GreenCorrector # FUTURE: Set a wavelength...see what happens. Dependency finding # probably wouldn't work. prerequisites: diff --git a/satpy/etc/composites/fci.yaml b/satpy/etc/composites/fci.yaml index 309bec89ac..24c75584c3 100644 --- a/satpy/etc/composites/fci.yaml +++ b/satpy/etc/composites/fci.yaml @@ -8,7 +8,7 @@ composites: the signal comes rather from aerosols and ash rather than vegetation. An effect is that vegetation in a true colour RGB looks rather brown than green. Mixing in some part of the NIR 0.8 channel reduced this effect. - compositor: !!python/name:satpy.composites.ahi.GreenCorrector + compositor: !!python/name:satpy.composites.spectral.GreenCorrector fractions: [0.93, 0.07] prerequisites: - name: vis_05 From 4cf470ebf6175b3974c875d69aa88414b2e2c4c8 Mon Sep 17 00:00:00 2001 From: Gerrit Holl Date: Tue, 16 Aug 2022 18:38:14 +0200 Subject: [PATCH 04/33] Improve docs for FCI true color composite --- satpy/etc/composites/fci.yaml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/satpy/etc/composites/fci.yaml b/satpy/etc/composites/fci.yaml index 24c75584c3..1fa8b32c3b 100644 --- a/satpy/etc/composites/fci.yaml +++ b/satpy/etc/composites/fci.yaml @@ -7,7 +7,8 @@ composites: The FCI green band at 0.51 µm deliberately misses the chlorophyl band, such that the signal comes rather from aerosols and ash rather than vegetation. An effect is that vegetation in a true colour RGB looks rather brown than green. Mixing in - some part of the NIR 0.8 channel reduced this effect. + some part of the NIR 0.8 channel reduced this effect. Note that the fractions + currently implemented are experimental and may change in future versions of Satpy. compositor: !!python/name:satpy.composites.spectral.GreenCorrector fractions: [0.93, 0.07] prerequisites: @@ -28,6 +29,10 @@ composites: true_color: compositor: !!python/name:satpy.composites.SelfSharpenedRGB + description: > + FCI true color composite. The green band is simulated based on a combination of + channels. This simulation may change in future versions of Satpy. See the description + of the corrected_green composites for details. prerequisites: - name: vis_06 modifiers: [sunz_corrected, rayleigh_corrected] From cd132f9a6d9754fa77bf97e8dffda3e736825455 Mon Sep 17 00:00:00 2001 From: Gerrit Holl Date: Wed, 17 Aug 2022 09:27:46 +0200 Subject: [PATCH 05/33] Add new module that was missing in previous commit Add the new file that was missing in the previous commit and that was causing test failures. --- satpy/composites/ahi.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 satpy/composites/ahi.py diff --git a/satpy/composites/ahi.py b/satpy/composites/ahi.py new file mode 100644 index 0000000000..3b97a46f26 --- /dev/null +++ b/satpy/composites/ahi.py @@ -0,0 +1,20 @@ +# Copyright (c) 2022- Satpy developers +# +# This file is part of satpy. +# +# satpy is free software: you can redistribute it and/or modify it under the +# terms of the GNU General Public License as published by the Free Software +# Foundation, either version 3 of the License, or (at your option) any later +# version. +# +# satpy is distributed in the hope that it will be useful, but WITHOUT ANY +# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR +# A PARTICULAR PURPOSE. See the GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along with +# satpy. If not, see . +"""Composite classes for AHI adjustments.""" + +# The green corrector used to be defined here, but was moved to spectral.py +# in Satpy 0.38 because it also applies to FCI. +from .spectral import GreenCorrector # noqa: F401 From be84ed4219d40f8ece424c7e23ba195c72d030e3 Mon Sep 17 00:00:00 2001 From: Gerrit Holl Date: Thu, 18 Aug 2022 09:27:39 +0200 Subject: [PATCH 06/33] try corrected green raw? --- satpy/etc/composites/fci.yaml | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/satpy/etc/composites/fci.yaml b/satpy/etc/composites/fci.yaml index 1fa8b32c3b..853fcff835 100644 --- a/satpy/etc/composites/fci.yaml +++ b/satpy/etc/composites/fci.yaml @@ -18,6 +18,16 @@ composites: modifiers: [sunz_corrected, rayleigh_corrected] standard_name: toa_bidirectional_reflectance + corrected_green_raw: + description: > + Alternative to corrected_green, but without solar zenith or rayleigh correction. + compositor: !!python/name:satpy.composites.spectral.GreenCorrector + fractions: [0.93, 0.07] + prerequisites: + - name: vis_05 + - name: vis_08 + standard_name: toa_bidirectional_reflectance + binary_cloud_mask: # This will set all clear pixels to '0', all pixles with cloudy features (meteorological/dust/ash clouds) to '1' and # missing/undefined pixels to 'nan'. This can be used for the the official EUMETSAT cloud mask product (CLM). @@ -40,3 +50,14 @@ composites: - name: vis_04 modifiers: [sunz_corrected, rayleigh_corrected] standard_name: true_color + + true_color_raw_with_corrected_green: + compositor: !!python/name:satpy.composites.SelfSharpenedRGB + description: > + FCI true color without solar zenith or rayleigh corrections, but with the + corrected green composite. + prerequisites: + - name: vis_06 + - name: corrected_green_raw + - name: vis_04 + standard_name: true_color_raw From a34de8de8356d49a2a2caa430bccf3de973b9faa Mon Sep 17 00:00:00 2001 From: David Hoese Date: Wed, 21 Sep 2022 12:02:12 -0500 Subject: [PATCH 07/33] Update URL to rasterio repository in CI --- .github/workflows/ci.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index c2fa26dafa..e775944b5a 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -103,7 +103,7 @@ jobs: git+https://github.com/dask/distributed \ git+https://github.com/zarr-developers/zarr \ git+https://github.com/Unidata/cftime \ - git+https://github.com/mapbox/rasterio \ + git+https://github.com/rasterio/rasterio \ git+https://github.com/pydata/bottleneck \ git+https://github.com/pydata/xarray \ git+https://github.com/astropy/astropy; From d053f22e5821690dd7707d91d0f2de6772ca38ae Mon Sep 17 00:00:00 2001 From: Gerrit Holl Date: Wed, 28 Sep 2022 10:20:04 +0200 Subject: [PATCH 08/33] Small fixes based on Dave feedback Some small fixes for the green corrector based on feedback from Dave. --- satpy/composites/ahi.py | 2 +- satpy/composites/spectral.py | 6 +++--- satpy/etc/composites/ami.yaml | 6 +++--- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/satpy/composites/ahi.py b/satpy/composites/ahi.py index 3b97a46f26..bb96a94581 100644 --- a/satpy/composites/ahi.py +++ b/satpy/composites/ahi.py @@ -13,7 +13,7 @@ # # You should have received a copy of the GNU General Public License along with # satpy. If not, see . -"""Composite classes for AHI adjustments.""" +"""Composite classes for AHI.""" # The green corrector used to be defined here, but was moved to spectral.py # in Satpy 0.38 because it also applies to FCI. diff --git a/satpy/composites/spectral.py b/satpy/composites/spectral.py index eafa7e6df2..ad29d30fc0 100644 --- a/satpy/composites/spectral.py +++ b/satpy/composites/spectral.py @@ -30,11 +30,11 @@ class GreenCorrector(GenericCompositor): The green band in FCI and AHI deliberately misses the chlorophyll peak in order to focus on aerosol and ash rather than on vegetation. This - makes true colour RGBs look like all the forests have burnt down, which - they haven't yet. To make them look a bit greener, this corrector allows + affects true colour RGBs, because vegetation looks brown rather than green. + To make vegetation look greener again, this corrector allows to simulate the green band as a fraction of two or more other channels. - To be used, the composite takes two re more input channels and a parameter + To be used, the composite takes two or more input channels and a parameter ``fractions`` that should be a list of floats with the same length as the number of channels. diff --git a/satpy/etc/composites/ami.yaml b/satpy/etc/composites/ami.yaml index 6049ef0a10..0c2635d3c7 100644 --- a/satpy/etc/composites/ami.yaml +++ b/satpy/etc/composites/ami.yaml @@ -2,7 +2,7 @@ sensor_name: visir/ami composites: green_raw: - compositor: !!python/name:satpy.composites.ahi.GreenCorrector + compositor: !!python/name:satpy.composites.spectral.GreenCorrector prerequisites: - name: VI005 modifiers: [sunz_corrected] @@ -12,7 +12,7 @@ composites: fractions: [0.85, 0.15] green: - compositor: !!python/name:satpy.composites.ahi.GreenCorrector + compositor: !!python/name:satpy.composites.spectral.GreenCorrector prerequisites: - name: VI005 modifiers: [sunz_corrected, rayleigh_corrected] @@ -22,7 +22,7 @@ composites: fractions: [0.85, 0.15] green_nocorr: - compositor: !!python/name:satpy.composites.ahi.GreenCorrector + compositor: !!python/name:satpy.composites.spectral.GreenCorrector prerequisites: - name: VI005 - name: VI008 From 43d6bba963be8dcd9a046803a98bf01fab18c4ba Mon Sep 17 00:00:00 2001 From: David Hoese Date: Wed, 28 Sep 2022 11:37:26 -0500 Subject: [PATCH 09/33] Update seadas_l2 reader to handle alternative NetCDF file format --- satpy/etc/readers/seadas_l2.yaml | 36 +++++--- satpy/readers/seadas_l2.py | 92 +++++++++++++++---- satpy/tests/reader_tests/test_seadas_l2.py | 101 +++++++++++++++++++-- 3 files changed, 188 insertions(+), 41 deletions(-) diff --git a/satpy/etc/readers/seadas_l2.yaml b/satpy/etc/readers/seadas_l2.yaml index ca6ce3d956..99ff897bac 100644 --- a/satpy/etc/readers/seadas_l2.yaml +++ b/satpy/etc/readers/seadas_l2.yaml @@ -15,41 +15,53 @@ file_types: - '{platform_indicator:1s}1.{start_time:%y%j.%H%M}.seadas.hdf' file_reader: !!python/name:satpy.readers.seadas_l2.SEADASL2HDFFileHandler geo_resolution: 1000 + chlora_seadas_nc: + file_patterns: + # IMAPP-style filenames: + - '{platform_indicator:1s}1.{start_time:%y%j.%H%M}.seadas.nc' + file_reader: !!python/name:satpy.readers.seadas_l2.SEADASL2NetCDFFileHandler + geo_resolution: 1000 chlora_seadas_viirs: # SEADAS_npp_d20211118_t1728125_e1739327.hdf file_patterns: - 'SEADAS_{platform_indicator:s}_d{start_time:%Y%m%d_t%H%M%S%f}_e{end_time:%H%M%S%f}.hdf' file_reader: !!python/name:satpy.readers.seadas_l2.SEADASL2HDFFileHandler geo_resolution: 750 + chlora_seadas_viirs_nc: + # SEADAS_npp_d20211118_t1728125_e1739327.nc + file_patterns: + - 'SEADAS_{platform_indicator:s}_d{start_time:%Y%m%d_t%H%M%S%f}_e{end_time:%H%M%S%f}.nc' + file_reader: !!python/name:satpy.readers.seadas_l2.SEADASL2NetCDFFileHandler + geo_resolution: 750 datasets: longitude: name: longitude - file_type: [chlora_seadas, seadas_hdf_viirs] - file_key: longitude + file_type: [chlora_seadas_viirs_nc, chlora_seadas_nc, seadas_hdf_viirs, chlora_sedas] + file_key: ["navigation_data/longitude", "longitude"] resolution: 1000: - file_type: chlora_seadas + file_type: [chlora_seadas_nc, chlora_seadas] 750: - file_type: chlora_seadas_viirs + file_type: [chlora_seadas_viirs_nc, chlora_seadas_viirs] latitude: name: latitude - file_type: [chlora_seadas, seadas_hdf_viirs] - file_key: latitude + file_type: [chlora_seadas_viirs_nc, chlora_seadas_nc, seadas_hdf_viirs, chlora_sedas] + file_key: ["navigation_data/latitude", "latitude"] resolution: 1000: - file_type: chlora_seadas + file_type: [chlora_seadas_nc, chlora_seadas] 750: - file_type: chlora_seadas_viirs + file_type: [chlora_seadas_viirs_nc, chlora_seadas_viirs] chlor_a: name: chlor_a - file_type: [chlora_seadas, seadas_hdf_viirs] - file_key: chlor_a + file_type: [chlora_seadas_viirs_nc, chlora_seadas_nc, seadas_hdf_viirs, chlora_sedas] + file_key: ["geophysical_data/chlor_a", "chlor_a"] resolution: 1000: - file_type: chlora_seadas + file_type: [chlora_seadas_nc, chlora_seadas] 750: - file_type: chlora_seadas_viirs + file_type: [chlora_seadas_viirs_nc, chlora_seadas_viirs] coordinates: [longitude, latitude] diff --git a/satpy/readers/seadas_l2.py b/satpy/readers/seadas_l2.py index 708b694fac..281a0132af 100644 --- a/satpy/readers/seadas_l2.py +++ b/satpy/readers/seadas_l2.py @@ -31,17 +31,16 @@ from datetime import datetime from .hdf4_utils import HDF4FileHandler +from .netcdf_utils import NetCDF4FileHandler -TIME_FORMAT = "%Y%j%H%M%S" - -class SEADASL2HDFFileHandler(HDF4FileHandler): +class _SEADASL2Base: """Simple handler of SEADAS L2 files.""" def __init__(self, filename, filename_info, filetype_info, apply_quality_flags=False): """Initialize file handler and determine if data quality flags should be applied.""" super().__init__(filename, filename_info, filetype_info) - self.apply_quality_flags = apply_quality_flags and "l2_flags" in self + self.apply_quality_flags = apply_quality_flags and self.l2_flags_var_name in self def _add_satpy_metadata(self, data): data.attrs["sensor"] = self.sensor_names @@ -57,7 +56,7 @@ def _rows_per_scan(self): raise ValueError(f"Don't know how to read data for sensors: {self.sensor_names}") def _platform_name(self): - platform = self["/attr/Mission"] + platform = self[self.platform_attr_name] platform_dict = {'NPP': 'Suomi-NPP', 'JPSS-1': 'NOAA-20', 'JPSS-2': 'NOAA-21'} @@ -66,38 +65,93 @@ def _platform_name(self): @property def start_time(self): """Get the starting observation time of this file's data.""" - start_time = self["/attr/Start Time"] - return datetime.strptime(start_time[:-3], TIME_FORMAT) + start_time = self[self.start_time_attr_name] + return datetime.strptime(start_time[:-3], self.time_format) @property def end_time(self): """Get the ending observation time of this file's data.""" - end_time = self["/attr/End Time"] - return datetime.strptime(end_time[:-3], TIME_FORMAT) + end_time = self[self.end_time_attr_name] + return datetime.strptime(end_time[:-3], self.time_format) @property def sensor_names(self): """Get sensor for the current file's data.""" # Example: MODISA or VIIRSN or VIIRSJ1 - sensor_name = self["/attr/Sensor Name"].lower() + sensor_name = self[self.sensor_attr_name].lower() if sensor_name.startswith("modis"): return {"modis"} return {"viirs"} def get_dataset(self, data_id, dataset_info): """Get DataArray for the specified DataID.""" - file_key = dataset_info.get("file_key", data_id["name"]) - data = self[file_key] - valid_range = data.attrs["valid_range"] - data = data.where(valid_range[0] <= data) - data = data.where(data <= valid_range[1]) - if self.apply_quality_flags and not ("lon" in file_key or "lat" in file_key): - l2_flags = self["l2_flags"] - mask = (l2_flags & 0b00000000010000000000000000000000) != 0 - data = data.where(~mask) + file_key, data = self._get_file_key_and_variable(data_id, dataset_info) + data = self._filter_by_valid_min_max(data) + data = self._rename_2d_dims_if_necessary(data) + data = self._mask_based_on_l2_flags(data) for attr_name in ("standard_name", "long_name", "units"): val = data.attrs[attr_name] if val[-1] == "\x00": data.attrs[attr_name] = data.attrs[attr_name][:-1] data = self._add_satpy_metadata(data) return data + + def _get_file_key_and_variable(self, data_id, dataset_info): + file_keys = dataset_info.get("file_key", data_id["name"]) + if not isinstance(file_keys, list): + file_keys = [file_keys] + for file_key in file_keys: + try: + data = self[file_key] + return file_key, data + except KeyError: + continue + raise KeyError(f"Unable to find any of the possible keys for {data_id}: {file_keys}") + + def _rename_2d_dims_if_necessary(self, data_arr): + if data_arr.ndim != 2 or data_arr.dims == ("y", "x"): + return data_arr + return data_arr.rename(dict(zip(data_arr.dims, ("y", "x")))) + + def _filter_by_valid_min_max(self, data_arr): + valid_range = self._valid_min_max(data_arr) + data_arr = data_arr.where(valid_range[0] <= data_arr) + data_arr = data_arr.where(data_arr <= valid_range[1]) + return data_arr + + def _valid_min_max(self, data_arr): + try: + return data_arr.attrs["valid_range"] + except KeyError: + return data_arr.attrs["valid_min"], data_arr.attrs["valid_max"] + + def _mask_based_on_l2_flags(self, data_arr): + standard_name = data_arr.attrs.get("standard_name", "") + if self.apply_quality_flags and not ("lon" in standard_name or "lat" in standard_name): + l2_flags = self[self.l2_flags_var_name] + l2_flags = self._rename_2d_dims_if_necessary(l2_flags) + mask = (l2_flags & 0b00000000010000000000000000000000) != 0 + data_arr = data_arr.where(~mask) + return data_arr + + +class SEADASL2NetCDFFileHandler(_SEADASL2Base, NetCDF4FileHandler): + """Simple handler of SEADAS L2 NetCDF4 files.""" + + start_time_attr_name = "/attr/time_coverage_start" + end_time_attr_name = "/attr/time_coverage_end" + time_format = "%Y-%m-%dT%H:%M:%S.%f" + platform_attr_name = "/attr/platform" + sensor_attr_name = "/attr/instrument" + l2_flags_var_name = "geophysical_data/l2_flags" + + +class SEADASL2HDFFileHandler(_SEADASL2Base, HDF4FileHandler): + """Simple handler of SEADAS L2 HDF4 files.""" + + start_time_attr_name = "/attr/Start Time" + end_time_attr_name = "/attr/End Time" + time_format = "%Y%j%H%M%S" + platform_attr_name = "/attr/Mission" + sensor_attr_name = "/attr/Sensor Name" + l2_flags_var_name = "l2_flags" diff --git a/satpy/tests/reader_tests/test_seadas_l2.py b/satpy/tests/reader_tests/test_seadas_l2.py index dfa55a557d..2cc10a4344 100644 --- a/satpy/tests/reader_tests/test_seadas_l2.py +++ b/satpy/tests/reader_tests/test_seadas_l2.py @@ -19,7 +19,6 @@ import numpy as np import pytest -from pyhdf.SD import SD, SDC from pyresample.geometry import SwathDefinition from pytest_lazyfixture import lazy_fixture @@ -31,7 +30,7 @@ def seadas_l2_modis_chlor_a(tmp_path_factory): """Create MODIS SEADAS file.""" filename = "a1.21322.1758.seadas.hdf" full_path = str(tmp_path_factory.mktemp("seadas_l2") / filename) - return _create_seadas_chlor_a_file(full_path, "Aqua", "MODISA") + return _create_seadas_chlor_a_hdf4_file(full_path, "Aqua", "MODISA") @pytest.fixture(scope="module") @@ -39,7 +38,7 @@ def seadas_l2_viirs_npp_chlor_a(tmp_path_factory): """Create VIIRS NPP SEADAS file.""" filename = "SEADAS_npp_d20211118_t1728125_e1739327.hdf" full_path = str(tmp_path_factory.mktemp("seadas") / filename) - return _create_seadas_chlor_a_file(full_path, "NPP", "VIIRSN") + return _create_seadas_chlor_a_hdf4_file(full_path, "NPP", "VIIRSN") @pytest.fixture(scope="module") @@ -47,10 +46,11 @@ def seadas_l2_viirs_j01_chlor_a(tmp_path_factory): """Create VIIRS JPSS-01 SEADAS file.""" filename = "SEADAS_j01_d20211118_t1728125_e1739327.hdf" full_path = str(tmp_path_factory.mktemp("seadas") / filename) - return _create_seadas_chlor_a_file(full_path, "JPSS-1", "VIIRSJ1") + return _create_seadas_chlor_a_hdf4_file(full_path, "JPSS-1", "VIIRSJ1") -def _create_seadas_chlor_a_file(full_path, mission, sensor): +def _create_seadas_chlor_a_hdf4_file(full_path, mission, sensor): + from pyhdf.SD import SD, SDC h = SD(full_path, SDC.WRITE | SDC.CREATE) setattr(h, "Sensor Name", sensor) h.Mission = mission @@ -79,8 +79,8 @@ def _create_seadas_chlor_a_file(full_path, mission, sensor): "valid_range": (-90.0, 90.0), } } - _add_variable_to_file(h, "longitude", lon_info) - _add_variable_to_file(h, "latitude", lat_info) + _add_variable_to_hdf4_file(h, "longitude", lon_info) + _add_variable_to_hdf4_file(h, "latitude", lat_info) chlor_a_info = { "type": SDC.FLOAT32, @@ -93,7 +93,7 @@ def _create_seadas_chlor_a_file(full_path, mission, sensor): "valid_range": (0.001, 100.0), } } - _add_variable_to_file(h, "chlor_a", chlor_a_info) + _add_variable_to_hdf4_file(h, "chlor_a", chlor_a_info) l2_flags = np.zeros((5, 5), dtype=np.int32) l2_flags[2, 2] = -1 @@ -103,11 +103,11 @@ def _create_seadas_chlor_a_file(full_path, mission, sensor): "dim_labels": ["Number of Scan Lines", "Number of Pixel Control Points"], "attrs": {}, } - _add_variable_to_file(h, "l2_flags", l2_flags_info) + _add_variable_to_hdf4_file(h, "l2_flags", l2_flags_info) return [full_path] -def _add_variable_to_file(h, var_name, var_info): +def _add_variable_to_hdf4_file(h, var_name, var_info): v = h.create(var_name, var_info['type'], var_info['data'].shape) v[:] = var_info['data'] for dim_count, dimension_name in enumerate(var_info['dim_labels']): @@ -118,6 +118,85 @@ def _add_variable_to_file(h, var_name, var_info): setattr(v, attr_key, attr_val) +@pytest.fixture(scope="module") +def seadas_l2_modis_chlor_a_netcdf(tmp_path_factory): + """Create MODIS SEADAS NetCDF file.""" + filename = "t1.21332.1758.seadas.nc" + full_path = str(tmp_path_factory.mktemp("seadas_l2") / filename) + return _create_seadas_chlor_a_netcdf_file(full_path, "Terra", "MODIS") + + +def _create_seadas_chlor_a_netcdf_file(full_path, mission, sensor): + from netCDF4 import Dataset + nc = Dataset(full_path, "w") + nc.createDimension("number_of_lines", 5) + nc.createDimension("pixels_per_line", 5) + nc.instrument = sensor + nc.platform = mission + nc.time_coverage_start = "2021-11-18T17:58:53.191Z" + nc.time_coverage_end = "2021-11-18T18:05:51.214Z" + + lon_info = { + "data": np.zeros((5, 5), dtype=np.float32), + "dim_labels": ("number_of_lines", "pixels_per_line"), + "attrs": { + "long_name": "Longitude", + "standard_name": "longitude", + "units": "degrees_east", + "valid_min": -180.0, + "valid_max": 180.0, + } + } + lat_info = { + "data": np.zeros((5, 5), np.float32), + "dim_labels": ("number_of_lines", "pixels_per_line"), + "attrs": { + "long_name": "Latitude", + "standard_name": "latitude", + "units": "degrees_north", + "valid_min": -90.0, + "valid_max": 90.0, + } + } + nav_group = nc.createGroup("navigation_data") + _add_variable_to_netcdf_file(nav_group, "longitude", lon_info) + _add_variable_to_netcdf_file(nav_group, "latitude", lat_info) + + chlor_a_info = { + "data": np.ones((5, 5), np.float32), + "dim_labels": ("number_of_lines", "pixels_per_line"), + "attrs": { + "long_name": "Chlorophyll Concentration, OCI Algorithm", + "units": "mg m^-3", + "standard_name": "mass_concentration_of_chlorophyll_in_sea_water", + "valid_min": 0.001, + "valid_max": 100.0, + } + } + l2_flags = np.zeros((5, 5), dtype=np.int32) + l2_flags[2, 2] = -1 + l2_flags_info = { + "data": l2_flags, + "dim_labels": ("number_of_lines", "pixels_per_line"), + "attrs": { + "valid_min": -2147483648, + "valid_max": 2147483647, + }, + } + geophys_group = nc.createGroup("geophysical_data") + _add_variable_to_netcdf_file(geophys_group, "chlor_a", chlor_a_info) + _add_variable_to_netcdf_file(geophys_group, "l2_flags", l2_flags_info) + return [full_path] + + +def _add_variable_to_netcdf_file(nc, var_name, var_info): + v = nc.createVariable(var_name, var_info["data"].dtype.str[1:], dimensions=var_info["dim_labels"], + fill_value=var_info.get("fill_value")) + v[:] = var_info['data'] + for attr_key, attr_val in var_info['attrs'].items(): + setattr(v, attr_key, attr_val) + + class TestSEADAS: """Test the SEADAS L2 file reader.""" @@ -145,6 +224,7 @@ def test_scene_available_datasets(self, input_files): (lazy_fixture("seadas_l2_modis_chlor_a"), "Aqua", {"modis"}, 10), (lazy_fixture("seadas_l2_viirs_npp_chlor_a"), "Suomi-NPP", {"viirs"}, 16), (lazy_fixture("seadas_l2_viirs_j01_chlor_a"), "NOAA-20", {"viirs"}, 16), + (lazy_fixture("seadas_l2_modis_chlor_a_netcdf"), "Terra", {"modis"}, 10), ]) @pytest.mark.parametrize("apply_quality_flags", [False, True]) def test_load_chlor_a(self, input_files, exp_plat, exp_sensor, exp_rps, apply_quality_flags): @@ -153,6 +233,7 @@ def test_load_chlor_a(self, input_files, exp_plat, exp_sensor, exp_rps, apply_qu scene = Scene(reader='seadas_l2', filenames=input_files, reader_kwargs=reader_kwargs) scene.load(['chlor_a']) data_arr = scene['chlor_a'] + assert data_arr.dims == ("y", "x") assert data_arr.attrs['platform_name'] == exp_plat assert data_arr.attrs['sensor'] == exp_sensor assert data_arr.attrs['units'] == 'mg m^-3' From 6e53c6d4352eaf5bb9eafb144987c223d3589980 Mon Sep 17 00:00:00 2001 From: David Hoese Date: Wed, 28 Sep 2022 13:38:02 -0500 Subject: [PATCH 10/33] Fix fine/coarse area checks for 1D SwathDefinitions --- satpy/scene.py | 3 ++- satpy/tests/test_scene.py | 13 ++++++++++--- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/satpy/scene.py b/satpy/scene.py index dd1ca61dcd..06f36e561e 100644 --- a/satpy/scene.py +++ b/satpy/scene.py @@ -275,7 +275,8 @@ def _compare_swath_defs(compare_func: Callable, swath_defs: list[SwathDefinition def _key_func(swath_def: SwathDefinition) -> tuple: attrs = getattr(swath_def.lons, "attrs", {}) lon_ds_name = attrs.get("name") - return swath_def.shape[1], swath_def.shape[0], lon_ds_name + rev_shape = swath_def.shape[::-1] + return rev_shape + (lon_ds_name,) return compare_func(swath_defs, key=_key_func) def _gather_all_areas(self, datasets): diff --git a/satpy/tests/test_scene.py b/satpy/tests/test_scene.py index bf2ffd948c..d17d69d01a 100644 --- a/satpy/tests/test_scene.py +++ b/satpy/tests/test_scene.py @@ -17,6 +17,7 @@ # satpy. If not, see . """Unit tests for scene.py.""" +import math import os import random import string @@ -709,7 +710,7 @@ def test_storage_options_from_reader_kwargs_per_reader(self): def _create_coarest_finest_data_array(shape, area_def, attrs=None): data_arr = xr.DataArray( - da.arange(shape[0] * shape[1]).reshape(shape), + da.arange(math.prod(shape)).reshape(shape), attrs={ 'area': area_def, }) @@ -735,8 +736,12 @@ def _create_coarsest_finest_area_def(shape, extents): def _create_coarsest_finest_swath_def(shape, extents, name_suffix): from pyresample import SwathDefinition - lons_arr = da.repeat(da.linspace(extents[0], extents[2], shape[1], dtype=np.float32)[None, :], shape[0], axis=0) - lats_arr = da.repeat(da.linspace(extents[1], extents[3], shape[0], dtype=np.float32)[:, None], shape[1], axis=1) + if len(shape) == 1: + lons_arr = da.linspace(extents[0], extents[2], shape[0], dtype=np.float32) + lats_arr = da.linspace(extents[1], extents[3], shape[0], dtype=np.float32) + else: + lons_arr = da.repeat(da.linspace(extents[0], extents[2], shape[1], dtype=np.float32)[None, :], shape[0], axis=0) + lats_arr = da.repeat(da.linspace(extents[1], extents[3], shape[0], dtype=np.float32)[:, None], shape[1], axis=1) lons_data_arr = xr.DataArray(lons_arr, attrs={"name": f"longitude{name_suffix}"}) lats_data_arr = xr.DataArray(lats_arr, attrs={"name": f"latitude1{name_suffix}"}) return SwathDefinition(lons_data_arr, lats_data_arr) @@ -754,6 +759,8 @@ class TestFinestCoarsestArea: _create_coarsest_finest_area_def((4, 10), (-1000.0, -1500.0, 1000.0, 1500.0))), (_create_coarsest_finest_swath_def((2, 5), (1000.0, 1500.0, -1000.0, -1500.0), "1"), _create_coarsest_finest_swath_def((4, 10), (1000.0, 1500.0, -1000.0, -1500.0), "1")), + (_create_coarsest_finest_swath_def((5,), (1000.0, 1500.0, -1000.0, -1500.0), "1"), + _create_coarsest_finest_swath_def((10,), (1000.0, 1500.0, -1000.0, -1500.0), "1")), ] ) def test_coarsest_finest_area_different_shape(self, coarse_area, fine_area): From 3b1c58ff5ff66b7e945fd10bf3e3692aaa73a716 Mon Sep 17 00:00:00 2001 From: David Hoese Date: Wed, 28 Sep 2022 13:39:01 -0500 Subject: [PATCH 11/33] Fix VIIRS EDR Active Fires reader using bad sensor and platform attributes --- satpy/readers/viirs_edr_active_fires.py | 4 ++-- satpy/tests/reader_tests/test_viirs_edr_active_fires.py | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/satpy/readers/viirs_edr_active_fires.py b/satpy/readers/viirs_edr_active_fires.py index 2fdfaf6bcf..f1bcf4d3cc 100644 --- a/satpy/readers/viirs_edr_active_fires.py +++ b/satpy/readers/viirs_edr_active_fires.py @@ -92,12 +92,12 @@ def end_time(self): @property def sensor_name(self): """Name of sensor for this file.""" - return self["sensor"].lower() + return self["/attr/instrument_name"].lower() @property def platform_name(self): """Name of platform/satellite for this file.""" - return self["platform_name"] + return self["/attr/satellite_name"] class VIIRSActiveFiresTextFileHandler(BaseFileHandler): diff --git a/satpy/tests/reader_tests/test_viirs_edr_active_fires.py b/satpy/tests/reader_tests/test_viirs_edr_active_fires.py index 9341ad2e21..df94283fba 100644 --- a/satpy/tests/reader_tests/test_viirs_edr_active_fires.py +++ b/satpy/tests/reader_tests/test_viirs_edr_active_fires.py @@ -61,8 +61,8 @@ def get_test_content(self, filename, filename_info, filename_type): """Mimic reader input file content.""" file_content = {} file_content['/attr/data_id'] = "AFMOD" - file_content['satellite_name'] = "npp" - file_content['sensor'] = 'VIIRS' + file_content['/attr/satellite_name'] = "NPP" + file_content['/attr/instrument_name'] = 'VIIRS' file_content['Fire Pixels/FP_latitude'] = DEFAULT_LATLON_FILE_DATA file_content['Fire Pixels/FP_longitude'] = DEFAULT_LATLON_FILE_DATA @@ -87,8 +87,8 @@ def get_test_content(self, filename, filename_info, filename_type): """Mimic reader input file content.""" file_content = {} file_content['/attr/data_id'] = "AFIMG" - file_content['satellite_name'] = "npp" - file_content['sensor'] = 'VIIRS' + file_content['/attr/satellite_name'] = "NPP" + file_content['/attr/instrument_name'] = 'VIIRS' file_content['Fire Pixels/FP_latitude'] = DEFAULT_LATLON_FILE_DATA file_content['Fire Pixels/FP_longitude'] = DEFAULT_LATLON_FILE_DATA From 6ec5b8c7934506b6625ae22c41926229364c0b0d Mon Sep 17 00:00:00 2001 From: Trygve Aspenes Date: Wed, 28 Sep 2022 21:41:46 +0200 Subject: [PATCH 12/33] Use the attributes of data when this has changed. Use keep_attrs=True in xarray.where --- satpy/readers/hy2_scat_l2b_h5.py | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/satpy/readers/hy2_scat_l2b_h5.py b/satpy/readers/hy2_scat_l2b_h5.py index 6bdb8f88a4..2108095852 100644 --- a/satpy/readers/hy2_scat_l2b_h5.py +++ b/satpy/readers/hy2_scat_l2b_h5.py @@ -107,8 +107,8 @@ def get_dataset(self, key, info): else: dim_map = {curr_dim: new_dim for curr_dim, new_dim in zip(data.dims, dims)} data = data.rename(dim_map) - data = self._mask_data(key['name'], data) - data = self._scale_data(key['name'], data) + data = self._mask_data(data) + data = self._scale_data(data) if key['name'] in 'wvc_lon': data = xr.where(data > 180, data - 360., data) @@ -120,13 +120,12 @@ def get_dataset(self, key, info): return data - def _scale_data(self, key_name, data): - return data * self[key_name].attrs['scale_factor'] + self[key_name].attrs['add_offset'] + def _scale_data(self, data): + return data * data.attrs['scale_factor'] + data.attrs['add_offset'] - def _mask_data(self, key_name, data): - data = xr.where(data == self[key_name].attrs['fill_value'], np.nan, data) - - valid_range = self[key_name].attrs['valid_range'] - data = xr.where(data < valid_range[0], np.nan, data) - data = xr.where(data > valid_range[1], np.nan, data) + def _mask_data(self, data): + data = xr.where(data == data.attrs['fill_value'], np.nan, data, keep_attrs=True) + valid_range = data.attrs['valid_range'] + data = xr.where(data < valid_range[0], np.nan, data, keep_attrs=True) + data = xr.where(data > valid_range[1], np.nan, data, keep_attrs=True) return data From 8e667e65be630a256fc3c709dcd764ff8a9bb4bc Mon Sep 17 00:00:00 2001 From: Trygve Aspenes Date: Thu, 29 Sep 2022 07:27:54 +0200 Subject: [PATCH 13/33] Use keep_attrs=True in xarray.where --- satpy/readers/hy2_scat_l2b_h5.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/satpy/readers/hy2_scat_l2b_h5.py b/satpy/readers/hy2_scat_l2b_h5.py index 2108095852..47680ade59 100644 --- a/satpy/readers/hy2_scat_l2b_h5.py +++ b/satpy/readers/hy2_scat_l2b_h5.py @@ -111,7 +111,7 @@ def get_dataset(self, key, info): data = self._scale_data(data) if key['name'] in 'wvc_lon': - data = xr.where(data > 180, data - 360., data) + data = xr.where(data > 180, data - 360., data, keep_attrs=True) data.attrs.update(info) data.attrs.update(self.get_metadata()) data.attrs.update(self.get_variable_metadata()) From 2fe859bfec52b880e7df8990c97514b1bc201d8a Mon Sep 17 00:00:00 2001 From: Trygve Aspenes Date: Thu, 29 Sep 2022 17:53:32 +0200 Subject: [PATCH 14/33] need to store attrs from data and update data.attrs after xarray.where --- satpy/readers/hy2_scat_l2b_h5.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/satpy/readers/hy2_scat_l2b_h5.py b/satpy/readers/hy2_scat_l2b_h5.py index 47680ade59..64520bae9a 100644 --- a/satpy/readers/hy2_scat_l2b_h5.py +++ b/satpy/readers/hy2_scat_l2b_h5.py @@ -111,7 +111,9 @@ def get_dataset(self, key, info): data = self._scale_data(data) if key['name'] in 'wvc_lon': - data = xr.where(data > 180, data - 360., data, keep_attrs=True) + _attrs = data.attrs + data = xr.where(data > 180, data - 360., data) + data.attrs.update(_attrs) data.attrs.update(info) data.attrs.update(self.get_metadata()) data.attrs.update(self.get_variable_metadata()) @@ -124,8 +126,10 @@ def _scale_data(self, data): return data * data.attrs['scale_factor'] + data.attrs['add_offset'] def _mask_data(self, data): - data = xr.where(data == data.attrs['fill_value'], np.nan, data, keep_attrs=True) + _attrs = data.attrs valid_range = data.attrs['valid_range'] - data = xr.where(data < valid_range[0], np.nan, data, keep_attrs=True) - data = xr.where(data > valid_range[1], np.nan, data, keep_attrs=True) + data = xr.where(data == data.attrs['fill_value'], np.nan, data) + data = xr.where(data < valid_range[0], np.nan, data) + data = xr.where(data > valid_range[1], np.nan, data) + data.attrs.update(_attrs) return data From 5d74a580f5ea4144f0c67e9fa5d757b62fd86931 Mon Sep 17 00:00:00 2001 From: Martin Raspaud Date: Fri, 30 Sep 2022 12:32:55 +0200 Subject: [PATCH 15/33] Fix coord renaming for AOD product in abi l2 --- satpy/readers/abi_base.py | 4 +- satpy/tests/reader_tests/test_abi_l2_nc.py | 66 ++++++++++++++++++++++ 2 files changed, 69 insertions(+), 1 deletion(-) diff --git a/satpy/readers/abi_base.py b/satpy/readers/abi_base.py index 2f97c925cc..b74c4cf728 100644 --- a/satpy/readers/abi_base.py +++ b/satpy/readers/abi_base.py @@ -18,6 +18,7 @@ """Advance Baseline Imager reader base class for the Level 1b and l2+ reader.""" import logging +from contextlib import suppress from datetime import datetime import numpy as np @@ -76,7 +77,8 @@ def _rename_dims(nc): if 't' in nc.dims or 't' in nc.coords: nc = nc.rename({'t': 'time'}) if 'goes_lat_lon_projection' in nc: - nc = nc.rename({'lon': 'x', 'lat': 'y'}) + with suppress(ValueError): + nc = nc.rename({'lon': 'x', 'lat': 'y'}) return nc @property diff --git a/satpy/tests/reader_tests/test_abi_l2_nc.py b/satpy/tests/reader_tests/test_abi_l2_nc.py index 02aaad977b..63014685f9 100644 --- a/satpy/tests/reader_tests/test_abi_l2_nc.py +++ b/satpy/tests/reader_tests/test_abi_l2_nc.py @@ -279,3 +279,69 @@ def test_get_area_def_latlon(self, adef): self.assertEqual(call_args[4], self.reader.ncols) self.assertEqual(call_args[5], self.reader.nlines) np.testing.assert_allclose(call_args[6], (-85.0, -20.0, -65.0, 20)) + + +class Test_NC_ABI_L2_area_AOD(unittest.TestCase): + """Test the NC_ABI_L2 reader for the AOD product.""" + + @mock.patch('satpy.readers.abi_base.xr') + def setUp(self, xr_): + """Create fake data for the tests.""" + from satpy.readers.abi_l2_nc import NC_ABI_L2 + proj = xr.DataArray( + [], + attrs={'semi_major_axis': 1., + 'semi_minor_axis': 1., + 'inverse_flattening': 1., + 'longitude_of_prime_meridian': 0.0, + } + ) + + proj_ext = xr.DataArray( + [], + attrs={'geospatial_westbound_longitude': -85.0, + 'geospatial_eastbound_longitude': -65.0, + 'geospatial_northbound_latitude': 20.0, + 'geospatial_southbound_latitude': -20.0, + 'geospatial_lat_center': 0.0, + 'geospatial_lon_center': -75.0, + }) + + x__ = xr.DataArray( + [0, 1], + attrs={'scale_factor': 2., 'add_offset': -1.}, + dims=('x',), + ) + y__ = xr.DataArray( + [0, 1], + attrs={'scale_factor': -2., 'add_offset': 1.}, + dims=('y',), + ) + fake_dataset = xr.Dataset( + data_vars={ + 'goes_lat_lon_projection': proj, + 'geospatial_lat_lon_extent': proj_ext, + 'x': x__, + 'y': y__, + 'RSR': xr.DataArray(np.ones((2, 2)), dims=('y', 'x')), + }, + ) + xr_.open_dataset.return_value = fake_dataset + + self.reader = NC_ABI_L2('filename', + {'platform_shortname': 'G16', 'observation_type': 'RSR', + 'scene_abbr': 'C', 'scan_mode': 'M3'}, + {'filetype': 'info'}) + + @mock.patch('satpy.readers.abi_base.geometry.AreaDefinition') + def test_get_area_def_xy(self, adef): + """Test the area generation.""" + self.reader.get_area_def(None) + + self.assertEqual(adef.call_count, 1) + call_args = tuple(adef.call_args)[0] + self.assertDictEqual(call_args[3], {'proj': 'latlong', 'a': 1.0, 'b': 1.0, 'fi': 1.0, 'pm': 0.0, + 'lon_0': -75.0, 'lat_0': 0.0}) + self.assertEqual(call_args[4], self.reader.ncols) + self.assertEqual(call_args[5], self.reader.nlines) + np.testing.assert_allclose(call_args[6], (-85.0, -20.0, -65.0, 20)) From 728bf8d0c2b6b5018e07f7773228f2a0a2d7bf47 Mon Sep 17 00:00:00 2001 From: David Hoese Date: Fri, 30 Sep 2022 13:15:18 -0500 Subject: [PATCH 16/33] Fix failing get_satpos precision test --- satpy/tests/test_utils.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/satpy/tests/test_utils.py b/satpy/tests/test_utils.py index 354c1886bb..3f0e055765 100644 --- a/satpy/tests/test_utils.py +++ b/satpy/tests/test_utils.py @@ -291,8 +291,10 @@ def test_get_satpos_from_satname(self, caplog): (lon, lat, alt) = get_satpos(data_arr, use_tle=True) assert "Orbital parameters missing from metadata" in caplog.text np.testing.assert_allclose( - (lon, lat, alt), - (119.39533705010592, -1.1491628298731498, 35803.19986408156)) + (lon, lat, alt), + (119.39533705010592, -1.1491628298731498, 35803.19986408156), + rtol=1e-4, + ) def test_make_fake_scene(): From 4986e536a97a3a84c71d6ba91a60fbb3ff0dfc71 Mon Sep 17 00:00:00 2001 From: David Hoese Date: Fri, 30 Sep 2022 13:54:45 -0500 Subject: [PATCH 17/33] Try not using new xarray 2022.9.0 --- continuous_integration/environment.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/continuous_integration/environment.yaml b/continuous_integration/environment.yaml index b86bc1e072..f8a94233bc 100644 --- a/continuous_integration/environment.yaml +++ b/continuous_integration/environment.yaml @@ -2,7 +2,7 @@ name: test-environment channels: - conda-forge dependencies: - - xarray + - xarray!=2022.9.0 - dask - distributed - donfig From 1a0401c1519646b38ca58244e0d1265855632d1c Mon Sep 17 00:00:00 2001 From: David Hoese Date: Sun, 2 Oct 2022 15:05:27 -0500 Subject: [PATCH 18/33] Fix NIRReflectance test compatibility with newer xarray Modifier internals will copy an input DataArray which now includes `.attrs`. If an object inside the `.attrs` is a Mock then it is also copied. To check if the properties of the Mock the test must check the new copy of the Mock, not the original Mock. --- satpy/tests/test_modifiers.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/satpy/tests/test_modifiers.py b/satpy/tests/test_modifiers.py index d8598a5b05..d43a1005d0 100644 --- a/satpy/tests/test_modifiers.py +++ b/satpy/tests/test_modifiers.py @@ -169,10 +169,10 @@ class TestNIRReflectance(unittest.TestCase): def setUp(self): """Set up the test case for the NIRReflectance compositor.""" - self.get_lonlats = mock.MagicMock() + get_lonlats = mock.MagicMock() self.lons, self.lats = 1, 2 - self.get_lonlats.return_value = (self.lons, self.lats) - area = mock.MagicMock(get_lonlats=self.get_lonlats) + get_lonlats.return_value = (self.lons, self.lats) + area = mock.MagicMock(get_lonlats=get_lonlats) self.start_time = 1 self.metadata = {'platform_name': 'Meteosat-11', @@ -241,7 +241,9 @@ def test_no_sunz_no_co2(self, calculator, apply_modifier_info, sza): info = {'modifiers': None} res = comp([self.nir, self.ir_], optional_datasets=[], **info) - self.get_lonlats.assert_called() + # due to copying of DataArrays, self.get_lonlats is not the same as the one that was called + # we must used the area from the final result DataArray + res.attrs["area"].get_lonlats.assert_called() sza.assert_called_with(self.start_time, self.lons, self.lats) self.refl_from_tbs.assert_called_with(self.da_sunz, self.nir.data, self.ir_.data, tb_ir_co2=None) assert np.allclose(res.data, self.refl * 100).compute() From 709aeb8a3a3fa10ff3d07f503c8092681e52b44b Mon Sep 17 00:00:00 2001 From: David Hoese Date: Mon, 3 Oct 2022 09:16:00 -0500 Subject: [PATCH 19/33] Fix water detection VIIRS enhancement if dask arrays are not copied in xarray Recent changes to xarray copy dask arrays. A simple fix being proposed would not copy the dask arrays but this revealed a bug in water_detection where the input array was being assigned to inplace. This is not good practice with dask map_blocks'd functions. --- satpy/enhancements/viirs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/satpy/enhancements/viirs.py b/satpy/enhancements/viirs.py index 6a465f161b..627fc80220 100644 --- a/satpy/enhancements/viirs.py +++ b/satpy/enhancements/viirs.py @@ -38,7 +38,7 @@ def water_detection(img, **kwargs): @exclude_alpha @using_map_blocks def _water_detection(img_data): - data = np.asarray(img_data) + data = np.asarray(img_data).copy() data[data == 150] = 31 data[data == 199] = 18 data[data >= 200] = data[data >= 200] - 100 From 753b293310d727e49daed73ab3284cd3658dad5c Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 3 Oct 2022 22:40:42 +0000 Subject: [PATCH 20/33] [pre-commit.ci] pre-commit autoupdate MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/pre-commit/mirrors-mypy: v0.971 → v0.981](https://github.com/pre-commit/mirrors-mypy/compare/v0.971...v0.981) --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index caeabd5305..1b5a9bbd01 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -20,7 +20,7 @@ repos: - id: bandit args: [--ini, .bandit] - repo: https://github.com/pre-commit/mirrors-mypy - rev: 'v0.971' # Use the sha / tag you want to point at + rev: 'v0.981' # Use the sha / tag you want to point at hooks: - id: mypy additional_dependencies: From a8e36d91a3604ddd03edb5c3088eead02a1e38b4 Mon Sep 17 00:00:00 2001 From: David Hoese Date: Tue, 4 Oct 2022 14:56:41 -0500 Subject: [PATCH 21/33] Fix CLAVR-x configuration in 'awips_tiled' writer to be backwards compatible --- satpy/etc/writers/awips_tiled.yaml | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/satpy/etc/writers/awips_tiled.yaml b/satpy/etc/writers/awips_tiled.yaml index 6fcc5aa948..e761414904 100644 --- a/satpy/etc/writers/awips_tiled.yaml +++ b/satpy/etc/writers/awips_tiled.yaml @@ -129,6 +129,12 @@ templates: physical_element: raw_value: CLAVR-x Cloud Type units: {} + encoding: + dtype: int16 + _Unsigned: "true" + scale_factor: 0.5 + add_offset: 0.0 + _FillValue: -128 clavrx_cld_temp_acha: reader: clavrx name: cld_temp_acha @@ -153,6 +159,12 @@ templates: units: {} physical_element: raw_value: CLAVR-x Cloud Phase + encoding: + dtype: int16 + _Unsigned: "true" + scale_factor: 0.5 + add_offset: 0.0 + _FillValue: -128 clavrx_cld_opd_dcomp: reader: clavrx name: cld_opd_dcomp From 718a27a1958e963e4d0f5f7b24eb70ba4bed784c Mon Sep 17 00:00:00 2001 From: Gerrit Holl Date: Wed, 5 Oct 2022 09:49:12 +0200 Subject: [PATCH 22/33] Remove superfluous header Remove superfluous header from composites/spectral.py, and update copyright notice. --- satpy/composites/spectral.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/satpy/composites/spectral.py b/satpy/composites/spectral.py index ad29d30fc0..d0e6dc9330 100644 --- a/satpy/composites/spectral.py +++ b/satpy/composites/spectral.py @@ -1,6 +1,4 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -# Copyright (c) 2015-2021 Satpy developers +# Copyright (c) 2015-2022 Satpy developers # # This file is part of satpy. # From 877f46bc3f10366e7ced16105aa414ff920a8cb6 Mon Sep 17 00:00:00 2001 From: "Adam.Dybbroe" Date: Wed, 5 Oct 2022 17:12:28 +0200 Subject: [PATCH 23/33] Fix ICI yaml file after latest update regarding MWS and the FrequencyDoubleSideBand type Signed-off-by: Adam.Dybbroe --- satpy/etc/readers/ici_l1b_nc.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/satpy/etc/readers/ici_l1b_nc.yaml b/satpy/etc/readers/ici_l1b_nc.yaml index 8d8f271867..cbf6c9f993 100644 --- a/satpy/etc/readers/ici_l1b_nc.yaml +++ b/satpy/etc/readers/ici_l1b_nc.yaml @@ -12,7 +12,7 @@ reader: name: required: true frequency_double_sideband: - type: !!python/name:satpy.readers.aapp_mhs_amsub_l1c.FrequencyDoubleSideBand + type: !!python/name:satpy.readers.pmw_channels_definitions.FrequencyDoubleSideBand polarization: enum: - H From f6fe92a2108c5fa28d441f877f3805ed352fccfa Mon Sep 17 00:00:00 2001 From: "Adam.Dybbroe" Date: Wed, 5 Oct 2022 17:18:11 +0200 Subject: [PATCH 24/33] Remove parts on auxillary data that is not yet supported Signed-off-by: Adam.Dybbroe --- satpy/etc/readers/mws_l1b_nc.yaml | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/satpy/etc/readers/mws_l1b_nc.yaml b/satpy/etc/readers/mws_l1b_nc.yaml index ce7fe527de..1f77bbe7a2 100644 --- a/satpy/etc/readers/mws_l1b_nc.yaml +++ b/satpy/etc/readers/mws_l1b_nc.yaml @@ -493,22 +493,6 @@ datasets: - mws_lon - mws_lat -# --- Land surface data --- - surface_type: - name: surface_type - standard_name: surface_type - file_type: mws_l1b_nc - coordinates: - - mws_lon - - mws_lat - terrain_elevation: - name: terrain_elevation - standard_name: terrain_elevation - file_type: mws_l1b_nc - coordinates: - - mws_lon - - mws_lat - file_types: mws_l1b_nc: From 25740d795f258a696385220810747e99a7906253 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 10 Oct 2022 23:37:27 +0000 Subject: [PATCH 25/33] [pre-commit.ci] pre-commit autoupdate MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/pre-commit/mirrors-mypy: v0.981 → v0.982](https://github.com/pre-commit/mirrors-mypy/compare/v0.981...v0.982) --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 1b5a9bbd01..3d36ec4301 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -20,7 +20,7 @@ repos: - id: bandit args: [--ini, .bandit] - repo: https://github.com/pre-commit/mirrors-mypy - rev: 'v0.981' # Use the sha / tag you want to point at + rev: 'v0.982' # Use the sha / tag you want to point at hooks: - id: mypy additional_dependencies: From 2d95747e706bd604f32030a01c008f734463175b Mon Sep 17 00:00:00 2001 From: Trygve Aspenes Date: Wed, 28 Sep 2022 21:41:46 +0200 Subject: [PATCH 26/33] Use the attributes of data when this has changed. Use keep_attrs=True in xarray.where --- satpy/readers/hy2_scat_l2b_h5.py | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/satpy/readers/hy2_scat_l2b_h5.py b/satpy/readers/hy2_scat_l2b_h5.py index 6bdb8f88a4..2108095852 100644 --- a/satpy/readers/hy2_scat_l2b_h5.py +++ b/satpy/readers/hy2_scat_l2b_h5.py @@ -107,8 +107,8 @@ def get_dataset(self, key, info): else: dim_map = {curr_dim: new_dim for curr_dim, new_dim in zip(data.dims, dims)} data = data.rename(dim_map) - data = self._mask_data(key['name'], data) - data = self._scale_data(key['name'], data) + data = self._mask_data(data) + data = self._scale_data(data) if key['name'] in 'wvc_lon': data = xr.where(data > 180, data - 360., data) @@ -120,13 +120,12 @@ def get_dataset(self, key, info): return data - def _scale_data(self, key_name, data): - return data * self[key_name].attrs['scale_factor'] + self[key_name].attrs['add_offset'] + def _scale_data(self, data): + return data * data.attrs['scale_factor'] + data.attrs['add_offset'] - def _mask_data(self, key_name, data): - data = xr.where(data == self[key_name].attrs['fill_value'], np.nan, data) - - valid_range = self[key_name].attrs['valid_range'] - data = xr.where(data < valid_range[0], np.nan, data) - data = xr.where(data > valid_range[1], np.nan, data) + def _mask_data(self, data): + data = xr.where(data == data.attrs['fill_value'], np.nan, data, keep_attrs=True) + valid_range = data.attrs['valid_range'] + data = xr.where(data < valid_range[0], np.nan, data, keep_attrs=True) + data = xr.where(data > valid_range[1], np.nan, data, keep_attrs=True) return data From f9fd9cd3953038a1396f44226c9603ea41b5a17f Mon Sep 17 00:00:00 2001 From: Trygve Aspenes Date: Thu, 29 Sep 2022 07:27:54 +0200 Subject: [PATCH 27/33] Use keep_attrs=True in xarray.where --- satpy/readers/hy2_scat_l2b_h5.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/satpy/readers/hy2_scat_l2b_h5.py b/satpy/readers/hy2_scat_l2b_h5.py index 2108095852..47680ade59 100644 --- a/satpy/readers/hy2_scat_l2b_h5.py +++ b/satpy/readers/hy2_scat_l2b_h5.py @@ -111,7 +111,7 @@ def get_dataset(self, key, info): data = self._scale_data(data) if key['name'] in 'wvc_lon': - data = xr.where(data > 180, data - 360., data) + data = xr.where(data > 180, data - 360., data, keep_attrs=True) data.attrs.update(info) data.attrs.update(self.get_metadata()) data.attrs.update(self.get_variable_metadata()) From 7f49a6d857612ddb076a31fc2213084ee96d3e77 Mon Sep 17 00:00:00 2001 From: Trygve Aspenes Date: Thu, 29 Sep 2022 17:53:32 +0200 Subject: [PATCH 28/33] need to store attrs from data and update data.attrs after xarray.where --- satpy/readers/hy2_scat_l2b_h5.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/satpy/readers/hy2_scat_l2b_h5.py b/satpy/readers/hy2_scat_l2b_h5.py index 47680ade59..64520bae9a 100644 --- a/satpy/readers/hy2_scat_l2b_h5.py +++ b/satpy/readers/hy2_scat_l2b_h5.py @@ -111,7 +111,9 @@ def get_dataset(self, key, info): data = self._scale_data(data) if key['name'] in 'wvc_lon': - data = xr.where(data > 180, data - 360., data, keep_attrs=True) + _attrs = data.attrs + data = xr.where(data > 180, data - 360., data) + data.attrs.update(_attrs) data.attrs.update(info) data.attrs.update(self.get_metadata()) data.attrs.update(self.get_variable_metadata()) @@ -124,8 +126,10 @@ def _scale_data(self, data): return data * data.attrs['scale_factor'] + data.attrs['add_offset'] def _mask_data(self, data): - data = xr.where(data == data.attrs['fill_value'], np.nan, data, keep_attrs=True) + _attrs = data.attrs valid_range = data.attrs['valid_range'] - data = xr.where(data < valid_range[0], np.nan, data, keep_attrs=True) - data = xr.where(data > valid_range[1], np.nan, data, keep_attrs=True) + data = xr.where(data == data.attrs['fill_value'], np.nan, data) + data = xr.where(data < valid_range[0], np.nan, data) + data = xr.where(data > valid_range[1], np.nan, data) + data.attrs.update(_attrs) return data From 1461f2082cee2316a5f1ef5e5e8a99e049d4352a Mon Sep 17 00:00:00 2001 From: Trygve Aspenes Date: Tue, 11 Oct 2022 16:02:48 +0200 Subject: [PATCH 29/33] test with copy of xarray datasets --- satpy/tests/reader_tests/test_hy2_scat_l2b_h5.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/satpy/tests/reader_tests/test_hy2_scat_l2b_h5.py b/satpy/tests/reader_tests/test_hy2_scat_l2b_h5.py index 1664c46ed4..ed8a45505f 100644 --- a/satpy/tests/reader_tests/test_hy2_scat_l2b_h5.py +++ b/satpy/tests/reader_tests/test_hy2_scat_l2b_h5.py @@ -40,6 +40,12 @@ class FakeHDF5FileHandler2(FakeHDF5FileHandler): """Swap-in HDF5 File Handler.""" + def __getitem__(self, key): + val = self.file_content[key] + if isinstance(val, xr.core.dataarray.DataArray): + val = val.copy() + return val + def _get_geo_data(self, num_rows, num_cols): geo = { 'wvc_lon': From df70a5474b6b44a3ca44698236ea3afc0be4616e Mon Sep 17 00:00:00 2001 From: Trygve Aspenes Date: Tue, 11 Oct 2022 16:08:48 +0200 Subject: [PATCH 30/33] docstring --- satpy/tests/reader_tests/test_hy2_scat_l2b_h5.py | 1 + 1 file changed, 1 insertion(+) diff --git a/satpy/tests/reader_tests/test_hy2_scat_l2b_h5.py b/satpy/tests/reader_tests/test_hy2_scat_l2b_h5.py index ed8a45505f..b4718eb368 100644 --- a/satpy/tests/reader_tests/test_hy2_scat_l2b_h5.py +++ b/satpy/tests/reader_tests/test_hy2_scat_l2b_h5.py @@ -41,6 +41,7 @@ class FakeHDF5FileHandler2(FakeHDF5FileHandler): """Swap-in HDF5 File Handler.""" def __getitem__(self, key): + """Return copy of dataarray to prevent manipulating attributes in the original""" val = self.file_content[key] if isinstance(val, xr.core.dataarray.DataArray): val = val.copy() From 2218d685601e4776fb97e9e92a5ae3e48c3c2a13 Mon Sep 17 00:00:00 2001 From: Trygve Aspenes Date: Tue, 11 Oct 2022 16:10:24 +0200 Subject: [PATCH 31/33] docstring --- satpy/tests/reader_tests/test_hy2_scat_l2b_h5.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/satpy/tests/reader_tests/test_hy2_scat_l2b_h5.py b/satpy/tests/reader_tests/test_hy2_scat_l2b_h5.py index b4718eb368..e8ba088caa 100644 --- a/satpy/tests/reader_tests/test_hy2_scat_l2b_h5.py +++ b/satpy/tests/reader_tests/test_hy2_scat_l2b_h5.py @@ -41,7 +41,7 @@ class FakeHDF5FileHandler2(FakeHDF5FileHandler): """Swap-in HDF5 File Handler.""" def __getitem__(self, key): - """Return copy of dataarray to prevent manipulating attributes in the original""" + """Return copy of dataarray to prevent manipulating attributes in the original.""" val = self.file_content[key] if isinstance(val, xr.core.dataarray.DataArray): val = val.copy() From a3e4f485f2f225ae0e9632b7e754bf9f412dca20 Mon Sep 17 00:00:00 2001 From: Trygve Aspenes Date: Tue, 11 Oct 2022 17:31:32 +0200 Subject: [PATCH 32/33] add properties test --- satpy/tests/reader_tests/test_hy2_scat_l2b_h5.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/satpy/tests/reader_tests/test_hy2_scat_l2b_h5.py b/satpy/tests/reader_tests/test_hy2_scat_l2b_h5.py index e8ba088caa..854bd00d6c 100644 --- a/satpy/tests/reader_tests/test_hy2_scat_l2b_h5.py +++ b/satpy/tests/reader_tests/test_hy2_scat_l2b_h5.py @@ -505,3 +505,19 @@ def test_reading_attrs_nsoas(self): with self.assertRaises(KeyError): self.assertEqual(res['wvc_lon'].attrs['L2B_Number_WVC_cells'], 10) self.assertEqual(res['wvc_lon'].attrs['L2B_Expected_WVC_Cells'], 10) + + def test_properties(self): + """Test platform_name.""" + from datetime import datetime + from satpy.readers import load_reader + filenames = [ + 'W_XX-EUMETSAT-Darmstadt,SURFACE+SATELLITE,HY2B+SM_C_EUMP_20200326------_07077_o_250_l2b.h5', ] + + reader = load_reader(self.reader_configs) + files = reader.select_files_from_pathnames(filenames) + reader.create_filehandlers(files) + # Make sure we have some files + res = reader.load(['wvc_lon']) + self.assertEqual(res['wvc_lon'].platform_name, 'HY-2B') + self.assertEqual(res['wvc_lon'].start_time, datetime(2020, 3, 26, 1, 11, 7)) + self.assertEqual(res['wvc_lon'].end_time, datetime(2020, 3, 26, 2, 55, 40)) From 4f269d70f45ba25e09ec715c0410295e4188791c Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 11 Oct 2022 15:43:05 +0000 Subject: [PATCH 33/33] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- satpy/tests/reader_tests/test_hy2_scat_l2b_h5.py | 1 + 1 file changed, 1 insertion(+) diff --git a/satpy/tests/reader_tests/test_hy2_scat_l2b_h5.py b/satpy/tests/reader_tests/test_hy2_scat_l2b_h5.py index 854bd00d6c..b2a5d4d3e1 100644 --- a/satpy/tests/reader_tests/test_hy2_scat_l2b_h5.py +++ b/satpy/tests/reader_tests/test_hy2_scat_l2b_h5.py @@ -509,6 +509,7 @@ def test_reading_attrs_nsoas(self): def test_properties(self): """Test platform_name.""" from datetime import datetime + from satpy.readers import load_reader filenames = [ 'W_XX-EUMETSAT-Darmstadt,SURFACE+SATELLITE,HY2B+SM_C_EUMP_20200326------_07077_o_250_l2b.h5', ]