diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index c2fa26dafa..e775944b5a 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -103,7 +103,7 @@ jobs: git+https://github.com/dask/distributed \ git+https://github.com/zarr-developers/zarr \ git+https://github.com/Unidata/cftime \ - git+https://github.com/mapbox/rasterio \ + git+https://github.com/rasterio/rasterio \ git+https://github.com/pydata/bottleneck \ git+https://github.com/pydata/xarray \ git+https://github.com/astropy/astropy; diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index caeabd5305..3d36ec4301 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -20,7 +20,7 @@ repos: - id: bandit args: [--ini, .bandit] - repo: https://github.com/pre-commit/mirrors-mypy - rev: 'v0.971' # Use the sha / tag you want to point at + rev: 'v0.982' # Use the sha / tag you want to point at hooks: - id: mypy additional_dependencies: diff --git a/continuous_integration/environment.yaml b/continuous_integration/environment.yaml index b86bc1e072..f8a94233bc 100644 --- a/continuous_integration/environment.yaml +++ b/continuous_integration/environment.yaml @@ -2,7 +2,7 @@ name: test-environment channels: - conda-forge dependencies: - - xarray + - xarray!=2022.9.0 - dask - distributed - donfig diff --git a/satpy/composites/ahi.py b/satpy/composites/ahi.py index edd195bbcf..bb96a94581 100644 --- a/satpy/composites/ahi.py +++ b/satpy/composites/ahi.py @@ -1,6 +1,4 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -# Copyright (c) 2015-2021 Satpy developers +# Copyright (c) 2022- Satpy developers # # This file is part of satpy. # @@ -15,30 +13,8 @@ # # You should have received a copy of the GNU General Public License along with # satpy. If not, see . -"""Composite classes for the AHI instrument.""" +"""Composite classes for AHI.""" -import logging - -from satpy.composites import GenericCompositor -from satpy.dataset import combine_metadata - -LOG = logging.getLogger(__name__) - - -class GreenCorrector(GenericCompositor): - """Corrector of the AHI green band to compensate for the deficit of chlorophyll signal.""" - - def __init__(self, *args, fractions=(0.85, 0.15), **kwargs): - """Set default keyword argument values.""" - # XXX: Should this be 0.93 and 0.07 - self.fractions = fractions - super(GreenCorrector, self).__init__(*args, **kwargs) - - def __call__(self, projectables, optional_datasets=None, **attrs): - """Boost vegetation effect thanks to NIR (0.8µm) band.""" - LOG.info('Boosting vegetation on green band') - - projectables = self.match_data_arrays(projectables) - new_green = sum(fraction * value for fraction, value in zip(self.fractions, projectables)) - new_green.attrs = combine_metadata(*projectables) - return super(GreenCorrector, self).__call__((new_green,), **attrs) +# The green corrector used to be defined here, but was moved to spectral.py +# in Satpy 0.38 because it also applies to FCI. +from .spectral import GreenCorrector # noqa: F401 diff --git a/satpy/composites/spectral.py b/satpy/composites/spectral.py new file mode 100644 index 0000000000..d0e6dc9330 --- /dev/null +++ b/satpy/composites/spectral.py @@ -0,0 +1,70 @@ +# Copyright (c) 2015-2022 Satpy developers +# +# This file is part of satpy. +# +# satpy is free software: you can redistribute it and/or modify it under the +# terms of the GNU General Public License as published by the Free Software +# Foundation, either version 3 of the License, or (at your option) any later +# version. +# +# satpy is distributed in the hope that it will be useful, but WITHOUT ANY +# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR +# A PARTICULAR PURPOSE. See the GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along with +# satpy. If not, see . +"""Composite classes for spectral adjustments.""" + +import logging + +from satpy.composites import GenericCompositor +from satpy.dataset import combine_metadata + +LOG = logging.getLogger(__name__) + + +class GreenCorrector(GenericCompositor): + """Corrector of the FCI or AHI green band. + + The green band in FCI and AHI deliberately misses the chlorophyll peak + in order to focus on aerosol and ash rather than on vegetation. This + affects true colour RGBs, because vegetation looks brown rather than green. + To make vegetation look greener again, this corrector allows + to simulate the green band as a fraction of two or more other channels. + + To be used, the composite takes two or more input channels and a parameter + ``fractions`` that should be a list of floats with the same length as the + number of channels. + + For example, to simulate an FCI corrected green composite, one could use + a combination of 93% from the green band (vis_05) and 7% from the + near-infrared 0.8 µm band (vis_08):: + + corrected_green: + compositor: !!python/name:satpy.composites.ahi.GreenCorrector + fractions: [0.93, 0.07] + prerequisites: + - name: vis_05 + modifiers: [sunz_corrected, rayleigh_corrected] + - name: vis_08 + modifiers: [sunz_corrected, rayleigh_corrected] + standard_name: toa_bidirectional_reflectance + + Other examples can be found in the ``fci.yaml`` and ``ahi.yaml`` composite + files in the satpy distribution. + """ + + def __init__(self, *args, fractions=(0.85, 0.15), **kwargs): + """Set default keyword argument values.""" + # XXX: Should this be 0.93 and 0.07 + self.fractions = fractions + super(GreenCorrector, self).__init__(*args, **kwargs) + + def __call__(self, projectables, optional_datasets=None, **attrs): + """Boost vegetation effect thanks to NIR (0.8µm) band.""" + LOG.info('Boosting vegetation on green band') + + projectables = self.match_data_arrays(projectables) + new_green = sum(fraction * value for fraction, value in zip(self.fractions, projectables)) + new_green.attrs = combine_metadata(*projectables) + return super(GreenCorrector, self).__call__((new_green,), **attrs) diff --git a/satpy/enhancements/viirs.py b/satpy/enhancements/viirs.py index 6a465f161b..627fc80220 100644 --- a/satpy/enhancements/viirs.py +++ b/satpy/enhancements/viirs.py @@ -38,7 +38,7 @@ def water_detection(img, **kwargs): @exclude_alpha @using_map_blocks def _water_detection(img_data): - data = np.asarray(img_data) + data = np.asarray(img_data).copy() data[data == 150] = 31 data[data == 199] = 18 data[data >= 200] = data[data >= 200] - 100 diff --git a/satpy/etc/composites/ahi.yaml b/satpy/etc/composites/ahi.yaml index 6c3d1860e6..ad77eeab50 100644 --- a/satpy/etc/composites/ahi.yaml +++ b/satpy/etc/composites/ahi.yaml @@ -16,7 +16,7 @@ modifiers: composites: green: - compositor: !!python/name:satpy.composites.ahi.GreenCorrector + compositor: !!python/name:satpy.composites.spectral.GreenCorrector # FUTURE: Set a wavelength...see what happens. Dependency finding # probably wouldn't work. prerequisites: @@ -31,7 +31,7 @@ composites: green_true_color_reproduction: # JMA True Color Reproduction green band # http://www.jma.go.jp/jma/jma-eng/satellite/introduction/TCR.html - compositor: !!python/name:satpy.composites.ahi.GreenCorrector + compositor: !!python/name:satpy.composites.spectral.GreenCorrector fractions: [0.6321, 0.2928, 0.0751] prerequisites: - name: B02 @@ -43,7 +43,7 @@ composites: standard_name: none green_nocorr: - compositor: !!python/name:satpy.composites.ahi.GreenCorrector + compositor: !!python/name:satpy.composites.spectral.GreenCorrector # FUTURE: Set a wavelength...see what happens. Dependency finding # probably wouldn't work. prerequisites: diff --git a/satpy/etc/composites/ami.yaml b/satpy/etc/composites/ami.yaml index 6049ef0a10..0c2635d3c7 100644 --- a/satpy/etc/composites/ami.yaml +++ b/satpy/etc/composites/ami.yaml @@ -2,7 +2,7 @@ sensor_name: visir/ami composites: green_raw: - compositor: !!python/name:satpy.composites.ahi.GreenCorrector + compositor: !!python/name:satpy.composites.spectral.GreenCorrector prerequisites: - name: VI005 modifiers: [sunz_corrected] @@ -12,7 +12,7 @@ composites: fractions: [0.85, 0.15] green: - compositor: !!python/name:satpy.composites.ahi.GreenCorrector + compositor: !!python/name:satpy.composites.spectral.GreenCorrector prerequisites: - name: VI005 modifiers: [sunz_corrected, rayleigh_corrected] @@ -22,7 +22,7 @@ composites: fractions: [0.85, 0.15] green_nocorr: - compositor: !!python/name:satpy.composites.ahi.GreenCorrector + compositor: !!python/name:satpy.composites.spectral.GreenCorrector prerequisites: - name: VI005 - name: VI008 diff --git a/satpy/etc/composites/fci.yaml b/satpy/etc/composites/fci.yaml index a3e558e455..853fcff835 100644 --- a/satpy/etc/composites/fci.yaml +++ b/satpy/etc/composites/fci.yaml @@ -2,6 +2,31 @@ sensor_name: visir/fci composites: + corrected_green: + description: > + The FCI green band at 0.51 µm deliberately misses the chlorophyl band, such that + the signal comes rather from aerosols and ash rather than vegetation. An effect + is that vegetation in a true colour RGB looks rather brown than green. Mixing in + some part of the NIR 0.8 channel reduced this effect. Note that the fractions + currently implemented are experimental and may change in future versions of Satpy. + compositor: !!python/name:satpy.composites.spectral.GreenCorrector + fractions: [0.93, 0.07] + prerequisites: + - name: vis_05 + modifiers: [sunz_corrected, rayleigh_corrected] + - name: vis_08 + modifiers: [sunz_corrected, rayleigh_corrected] + standard_name: toa_bidirectional_reflectance + + corrected_green_raw: + description: > + Alternative to corrected_green, but without solar zenith or rayleigh correction. + compositor: !!python/name:satpy.composites.spectral.GreenCorrector + fractions: [0.93, 0.07] + prerequisites: + - name: vis_05 + - name: vis_08 + standard_name: toa_bidirectional_reflectance binary_cloud_mask: # This will set all clear pixels to '0', all pixles with cloudy features (meteorological/dust/ash clouds) to '1' and @@ -11,3 +36,28 @@ composites: - name: 'cloud_state' lut: [.nan, 0, 1, 1, 1, 1, 1, 1, 0, .nan] standard_name: binary_cloud_mask + + true_color: + compositor: !!python/name:satpy.composites.SelfSharpenedRGB + description: > + FCI true color composite. The green band is simulated based on a combination of + channels. This simulation may change in future versions of Satpy. See the description + of the corrected_green composites for details. + prerequisites: + - name: vis_06 + modifiers: [sunz_corrected, rayleigh_corrected] + - name: corrected_green + - name: vis_04 + modifiers: [sunz_corrected, rayleigh_corrected] + standard_name: true_color + + true_color_raw_with_corrected_green: + compositor: !!python/name:satpy.composites.SelfSharpenedRGB + description: > + FCI true color without solar zenith or rayleigh corrections, but with the + corrected green composite. + prerequisites: + - name: vis_06 + - name: corrected_green_raw + - name: vis_04 + standard_name: true_color_raw diff --git a/satpy/etc/readers/ici_l1b_nc.yaml b/satpy/etc/readers/ici_l1b_nc.yaml index 8d8f271867..cbf6c9f993 100644 --- a/satpy/etc/readers/ici_l1b_nc.yaml +++ b/satpy/etc/readers/ici_l1b_nc.yaml @@ -12,7 +12,7 @@ reader: name: required: true frequency_double_sideband: - type: !!python/name:satpy.readers.aapp_mhs_amsub_l1c.FrequencyDoubleSideBand + type: !!python/name:satpy.readers.pmw_channels_definitions.FrequencyDoubleSideBand polarization: enum: - H diff --git a/satpy/etc/readers/mws_l1b_nc.yaml b/satpy/etc/readers/mws_l1b_nc.yaml index ce7fe527de..1f77bbe7a2 100644 --- a/satpy/etc/readers/mws_l1b_nc.yaml +++ b/satpy/etc/readers/mws_l1b_nc.yaml @@ -493,22 +493,6 @@ datasets: - mws_lon - mws_lat -# --- Land surface data --- - surface_type: - name: surface_type - standard_name: surface_type - file_type: mws_l1b_nc - coordinates: - - mws_lon - - mws_lat - terrain_elevation: - name: terrain_elevation - standard_name: terrain_elevation - file_type: mws_l1b_nc - coordinates: - - mws_lon - - mws_lat - file_types: mws_l1b_nc: diff --git a/satpy/etc/readers/seadas_l2.yaml b/satpy/etc/readers/seadas_l2.yaml index ca6ce3d956..99ff897bac 100644 --- a/satpy/etc/readers/seadas_l2.yaml +++ b/satpy/etc/readers/seadas_l2.yaml @@ -15,41 +15,53 @@ file_types: - '{platform_indicator:1s}1.{start_time:%y%j.%H%M}.seadas.hdf' file_reader: !!python/name:satpy.readers.seadas_l2.SEADASL2HDFFileHandler geo_resolution: 1000 + chlora_seadas_nc: + file_patterns: + # IMAPP-style filenames: + - '{platform_indicator:1s}1.{start_time:%y%j.%H%M}.seadas.nc' + file_reader: !!python/name:satpy.readers.seadas_l2.SEADASL2NetCDFFileHandler + geo_resolution: 1000 chlora_seadas_viirs: # SEADAS_npp_d20211118_t1728125_e1739327.hdf file_patterns: - 'SEADAS_{platform_indicator:s}_d{start_time:%Y%m%d_t%H%M%S%f}_e{end_time:%H%M%S%f}.hdf' file_reader: !!python/name:satpy.readers.seadas_l2.SEADASL2HDFFileHandler geo_resolution: 750 + chlora_seadas_viirs_nc: + # SEADAS_npp_d20211118_t1728125_e1739327.nc + file_patterns: + - 'SEADAS_{platform_indicator:s}_d{start_time:%Y%m%d_t%H%M%S%f}_e{end_time:%H%M%S%f}.nc' + file_reader: !!python/name:satpy.readers.seadas_l2.SEADASL2NetCDFFileHandler + geo_resolution: 750 datasets: longitude: name: longitude - file_type: [chlora_seadas, seadas_hdf_viirs] - file_key: longitude + file_type: [chlora_seadas_viirs_nc, chlora_seadas_nc, seadas_hdf_viirs, chlora_sedas] + file_key: ["navigation_data/longitude", "longitude"] resolution: 1000: - file_type: chlora_seadas + file_type: [chlora_seadas_nc, chlora_seadas] 750: - file_type: chlora_seadas_viirs + file_type: [chlora_seadas_viirs_nc, chlora_seadas_viirs] latitude: name: latitude - file_type: [chlora_seadas, seadas_hdf_viirs] - file_key: latitude + file_type: [chlora_seadas_viirs_nc, chlora_seadas_nc, seadas_hdf_viirs, chlora_sedas] + file_key: ["navigation_data/latitude", "latitude"] resolution: 1000: - file_type: chlora_seadas + file_type: [chlora_seadas_nc, chlora_seadas] 750: - file_type: chlora_seadas_viirs + file_type: [chlora_seadas_viirs_nc, chlora_seadas_viirs] chlor_a: name: chlor_a - file_type: [chlora_seadas, seadas_hdf_viirs] - file_key: chlor_a + file_type: [chlora_seadas_viirs_nc, chlora_seadas_nc, seadas_hdf_viirs, chlora_sedas] + file_key: ["geophysical_data/chlor_a", "chlor_a"] resolution: 1000: - file_type: chlora_seadas + file_type: [chlora_seadas_nc, chlora_seadas] 750: - file_type: chlora_seadas_viirs + file_type: [chlora_seadas_viirs_nc, chlora_seadas_viirs] coordinates: [longitude, latitude] diff --git a/satpy/etc/writers/awips_tiled.yaml b/satpy/etc/writers/awips_tiled.yaml index 6fcc5aa948..e761414904 100644 --- a/satpy/etc/writers/awips_tiled.yaml +++ b/satpy/etc/writers/awips_tiled.yaml @@ -129,6 +129,12 @@ templates: physical_element: raw_value: CLAVR-x Cloud Type units: {} + encoding: + dtype: int16 + _Unsigned: "true" + scale_factor: 0.5 + add_offset: 0.0 + _FillValue: -128 clavrx_cld_temp_acha: reader: clavrx name: cld_temp_acha @@ -153,6 +159,12 @@ templates: units: {} physical_element: raw_value: CLAVR-x Cloud Phase + encoding: + dtype: int16 + _Unsigned: "true" + scale_factor: 0.5 + add_offset: 0.0 + _FillValue: -128 clavrx_cld_opd_dcomp: reader: clavrx name: cld_opd_dcomp diff --git a/satpy/readers/abi_base.py b/satpy/readers/abi_base.py index 2f97c925cc..b74c4cf728 100644 --- a/satpy/readers/abi_base.py +++ b/satpy/readers/abi_base.py @@ -18,6 +18,7 @@ """Advance Baseline Imager reader base class for the Level 1b and l2+ reader.""" import logging +from contextlib import suppress from datetime import datetime import numpy as np @@ -76,7 +77,8 @@ def _rename_dims(nc): if 't' in nc.dims or 't' in nc.coords: nc = nc.rename({'t': 'time'}) if 'goes_lat_lon_projection' in nc: - nc = nc.rename({'lon': 'x', 'lat': 'y'}) + with suppress(ValueError): + nc = nc.rename({'lon': 'x', 'lat': 'y'}) return nc @property diff --git a/satpy/readers/hy2_scat_l2b_h5.py b/satpy/readers/hy2_scat_l2b_h5.py index 6bdb8f88a4..64520bae9a 100644 --- a/satpy/readers/hy2_scat_l2b_h5.py +++ b/satpy/readers/hy2_scat_l2b_h5.py @@ -107,11 +107,13 @@ def get_dataset(self, key, info): else: dim_map = {curr_dim: new_dim for curr_dim, new_dim in zip(data.dims, dims)} data = data.rename(dim_map) - data = self._mask_data(key['name'], data) - data = self._scale_data(key['name'], data) + data = self._mask_data(data) + data = self._scale_data(data) if key['name'] in 'wvc_lon': + _attrs = data.attrs data = xr.where(data > 180, data - 360., data) + data.attrs.update(_attrs) data.attrs.update(info) data.attrs.update(self.get_metadata()) data.attrs.update(self.get_variable_metadata()) @@ -120,13 +122,14 @@ def get_dataset(self, key, info): return data - def _scale_data(self, key_name, data): - return data * self[key_name].attrs['scale_factor'] + self[key_name].attrs['add_offset'] + def _scale_data(self, data): + return data * data.attrs['scale_factor'] + data.attrs['add_offset'] - def _mask_data(self, key_name, data): - data = xr.where(data == self[key_name].attrs['fill_value'], np.nan, data) - - valid_range = self[key_name].attrs['valid_range'] + def _mask_data(self, data): + _attrs = data.attrs + valid_range = data.attrs['valid_range'] + data = xr.where(data == data.attrs['fill_value'], np.nan, data) data = xr.where(data < valid_range[0], np.nan, data) data = xr.where(data > valid_range[1], np.nan, data) + data.attrs.update(_attrs) return data diff --git a/satpy/readers/seadas_l2.py b/satpy/readers/seadas_l2.py index 708b694fac..281a0132af 100644 --- a/satpy/readers/seadas_l2.py +++ b/satpy/readers/seadas_l2.py @@ -31,17 +31,16 @@ from datetime import datetime from .hdf4_utils import HDF4FileHandler +from .netcdf_utils import NetCDF4FileHandler -TIME_FORMAT = "%Y%j%H%M%S" - -class SEADASL2HDFFileHandler(HDF4FileHandler): +class _SEADASL2Base: """Simple handler of SEADAS L2 files.""" def __init__(self, filename, filename_info, filetype_info, apply_quality_flags=False): """Initialize file handler and determine if data quality flags should be applied.""" super().__init__(filename, filename_info, filetype_info) - self.apply_quality_flags = apply_quality_flags and "l2_flags" in self + self.apply_quality_flags = apply_quality_flags and self.l2_flags_var_name in self def _add_satpy_metadata(self, data): data.attrs["sensor"] = self.sensor_names @@ -57,7 +56,7 @@ def _rows_per_scan(self): raise ValueError(f"Don't know how to read data for sensors: {self.sensor_names}") def _platform_name(self): - platform = self["/attr/Mission"] + platform = self[self.platform_attr_name] platform_dict = {'NPP': 'Suomi-NPP', 'JPSS-1': 'NOAA-20', 'JPSS-2': 'NOAA-21'} @@ -66,38 +65,93 @@ def _platform_name(self): @property def start_time(self): """Get the starting observation time of this file's data.""" - start_time = self["/attr/Start Time"] - return datetime.strptime(start_time[:-3], TIME_FORMAT) + start_time = self[self.start_time_attr_name] + return datetime.strptime(start_time[:-3], self.time_format) @property def end_time(self): """Get the ending observation time of this file's data.""" - end_time = self["/attr/End Time"] - return datetime.strptime(end_time[:-3], TIME_FORMAT) + end_time = self[self.end_time_attr_name] + return datetime.strptime(end_time[:-3], self.time_format) @property def sensor_names(self): """Get sensor for the current file's data.""" # Example: MODISA or VIIRSN or VIIRSJ1 - sensor_name = self["/attr/Sensor Name"].lower() + sensor_name = self[self.sensor_attr_name].lower() if sensor_name.startswith("modis"): return {"modis"} return {"viirs"} def get_dataset(self, data_id, dataset_info): """Get DataArray for the specified DataID.""" - file_key = dataset_info.get("file_key", data_id["name"]) - data = self[file_key] - valid_range = data.attrs["valid_range"] - data = data.where(valid_range[0] <= data) - data = data.where(data <= valid_range[1]) - if self.apply_quality_flags and not ("lon" in file_key or "lat" in file_key): - l2_flags = self["l2_flags"] - mask = (l2_flags & 0b00000000010000000000000000000000) != 0 - data = data.where(~mask) + file_key, data = self._get_file_key_and_variable(data_id, dataset_info) + data = self._filter_by_valid_min_max(data) + data = self._rename_2d_dims_if_necessary(data) + data = self._mask_based_on_l2_flags(data) for attr_name in ("standard_name", "long_name", "units"): val = data.attrs[attr_name] if val[-1] == "\x00": data.attrs[attr_name] = data.attrs[attr_name][:-1] data = self._add_satpy_metadata(data) return data + + def _get_file_key_and_variable(self, data_id, dataset_info): + file_keys = dataset_info.get("file_key", data_id["name"]) + if not isinstance(file_keys, list): + file_keys = [file_keys] + for file_key in file_keys: + try: + data = self[file_key] + return file_key, data + except KeyError: + continue + raise KeyError(f"Unable to find any of the possible keys for {data_id}: {file_keys}") + + def _rename_2d_dims_if_necessary(self, data_arr): + if data_arr.ndim != 2 or data_arr.dims == ("y", "x"): + return data_arr + return data_arr.rename(dict(zip(data_arr.dims, ("y", "x")))) + + def _filter_by_valid_min_max(self, data_arr): + valid_range = self._valid_min_max(data_arr) + data_arr = data_arr.where(valid_range[0] <= data_arr) + data_arr = data_arr.where(data_arr <= valid_range[1]) + return data_arr + + def _valid_min_max(self, data_arr): + try: + return data_arr.attrs["valid_range"] + except KeyError: + return data_arr.attrs["valid_min"], data_arr.attrs["valid_max"] + + def _mask_based_on_l2_flags(self, data_arr): + standard_name = data_arr.attrs.get("standard_name", "") + if self.apply_quality_flags and not ("lon" in standard_name or "lat" in standard_name): + l2_flags = self[self.l2_flags_var_name] + l2_flags = self._rename_2d_dims_if_necessary(l2_flags) + mask = (l2_flags & 0b00000000010000000000000000000000) != 0 + data_arr = data_arr.where(~mask) + return data_arr + + +class SEADASL2NetCDFFileHandler(_SEADASL2Base, NetCDF4FileHandler): + """Simple handler of SEADAS L2 NetCDF4 files.""" + + start_time_attr_name = "/attr/time_coverage_start" + end_time_attr_name = "/attr/time_coverage_end" + time_format = "%Y-%m-%dT%H:%M:%S.%f" + platform_attr_name = "/attr/platform" + sensor_attr_name = "/attr/instrument" + l2_flags_var_name = "geophysical_data/l2_flags" + + +class SEADASL2HDFFileHandler(_SEADASL2Base, HDF4FileHandler): + """Simple handler of SEADAS L2 HDF4 files.""" + + start_time_attr_name = "/attr/Start Time" + end_time_attr_name = "/attr/End Time" + time_format = "%Y%j%H%M%S" + platform_attr_name = "/attr/Mission" + sensor_attr_name = "/attr/Sensor Name" + l2_flags_var_name = "l2_flags" diff --git a/satpy/readers/viirs_edr_active_fires.py b/satpy/readers/viirs_edr_active_fires.py index 2fdfaf6bcf..f1bcf4d3cc 100644 --- a/satpy/readers/viirs_edr_active_fires.py +++ b/satpy/readers/viirs_edr_active_fires.py @@ -92,12 +92,12 @@ def end_time(self): @property def sensor_name(self): """Name of sensor for this file.""" - return self["sensor"].lower() + return self["/attr/instrument_name"].lower() @property def platform_name(self): """Name of platform/satellite for this file.""" - return self["platform_name"] + return self["/attr/satellite_name"] class VIIRSActiveFiresTextFileHandler(BaseFileHandler): diff --git a/satpy/scene.py b/satpy/scene.py index dd1ca61dcd..06f36e561e 100644 --- a/satpy/scene.py +++ b/satpy/scene.py @@ -275,7 +275,8 @@ def _compare_swath_defs(compare_func: Callable, swath_defs: list[SwathDefinition def _key_func(swath_def: SwathDefinition) -> tuple: attrs = getattr(swath_def.lons, "attrs", {}) lon_ds_name = attrs.get("name") - return swath_def.shape[1], swath_def.shape[0], lon_ds_name + rev_shape = swath_def.shape[::-1] + return rev_shape + (lon_ds_name,) return compare_func(swath_defs, key=_key_func) def _gather_all_areas(self, datasets): diff --git a/satpy/tests/reader_tests/test_abi_l2_nc.py b/satpy/tests/reader_tests/test_abi_l2_nc.py index 02aaad977b..63014685f9 100644 --- a/satpy/tests/reader_tests/test_abi_l2_nc.py +++ b/satpy/tests/reader_tests/test_abi_l2_nc.py @@ -279,3 +279,69 @@ def test_get_area_def_latlon(self, adef): self.assertEqual(call_args[4], self.reader.ncols) self.assertEqual(call_args[5], self.reader.nlines) np.testing.assert_allclose(call_args[6], (-85.0, -20.0, -65.0, 20)) + + +class Test_NC_ABI_L2_area_AOD(unittest.TestCase): + """Test the NC_ABI_L2 reader for the AOD product.""" + + @mock.patch('satpy.readers.abi_base.xr') + def setUp(self, xr_): + """Create fake data for the tests.""" + from satpy.readers.abi_l2_nc import NC_ABI_L2 + proj = xr.DataArray( + [], + attrs={'semi_major_axis': 1., + 'semi_minor_axis': 1., + 'inverse_flattening': 1., + 'longitude_of_prime_meridian': 0.0, + } + ) + + proj_ext = xr.DataArray( + [], + attrs={'geospatial_westbound_longitude': -85.0, + 'geospatial_eastbound_longitude': -65.0, + 'geospatial_northbound_latitude': 20.0, + 'geospatial_southbound_latitude': -20.0, + 'geospatial_lat_center': 0.0, + 'geospatial_lon_center': -75.0, + }) + + x__ = xr.DataArray( + [0, 1], + attrs={'scale_factor': 2., 'add_offset': -1.}, + dims=('x',), + ) + y__ = xr.DataArray( + [0, 1], + attrs={'scale_factor': -2., 'add_offset': 1.}, + dims=('y',), + ) + fake_dataset = xr.Dataset( + data_vars={ + 'goes_lat_lon_projection': proj, + 'geospatial_lat_lon_extent': proj_ext, + 'x': x__, + 'y': y__, + 'RSR': xr.DataArray(np.ones((2, 2)), dims=('y', 'x')), + }, + ) + xr_.open_dataset.return_value = fake_dataset + + self.reader = NC_ABI_L2('filename', + {'platform_shortname': 'G16', 'observation_type': 'RSR', + 'scene_abbr': 'C', 'scan_mode': 'M3'}, + {'filetype': 'info'}) + + @mock.patch('satpy.readers.abi_base.geometry.AreaDefinition') + def test_get_area_def_xy(self, adef): + """Test the area generation.""" + self.reader.get_area_def(None) + + self.assertEqual(adef.call_count, 1) + call_args = tuple(adef.call_args)[0] + self.assertDictEqual(call_args[3], {'proj': 'latlong', 'a': 1.0, 'b': 1.0, 'fi': 1.0, 'pm': 0.0, + 'lon_0': -75.0, 'lat_0': 0.0}) + self.assertEqual(call_args[4], self.reader.ncols) + self.assertEqual(call_args[5], self.reader.nlines) + np.testing.assert_allclose(call_args[6], (-85.0, -20.0, -65.0, 20)) diff --git a/satpy/tests/reader_tests/test_hy2_scat_l2b_h5.py b/satpy/tests/reader_tests/test_hy2_scat_l2b_h5.py index 1664c46ed4..b2a5d4d3e1 100644 --- a/satpy/tests/reader_tests/test_hy2_scat_l2b_h5.py +++ b/satpy/tests/reader_tests/test_hy2_scat_l2b_h5.py @@ -40,6 +40,13 @@ class FakeHDF5FileHandler2(FakeHDF5FileHandler): """Swap-in HDF5 File Handler.""" + def __getitem__(self, key): + """Return copy of dataarray to prevent manipulating attributes in the original.""" + val = self.file_content[key] + if isinstance(val, xr.core.dataarray.DataArray): + val = val.copy() + return val + def _get_geo_data(self, num_rows, num_cols): geo = { 'wvc_lon': @@ -498,3 +505,20 @@ def test_reading_attrs_nsoas(self): with self.assertRaises(KeyError): self.assertEqual(res['wvc_lon'].attrs['L2B_Number_WVC_cells'], 10) self.assertEqual(res['wvc_lon'].attrs['L2B_Expected_WVC_Cells'], 10) + + def test_properties(self): + """Test platform_name.""" + from datetime import datetime + + from satpy.readers import load_reader + filenames = [ + 'W_XX-EUMETSAT-Darmstadt,SURFACE+SATELLITE,HY2B+SM_C_EUMP_20200326------_07077_o_250_l2b.h5', ] + + reader = load_reader(self.reader_configs) + files = reader.select_files_from_pathnames(filenames) + reader.create_filehandlers(files) + # Make sure we have some files + res = reader.load(['wvc_lon']) + self.assertEqual(res['wvc_lon'].platform_name, 'HY-2B') + self.assertEqual(res['wvc_lon'].start_time, datetime(2020, 3, 26, 1, 11, 7)) + self.assertEqual(res['wvc_lon'].end_time, datetime(2020, 3, 26, 2, 55, 40)) diff --git a/satpy/tests/reader_tests/test_seadas_l2.py b/satpy/tests/reader_tests/test_seadas_l2.py index dfa55a557d..2cc10a4344 100644 --- a/satpy/tests/reader_tests/test_seadas_l2.py +++ b/satpy/tests/reader_tests/test_seadas_l2.py @@ -19,7 +19,6 @@ import numpy as np import pytest -from pyhdf.SD import SD, SDC from pyresample.geometry import SwathDefinition from pytest_lazyfixture import lazy_fixture @@ -31,7 +30,7 @@ def seadas_l2_modis_chlor_a(tmp_path_factory): """Create MODIS SEADAS file.""" filename = "a1.21322.1758.seadas.hdf" full_path = str(tmp_path_factory.mktemp("seadas_l2") / filename) - return _create_seadas_chlor_a_file(full_path, "Aqua", "MODISA") + return _create_seadas_chlor_a_hdf4_file(full_path, "Aqua", "MODISA") @pytest.fixture(scope="module") @@ -39,7 +38,7 @@ def seadas_l2_viirs_npp_chlor_a(tmp_path_factory): """Create VIIRS NPP SEADAS file.""" filename = "SEADAS_npp_d20211118_t1728125_e1739327.hdf" full_path = str(tmp_path_factory.mktemp("seadas") / filename) - return _create_seadas_chlor_a_file(full_path, "NPP", "VIIRSN") + return _create_seadas_chlor_a_hdf4_file(full_path, "NPP", "VIIRSN") @pytest.fixture(scope="module") @@ -47,10 +46,11 @@ def seadas_l2_viirs_j01_chlor_a(tmp_path_factory): """Create VIIRS JPSS-01 SEADAS file.""" filename = "SEADAS_j01_d20211118_t1728125_e1739327.hdf" full_path = str(tmp_path_factory.mktemp("seadas") / filename) - return _create_seadas_chlor_a_file(full_path, "JPSS-1", "VIIRSJ1") + return _create_seadas_chlor_a_hdf4_file(full_path, "JPSS-1", "VIIRSJ1") -def _create_seadas_chlor_a_file(full_path, mission, sensor): +def _create_seadas_chlor_a_hdf4_file(full_path, mission, sensor): + from pyhdf.SD import SD, SDC h = SD(full_path, SDC.WRITE | SDC.CREATE) setattr(h, "Sensor Name", sensor) h.Mission = mission @@ -79,8 +79,8 @@ def _create_seadas_chlor_a_file(full_path, mission, sensor): "valid_range": (-90.0, 90.0), } } - _add_variable_to_file(h, "longitude", lon_info) - _add_variable_to_file(h, "latitude", lat_info) + _add_variable_to_hdf4_file(h, "longitude", lon_info) + _add_variable_to_hdf4_file(h, "latitude", lat_info) chlor_a_info = { "type": SDC.FLOAT32, @@ -93,7 +93,7 @@ def _create_seadas_chlor_a_file(full_path, mission, sensor): "valid_range": (0.001, 100.0), } } - _add_variable_to_file(h, "chlor_a", chlor_a_info) + _add_variable_to_hdf4_file(h, "chlor_a", chlor_a_info) l2_flags = np.zeros((5, 5), dtype=np.int32) l2_flags[2, 2] = -1 @@ -103,11 +103,11 @@ def _create_seadas_chlor_a_file(full_path, mission, sensor): "dim_labels": ["Number of Scan Lines", "Number of Pixel Control Points"], "attrs": {}, } - _add_variable_to_file(h, "l2_flags", l2_flags_info) + _add_variable_to_hdf4_file(h, "l2_flags", l2_flags_info) return [full_path] -def _add_variable_to_file(h, var_name, var_info): +def _add_variable_to_hdf4_file(h, var_name, var_info): v = h.create(var_name, var_info['type'], var_info['data'].shape) v[:] = var_info['data'] for dim_count, dimension_name in enumerate(var_info['dim_labels']): @@ -118,6 +118,85 @@ def _add_variable_to_file(h, var_name, var_info): setattr(v, attr_key, attr_val) +@pytest.fixture(scope="module") +def seadas_l2_modis_chlor_a_netcdf(tmp_path_factory): + """Create MODIS SEADAS NetCDF file.""" + filename = "t1.21332.1758.seadas.nc" + full_path = str(tmp_path_factory.mktemp("seadas_l2") / filename) + return _create_seadas_chlor_a_netcdf_file(full_path, "Terra", "MODIS") + + +def _create_seadas_chlor_a_netcdf_file(full_path, mission, sensor): + from netCDF4 import Dataset + nc = Dataset(full_path, "w") + nc.createDimension("number_of_lines", 5) + nc.createDimension("pixels_per_line", 5) + nc.instrument = sensor + nc.platform = mission + nc.time_coverage_start = "2021-11-18T17:58:53.191Z" + nc.time_coverage_end = "2021-11-18T18:05:51.214Z" + + lon_info = { + "data": np.zeros((5, 5), dtype=np.float32), + "dim_labels": ("number_of_lines", "pixels_per_line"), + "attrs": { + "long_name": "Longitude", + "standard_name": "longitude", + "units": "degrees_east", + "valid_min": -180.0, + "valid_max": 180.0, + } + } + lat_info = { + "data": np.zeros((5, 5), np.float32), + "dim_labels": ("number_of_lines", "pixels_per_line"), + "attrs": { + "long_name": "Latitude", + "standard_name": "latitude", + "units": "degrees_north", + "valid_min": -90.0, + "valid_max": 90.0, + } + } + nav_group = nc.createGroup("navigation_data") + _add_variable_to_netcdf_file(nav_group, "longitude", lon_info) + _add_variable_to_netcdf_file(nav_group, "latitude", lat_info) + + chlor_a_info = { + "data": np.ones((5, 5), np.float32), + "dim_labels": ("number_of_lines", "pixels_per_line"), + "attrs": { + "long_name": "Chlorophyll Concentration, OCI Algorithm", + "units": "mg m^-3", + "standard_name": "mass_concentration_of_chlorophyll_in_sea_water", + "valid_min": 0.001, + "valid_max": 100.0, + } + } + l2_flags = np.zeros((5, 5), dtype=np.int32) + l2_flags[2, 2] = -1 + l2_flags_info = { + "data": l2_flags, + "dim_labels": ("number_of_lines", "pixels_per_line"), + "attrs": { + "valid_min": -2147483648, + "valid_max": 2147483647, + }, + } + geophys_group = nc.createGroup("geophysical_data") + _add_variable_to_netcdf_file(geophys_group, "chlor_a", chlor_a_info) + _add_variable_to_netcdf_file(geophys_group, "l2_flags", l2_flags_info) + return [full_path] + + +def _add_variable_to_netcdf_file(nc, var_name, var_info): + v = nc.createVariable(var_name, var_info["data"].dtype.str[1:], dimensions=var_info["dim_labels"], + fill_value=var_info.get("fill_value")) + v[:] = var_info['data'] + for attr_key, attr_val in var_info['attrs'].items(): + setattr(v, attr_key, attr_val) + + class TestSEADAS: """Test the SEADAS L2 file reader.""" @@ -145,6 +224,7 @@ def test_scene_available_datasets(self, input_files): (lazy_fixture("seadas_l2_modis_chlor_a"), "Aqua", {"modis"}, 10), (lazy_fixture("seadas_l2_viirs_npp_chlor_a"), "Suomi-NPP", {"viirs"}, 16), (lazy_fixture("seadas_l2_viirs_j01_chlor_a"), "NOAA-20", {"viirs"}, 16), + (lazy_fixture("seadas_l2_modis_chlor_a_netcdf"), "Terra", {"modis"}, 10), ]) @pytest.mark.parametrize("apply_quality_flags", [False, True]) def test_load_chlor_a(self, input_files, exp_plat, exp_sensor, exp_rps, apply_quality_flags): @@ -153,6 +233,7 @@ def test_load_chlor_a(self, input_files, exp_plat, exp_sensor, exp_rps, apply_qu scene = Scene(reader='seadas_l2', filenames=input_files, reader_kwargs=reader_kwargs) scene.load(['chlor_a']) data_arr = scene['chlor_a'] + assert data_arr.dims == ("y", "x") assert data_arr.attrs['platform_name'] == exp_plat assert data_arr.attrs['sensor'] == exp_sensor assert data_arr.attrs['units'] == 'mg m^-3' diff --git a/satpy/tests/reader_tests/test_viirs_edr_active_fires.py b/satpy/tests/reader_tests/test_viirs_edr_active_fires.py index 9341ad2e21..df94283fba 100644 --- a/satpy/tests/reader_tests/test_viirs_edr_active_fires.py +++ b/satpy/tests/reader_tests/test_viirs_edr_active_fires.py @@ -61,8 +61,8 @@ def get_test_content(self, filename, filename_info, filename_type): """Mimic reader input file content.""" file_content = {} file_content['/attr/data_id'] = "AFMOD" - file_content['satellite_name'] = "npp" - file_content['sensor'] = 'VIIRS' + file_content['/attr/satellite_name'] = "NPP" + file_content['/attr/instrument_name'] = 'VIIRS' file_content['Fire Pixels/FP_latitude'] = DEFAULT_LATLON_FILE_DATA file_content['Fire Pixels/FP_longitude'] = DEFAULT_LATLON_FILE_DATA @@ -87,8 +87,8 @@ def get_test_content(self, filename, filename_info, filename_type): """Mimic reader input file content.""" file_content = {} file_content['/attr/data_id'] = "AFIMG" - file_content['satellite_name'] = "npp" - file_content['sensor'] = 'VIIRS' + file_content['/attr/satellite_name'] = "NPP" + file_content['/attr/instrument_name'] = 'VIIRS' file_content['Fire Pixels/FP_latitude'] = DEFAULT_LATLON_FILE_DATA file_content['Fire Pixels/FP_longitude'] = DEFAULT_LATLON_FILE_DATA diff --git a/satpy/tests/test_modifiers.py b/satpy/tests/test_modifiers.py index d8598a5b05..d43a1005d0 100644 --- a/satpy/tests/test_modifiers.py +++ b/satpy/tests/test_modifiers.py @@ -169,10 +169,10 @@ class TestNIRReflectance(unittest.TestCase): def setUp(self): """Set up the test case for the NIRReflectance compositor.""" - self.get_lonlats = mock.MagicMock() + get_lonlats = mock.MagicMock() self.lons, self.lats = 1, 2 - self.get_lonlats.return_value = (self.lons, self.lats) - area = mock.MagicMock(get_lonlats=self.get_lonlats) + get_lonlats.return_value = (self.lons, self.lats) + area = mock.MagicMock(get_lonlats=get_lonlats) self.start_time = 1 self.metadata = {'platform_name': 'Meteosat-11', @@ -241,7 +241,9 @@ def test_no_sunz_no_co2(self, calculator, apply_modifier_info, sza): info = {'modifiers': None} res = comp([self.nir, self.ir_], optional_datasets=[], **info) - self.get_lonlats.assert_called() + # due to copying of DataArrays, self.get_lonlats is not the same as the one that was called + # we must used the area from the final result DataArray + res.attrs["area"].get_lonlats.assert_called() sza.assert_called_with(self.start_time, self.lons, self.lats) self.refl_from_tbs.assert_called_with(self.da_sunz, self.nir.data, self.ir_.data, tb_ir_co2=None) assert np.allclose(res.data, self.refl * 100).compute() diff --git a/satpy/tests/test_scene.py b/satpy/tests/test_scene.py index bf2ffd948c..d17d69d01a 100644 --- a/satpy/tests/test_scene.py +++ b/satpy/tests/test_scene.py @@ -17,6 +17,7 @@ # satpy. If not, see . """Unit tests for scene.py.""" +import math import os import random import string @@ -709,7 +710,7 @@ def test_storage_options_from_reader_kwargs_per_reader(self): def _create_coarest_finest_data_array(shape, area_def, attrs=None): data_arr = xr.DataArray( - da.arange(shape[0] * shape[1]).reshape(shape), + da.arange(math.prod(shape)).reshape(shape), attrs={ 'area': area_def, }) @@ -735,8 +736,12 @@ def _create_coarsest_finest_area_def(shape, extents): def _create_coarsest_finest_swath_def(shape, extents, name_suffix): from pyresample import SwathDefinition - lons_arr = da.repeat(da.linspace(extents[0], extents[2], shape[1], dtype=np.float32)[None, :], shape[0], axis=0) - lats_arr = da.repeat(da.linspace(extents[1], extents[3], shape[0], dtype=np.float32)[:, None], shape[1], axis=1) + if len(shape) == 1: + lons_arr = da.linspace(extents[0], extents[2], shape[0], dtype=np.float32) + lats_arr = da.linspace(extents[1], extents[3], shape[0], dtype=np.float32) + else: + lons_arr = da.repeat(da.linspace(extents[0], extents[2], shape[1], dtype=np.float32)[None, :], shape[0], axis=0) + lats_arr = da.repeat(da.linspace(extents[1], extents[3], shape[0], dtype=np.float32)[:, None], shape[1], axis=1) lons_data_arr = xr.DataArray(lons_arr, attrs={"name": f"longitude{name_suffix}"}) lats_data_arr = xr.DataArray(lats_arr, attrs={"name": f"latitude1{name_suffix}"}) return SwathDefinition(lons_data_arr, lats_data_arr) @@ -754,6 +759,8 @@ class TestFinestCoarsestArea: _create_coarsest_finest_area_def((4, 10), (-1000.0, -1500.0, 1000.0, 1500.0))), (_create_coarsest_finest_swath_def((2, 5), (1000.0, 1500.0, -1000.0, -1500.0), "1"), _create_coarsest_finest_swath_def((4, 10), (1000.0, 1500.0, -1000.0, -1500.0), "1")), + (_create_coarsest_finest_swath_def((5,), (1000.0, 1500.0, -1000.0, -1500.0), "1"), + _create_coarsest_finest_swath_def((10,), (1000.0, 1500.0, -1000.0, -1500.0), "1")), ] ) def test_coarsest_finest_area_different_shape(self, coarse_area, fine_area): diff --git a/satpy/tests/test_utils.py b/satpy/tests/test_utils.py index 354c1886bb..3f0e055765 100644 --- a/satpy/tests/test_utils.py +++ b/satpy/tests/test_utils.py @@ -291,8 +291,10 @@ def test_get_satpos_from_satname(self, caplog): (lon, lat, alt) = get_satpos(data_arr, use_tle=True) assert "Orbital parameters missing from metadata" in caplog.text np.testing.assert_allclose( - (lon, lat, alt), - (119.39533705010592, -1.1491628298731498, 35803.19986408156)) + (lon, lat, alt), + (119.39533705010592, -1.1491628298731498, 35803.19986408156), + rtol=1e-4, + ) def test_make_fake_scene():