From 58ddaa0fdcf457066a77eebe6d0f3beb17d25a1a Mon Sep 17 00:00:00 2001 From: "Adam.Dybbroe" Date: Wed, 1 Jun 2022 11:42:24 +0200 Subject: [PATCH 01/25] Add first draft reader for MWS onboard EPS-SG-A Signed-off-by: Adam.Dybbroe --- satpy/etc/readers/mws_l1b_nc.yaml | 255 ++++++++++++++++++++++++++++ satpy/readers/aapp_mhs_amsub_l1c.py | 5 +- satpy/readers/mws_l1b.py | 194 +++++++++++++++++++++ satpy/readers/yaml_reader.py | 2 +- 4 files changed, 452 insertions(+), 4 deletions(-) create mode 100644 satpy/etc/readers/mws_l1b_nc.yaml create mode 100644 satpy/readers/mws_l1b.py diff --git a/satpy/etc/readers/mws_l1b_nc.yaml b/satpy/etc/readers/mws_l1b_nc.yaml new file mode 100644 index 0000000000..ecfced95ff --- /dev/null +++ b/satpy/etc/readers/mws_l1b_nc.yaml @@ -0,0 +1,255 @@ +reader: + name: mws_l1b_nc + description: EPS-SG l1b Reader for reading the MWS (Microwave Sounder) level-1 data + reader: !!python/name:satpy.readers.yaml_reader.FileYAMLReader + sensors: [mws,] + default_channels: [] + + data_identification_keys: + name: + required: true + frequency_double_sideband: + type: !!python/name:satpy.readers.aapp_mhs_amsub_l1c.FrequencyDoubleSideBand + frequency_range: + type: !!python/name:satpy.readers.aapp_mhs_amsub_l1c.FrequencyRange + resolution: + polarization: + enum: + - QH + - QV + calibration: + enum: + - brightness_temperature + transitive: true + modifiers: + required: true + default: [] + type: !!python/name:satpy.dataset.ModifierTuple + +datasets: + '1': + name: '1' + frequency_range: + central: 23.8 + bandwidth: 0.270 + unit: GHz + polarization: 'QH' + resolution: 40000 + calibration: + brightness_temperature: + standard_name: toa_brightness_temperature + coordinates: + - mws_lon + - mws_lat + file_type: mws_l1b_nc + + '2': + name: '2' + frequency_range: + central: 31.4 + bandwidth: 0.180 + unit: GHz + polarization: 'QH' + resolution: 40000 + calibration: + brightness_temperature: + standard_name: toa_brightness_temperature + coordinates: + - mws_lon + - mws_lat + file_type: mws_l1b_nc + '3': + name: '3' + frequency_range: + central: 50.3 + bandwidth: 0.180 + unit: GHz + polarization: 'QH' + resolution: 20000 + calibration: + brightness_temperature: + standard_name: toa_brightness_temperature + coordinates: + - mws_lon + - mws_lat + file_type: mws_l1b_nc + '4': + name: '4' + frequency_range: + central: 52.8 + bandwidth: 0.400 + unit: GHz + polarization: 'QH' + resolution: 20000 + calibration: + brightness_temperature: + standard_name: toa_brightness_temperature + coordinates: + - mws_lon + - mws_lat + file_type: mws_l1b_nc + '5': + name: '5' + frequency_double_sideband: + central: 53.246 + side: 0.08 + bandwidth: 0.140 + unit: GHz + polarization: 'QH' + resolution: 20000 + calibration: + brightness_temperature: + standard_name: toa_brightness_temperature + coordinates: + - mws_lon + - mws_lat + file_type: mws_l1b_nc + '6': + name: '6' + frequency_double_sideband: + central: 53.596 + side: 0.115 + bandwidth: 0.170 + unit: GHz + polarization: 'QH' + resolution: 20000 + calibration: + brightness_temperature: + standard_name: toa_brightness_temperature + coordinates: + - mws_lon + - mws_lat + file_type: mws_l1b_nc + '7': + name: '7' + frequency_double_sideband: + central: 53.948 + side: 0.081 + bandwidth: 0.142 + unit: GHz + polarization: 'QH' + resolution: 20000 + calibration: + brightness_temperature: + standard_name: toa_brightness_temperature + coordinates: + - mws_lon + - mws_lat + file_type: mws_l1b_nc + '8': + name: '8' + frequency_range: + central: 54.4 + bandwidth: 0.400 + unit: GHz + polarization: 'QH' + resolution: 20000 + calibration: + brightness_temperature: + standard_name: toa_brightness_temperature + coordinates: + - mws_lon + - mws_lat + file_type: mws_l1b_nc + '9': + name: '9' + frequency_range: + central: 54.94 + bandwidth: 0.400 + unit: GHz + polarization: 'QH' + resolution: 20000 + calibration: + brightness_temperature: + standard_name: toa_brightness_temperature + coordinates: + - mws_lon + - mws_lat + file_type: mws_l1b_nc + '10': + name: '10' + frequency_range: + central: 55.5 + bandwidth: 0.330 + unit: GHz + polarization: 'QH' + resolution: 20000 + calibration: + brightness_temperature: + standard_name: toa_brightness_temperature + coordinates: + - mws_lon + - mws_lat + file_type: mws_l1b_nc + '11': + name: '11' + frequency_range: + central: 57.290344 + bandwidth: 0.330 + unit: GHz + polarization: 'QH' + resolution: 20000 + calibration: + brightness_temperature: + standard_name: toa_brightness_temperature + coordinates: + - mws_lon + - mws_lat + file_type: mws_l1b_nc + '12': + #57.290344±0.217 + name: '12' + frequency_double_sideband: + central: 57.290344 + side: 0.217 + bandwidth: 0.078 + unit: GHz + polarization: 'QH' + resolution: 20000 + calibration: + brightness_temperature: + standard_name: toa_brightness_temperature + coordinates: + - mws_lon + - mws_lat + file_type: mws_l1b_nc + '13': + #57.290344±0.3222±0.048 + name: '13' + frequency_quadrouble_sideband: + central: 57.290344 + side: 0.3222 + sideside: 0.048 + bandwidth: 0.036 + unit: GHz + polarization: 'QH' + resolution: 20000 + calibration: + brightness_temperature: + standard_name: toa_brightness_temperature + coordinates: + - mws_lon + - mws_lat + file_type: mws_l1b_nc + + + mws_lat: + name: mws_lat + resolution: 40000 + file_type: mws_l1b_nc + standard_name: latitude + units: degrees_north + + mws_lon: + name: mws_lon + resolution: 40000 + file_type: mws_l1b_nc + standard_name: longitude + units: degrees_east + +file_types: + mws_l1b_nc: + # EPS-SG_MWS-1B-RAD.nc + # W_XX-EUMETSAT-Darmstadt,SAT,SGA1-MWS-1B-RAD_C_EUMT_20210609095009_G_D_20070912084321_20070912102225_T_N____.nc + file_reader: !!python/name:satpy.readers.mws_l1b.MWSL1BFile + file_patterns: ['W_XX-EUMETSAT-Darmstadt,SAT,{platform_shortname}-MWS-1B-RAD_C_EUMT_{processing_time:%Y%m%d%H%M%S}_G_D_{start_time:%Y%m%d%H%M%S}_{end_time:%Y%m%d%H%M%S}_T_N____.nc'] diff --git a/satpy/readers/aapp_mhs_amsub_l1c.py b/satpy/readers/aapp_mhs_amsub_l1c.py index 4a2acd25d2..320700d502 100644 --- a/satpy/readers/aapp_mhs_amsub_l1c.py +++ b/satpy/readers/aapp_mhs_amsub_l1c.py @@ -1,7 +1,7 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- -# Copyright (c) 2020, 2021 Pytroll developers +# Copyright (c) 2020, 2021, 2022 Pytroll developers # Author(s): @@ -304,8 +304,7 @@ class MHS_AMSUB_AAPPL1CFile(AAPPL1BaseFileHandler): def __init__(self, filename, filename_info, filetype_info): """Initialize object information by reading the input file.""" - super(MHS_AMSUB_AAPPL1CFile, self).__init__(filename, filename_info, - filetype_info) + super().__init__(filename, filename_info, filetype_info) self.channels = {i: None for i in MHS_AMSUB_CHANNEL_NAMES} self.units = {i: 'brightness_temperature' for i in MHS_AMSUB_CHANNEL_NAMES} diff --git a/satpy/readers/mws_l1b.py b/satpy/readers/mws_l1b.py new file mode 100644 index 0000000000..772c7d74d1 --- /dev/null +++ b/satpy/readers/mws_l1b.py @@ -0,0 +1,194 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +# Copyright (c) 2022 Adam.Dybbroe + +# Author(s): + +# Adam.Dybbroe + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +"""Reader for the EPS-SG Microwave Sounder (MWS) level-1b data. + +Documentation: https://www.eumetsat.int/media/44139 +""" + +import logging + +import numpy as np +from netCDF4 import default_fillvals + +from .netcdf_utils import NetCDF4FileHandler + +logger = logging.getLogger(__name__) + + +# dict containing all available auxiliary data parameters to be read using the index map. Keys are the +# parameter name and values are the paths to the variable inside the netcdf + +AUX_DATA = { + 'subsatellite_latitude': 'state/platform/subsatellite_latitude', + 'subsatellite_longitude': 'state/platform/subsatellite_longitude', + 'platform_altitude': 'state/platform/platform_altitude', + 'subsolar_latitude': 'state/celestial/subsolar_latitude', + 'subsolar_longitude': 'state/celestial/subsolar_longitude', + 'earth_sun_distance': 'state/celestial/earth_sun_distance', + 'sun_satellite_distance': 'state/celestial/sun_satellite_distance', + 'time': 'time', + 'swath_number': 'data/swath_number', + 'swath_direction': 'data/swath_direction', + 'mws_lat': 'data/navigation/mws_lat', + 'mws_lon': 'data/navigation/mws_lon', +} + +MWS_CHANNEL_NAMES_TO_NUMBER = {'1': 1, '2': 2, '3': 3, '4': 4, + '5': 5, '6': 6, '7': 7, '8': 8} + +MWS_CHANNEL_NAMES = ['1', '2', '3', '4', '5', '6'] + + +def get_channel_index_from_name(chname): + """Get the MWS channel index from the channel name.""" + chindex = MWS_CHANNEL_NAMES_TO_NUMBER.get(chname, 0) - 1 + if 0 <= chindex < 24: + return chindex + raise AttributeError("Channel name %s not supported: " % chname) + + +def _get_aux_data_name_from_dsname(dsname): + aux_data_name = [key for key in AUX_DATA.keys() if key in dsname] + if len(aux_data_name) > 0: + return aux_data_name[0] + + return None + + +class MWSL1BFile(NetCDF4FileHandler): + """Class implementing the EPS-SG-A1 MWS L1b Filehandler. + + This class implements the European Polar System Second Generation (EPS-SG) + Microwave Sounder (MWS) Level-1b NetCDF reader. It is designed to be used + through the :class:`~satpy.Scene` class using the :mod:`~satpy.Scene.load` + method with the reader ``"mws_l1b_nc"``. + + """ + + # FIXME! + # + # After launch: translate to Metop-X instead? + _platform_name_translate = { + "SGA1": "Metop-SG-A1", + "SGA2": "Metop-SG-A2", + "SGA3": "Metop-SG-A3"} + + def __init__(self, filename, filename_info, filetype_info): + """Initialize file handler.""" + super().__init__(filename, filename_info, + filetype_info, + cache_var_size=10000, + cache_handle=True) + logger.debug('Reading: {}'.format(self.filename)) + logger.debug('Start: {}'.format(self.start_time)) + logger.debug('End: {}'.format(self.end_time)) + + self._cache = {} + + self._channel_names = MWS_CHANNEL_NAMES + + @property + def start_time(self): + """Get start time.""" + return self.filename_info['start_time'] + + @property + def end_time(self): + """Get end time.""" + return self.filename_info['end_time'] + + def get_dataset(self, key, info=None): + """Load a dataset.""" + logger.debug('Reading {} from {}'.format(key['name'], self.filename)) + + if _get_aux_data_name_from_dsname(key['name']) is not None: + return self._get_dataset_aux_data(key['name'], info=info) + elif any(lb in key['name'] for lb in {"1", "2", "3", "4"}): + return self._get_dataset_channel(key, info=info) + else: + raise ValueError("Unknown dataset key, not a channel, quality or auxiliary data: " + f"{key['name']:s}") + + def _standardize_dims(self, variable): + """Standardize dims to y, x.""" + if 'n_scans' in variable.dims: + variable = variable.rename({'n_fovs': 'x', 'n_scans': 'y'}) + if variable.dims[0] == 'x': + variable = variable.transpose('y', 'x') + return variable + + def _get_dataset_channel(self, key, info=None): + """Load dataset corresponding to channel measurement. + + Load a dataset when the key refers to a measurand, whether uncalibrated + (counts) or calibrated in terms of brightness temperature, radiance, or + reflectance. + """ + # Get the dataset + # Get metadata for given dataset + grp_pth = 'data/calibration/mws_toa_brightness_temperature' + channel_index = get_channel_index_from_name(key['name']) + + data = self[grp_pth][:, :, channel_index] + attrs = data.attrs.copy() + + fv = attrs.pop( + "FillValue", + default_fillvals.get(data.dtype.str[1:], np.nan)) + vr = attrs.get("valid_range", [-np.inf, np.inf]) + if key['calibration'] == "counts": + attrs["_FillValue"] = fv + nfv = fv + else: + nfv = np.nan + data = data.where(data >= vr[0], nfv) + data = data.where(data <= vr[1], nfv) + + # Manage the attributes of the dataset + data.attrs.setdefault('units', None) + data.attrs.update(info) + # variable.attrs.update(self._get_global_attributes()) # FIXME! See VII reader + + return data + + def _get_dataset_aux_data(self, dsname, info=None): + """Get the auxiliary data arrays using the index map.""" + # Geolocation: + if dsname in ['mws_lat', 'mws_lon']: + var_key = AUX_DATA.get(dsname) + else: + raise NotImplementedError("Only lons and lats supported - no other auxillary data yet...") + + try: + variable = self[var_key] + except KeyError: + logger.warning("Could not find key %s in NetCDF file, no valid Dataset created", var_key) + return None + + # Scale the data: + variable.data = variable.data * variable.attrs['scale_factor'] + variable.attrs['add_offset'] + + # Manage the attributes of the dataset + variable.attrs.setdefault('units', None) + variable.attrs.update(info) + # variable.attrs.update(self._get_global_attributes()) # FIXME! See VII reader + return variable diff --git a/satpy/readers/yaml_reader.py b/satpy/readers/yaml_reader.py index 1d83466c81..3d64d71690 100644 --- a/satpy/readers/yaml_reader.py +++ b/satpy/readers/yaml_reader.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- -# Copyright (c) 2016-2019, 2021 Satpy developers +# Copyright (c) 2016-2022 Satpy developers # # This file is part of satpy. # From e7b4e63a00831d3154105be6d1cf44927102b6e6 Mon Sep 17 00:00:00 2001 From: "Adam.Dybbroe" Date: Tue, 14 Jun 2022 11:13:08 +0200 Subject: [PATCH 02/25] Refactor frequency handling Signed-off-by: Adam.Dybbroe --- satpy/etc/readers/mhs_l1c_aapp.yaml | 4 +- satpy/etc/readers/mws_l1b_nc.yaml | 189 ++++++++++++++++++++- satpy/readers/aapp_mhs_amsub_l1c.py | 247 ---------------------------- satpy/tests/test_dataset.py | 87 +++++++--- 4 files changed, 251 insertions(+), 276 deletions(-) diff --git a/satpy/etc/readers/mhs_l1c_aapp.yaml b/satpy/etc/readers/mhs_l1c_aapp.yaml index dee4402ee0..3ace72a927 100644 --- a/satpy/etc/readers/mhs_l1c_aapp.yaml +++ b/satpy/etc/readers/mhs_l1c_aapp.yaml @@ -9,9 +9,9 @@ reader: name: required: true frequency_double_sideband: - type: !!python/name:satpy.readers.aapp_mhs_amsub_l1c.FrequencyDoubleSideBand + type: !!python/name:satpy.readers.pmw_channels_definitions.FrequencyDoubleSideBand frequency_range: - type: !!python/name:satpy.readers.aapp_mhs_amsub_l1c.FrequencyRange + type: !!python/name:satpy.readers.pmw_channels_definitions.FrequencyRange resolution: polarization: enum: diff --git a/satpy/etc/readers/mws_l1b_nc.yaml b/satpy/etc/readers/mws_l1b_nc.yaml index ecfced95ff..0f3aeffcbc 100644 --- a/satpy/etc/readers/mws_l1b_nc.yaml +++ b/satpy/etc/readers/mws_l1b_nc.yaml @@ -8,10 +8,12 @@ reader: data_identification_keys: name: required: true + frequency_quadruple_sideband: + type: !!python/name:satpy.readers.pmw_channels_definitions.FrequencyQuadrupleSideBand frequency_double_sideband: - type: !!python/name:satpy.readers.aapp_mhs_amsub_l1c.FrequencyDoubleSideBand + type: !!python/name:satpy.readers.pmw_channels_definitions.FrequencyDoubleSideBand frequency_range: - type: !!python/name:satpy.readers.aapp_mhs_amsub_l1c.FrequencyRange + type: !!python/name:satpy.readers.pmw_channels_definitions.FrequencyRange resolution: polarization: enum: @@ -42,7 +44,6 @@ datasets: - mws_lon - mws_lat file_type: mws_l1b_nc - '2': name: '2' frequency_range: @@ -216,7 +217,7 @@ datasets: '13': #57.290344±0.3222±0.048 name: '13' - frequency_quadrouble_sideband: + frequency_quadruple_sideband: central: 57.290344 side: 0.3222 sideside: 0.048 @@ -231,6 +232,186 @@ datasets: - mws_lon - mws_lat file_type: mws_l1b_nc + '14': + #57.290344±0.3222±0.022 + name: '14' + frequency_quadruple_sideband: + central: 57.290344 + side: 0.3222 + sideside: 0.022 + bandwidth: 0.016 + unit: GHz + polarization: 'QH' + resolution: 20000 + calibration: + brightness_temperature: + standard_name: toa_brightness_temperature + coordinates: + - mws_lon + - mws_lat + file_type: mws_l1b_nc + '15': + #57.290344±0.3222±0.010 + name: '15' + frequency_quadruple_sideband: + central: 57.290344 + side: 0.3222 + sideside: 0.010 + bandwidth: 0.008 + unit: GHz + polarization: 'QH' + resolution: 20000 + calibration: + brightness_temperature: + standard_name: toa_brightness_temperature + coordinates: + - mws_lon + - mws_lat + file_type: mws_l1b_nc + '16': + #57.290344±0.3222±0.0045 + name: '16' + frequency_quadruple_sideband: + central: 57.290344 + side: 0.3222 + sideside: 0.0045 + bandwidth: 0.004 + unit: GHz + polarization: 'QH' + resolution: 20000 + calibration: + brightness_temperature: + standard_name: toa_brightness_temperature + coordinates: + - mws_lon + - mws_lat + file_type: mws_l1b_nc + '17': + name: '17' + frequency_range: + central: 89.0 + bandwidth: 4.0 + unit: GHz + polarization: 'QV' + resolution: 17000 + calibration: + brightness_temperature: + standard_name: toa_brightness_temperature + coordinates: + - mws_lon + - mws_lat + file_type: mws_l1b_nc + '18': + name: '18' + # FIXME! Is this a souble side band or what? MWS-18; 164–167; 2 x 1350; QH + frequency_range: + central: 166.0 + bandwidth: 2.700 + unit: GHz + polarization: 'QH' + resolution: 17000 + calibration: + brightness_temperature: + standard_name: toa_brightness_temperature + coordinates: + - mws_lon + - mws_lat + file_type: mws_l1b_nc + '19': + name: '19' + frequency_double_sideband: + central: 183.311 + side: 7.0 + bandwidth: 2.0 + unit: GHz + polarization: 'QV' + resolution: 20000 + calibration: + brightness_temperature: + standard_name: toa_brightness_temperature + coordinates: + - mws_lon + - mws_lat + file_type: mws_l1b_nc + '20': + name: '20' + frequency_double_sideband: + central: 183.311 + side: 4.5 + bandwidth: 2.0 + unit: GHz + polarization: 'QV' + resolution: 20000 + calibration: + brightness_temperature: + standard_name: toa_brightness_temperature + coordinates: + - mws_lon + - mws_lat + file_type: mws_l1b_nc + '21': + name: '21' + frequency_double_sideband: + central: 183.311 + side: 3.0 + bandwidth: 1.0 + unit: GHz + polarization: 'QV' + resolution: 20000 + calibration: + brightness_temperature: + standard_name: toa_brightness_temperature + coordinates: + - mws_lon + - mws_lat + file_type: mws_l1b_nc + '22': + name: '22' + frequency_double_sideband: + central: 183.311 + side: 1.8 + bandwidth: 1.0 + unit: GHz + polarization: 'QV' + resolution: 20000 + calibration: + brightness_temperature: + standard_name: toa_brightness_temperature + coordinates: + - mws_lon + - mws_lat + file_type: mws_l1b_nc + '23': + name: '23' + frequency_double_sideband: + central: 183.311 + side: 1.0 + bandwidth: 0.5 + unit: GHz + polarization: 'QV' + resolution: 20000 + calibration: + brightness_temperature: + standard_name: toa_brightness_temperature + coordinates: + - mws_lon + - mws_lat + file_type: mws_l1b_nc + '24': + name: '24' + frequency_range: + central: 229. + bandwidth: 2.0 + unit: GHz + polarization: 'QV' + resolution: 17000 + calibration: + brightness_temperature: + standard_name: toa_brightness_temperature + coordinates: + - mws_lon + - mws_lat + file_type: mws_l1b_nc mws_lat: diff --git a/satpy/readers/aapp_mhs_amsub_l1c.py b/satpy/readers/aapp_mhs_amsub_l1c.py index 320700d502..f590214953 100644 --- a/satpy/readers/aapp_mhs_amsub_l1c.py +++ b/satpy/readers/aapp_mhs_amsub_l1c.py @@ -3,10 +3,6 @@ # Copyright (c) 2020, 2021, 2022 Pytroll developers -# Author(s): - -# Adam Dybbroe - # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or @@ -27,9 +23,6 @@ """ import logging -import numbers -from contextlib import suppress -from typing import NamedTuple import dask.array as da import numpy as np @@ -59,246 +52,6 @@ MHS_AMSUB_PLATFORMS = ['Metop-A', 'Metop-B', 'Metop-C', 'NOAA-18', 'NOAA-19'] -class FrequencyDoubleSideBandBase(NamedTuple): - """Base class for a frequency double side band. - - Frequency Double Side Band is supposed to describe the special type of bands - commonly used in humidty sounding from Passive Microwave Sensors. When the - absorption band being observed is symmetrical it is advantageous (giving - better NeDT) to sense in a band both right and left of the central - absorption frequency. - - This is needed because of this bug: https://bugs.python.org/issue41629 - - """ - - central: float - side: float - bandwidth: float - unit: str = "GHz" - - -class FrequencyDoubleSideBand(FrequencyDoubleSideBandBase): - """The frequency double side band class. - - The elements of the double-side-band type frequency band are the central - frquency, the relative side band frequency (relative to the center - left - and right) and their bandwidths, and optionally a unit (defaults to - GHz). No clever unit conversion is done here, it's just used for checking - that two ranges are comparable. - - Frequency Double Side Band is supposed to describe the special type of bands - commonly used in humidty sounding from Passive Microwave Sensors. When the - absorption band being observed is symmetrical it is advantageous (giving - better NeDT) to sense in a band both right and left of the central - absorption frequency. - - """ - - def __eq__(self, other): - """Return if two channel frequencies are equal. - - Args: - other (tuple or scalar): (central frq, side band frq and band width frq) or scalar frq - - Return: - True if other is a scalar and min <= other <= max, or if other is - a tuple equal to self, False otherwise. - - """ - if other is None: - return False - if isinstance(other, numbers.Number): - return other in self - if isinstance(other, (tuple, list)) and len(other) == 3: - return other in self - return super().__eq__(other) - - def __ne__(self, other): - """Return the opposite of `__eq__`.""" - return not self == other - - def __lt__(self, other): - """Compare to another frequency.""" - if other is None: - return False - return super().__lt__(other) - - def __gt__(self, other): - """Compare to another frequency.""" - if other is None: - return True - return super().__gt__(other) - - def __hash__(self): - """Hash this tuple.""" - return tuple.__hash__(self) - - def __str__(self): - """Format for print out.""" - return "{0.central} {0.unit} ({0.side}_{0.bandwidth} {0.unit})".format(self) - - def __contains__(self, other): - """Check if this double-side-band 'contains' *other*.""" - if other is None: - return False - if isinstance(other, numbers.Number): - if (self.central + self.side - self.bandwidth/2. <= other - <= self.central + self.side + self.bandwidth/2.): - return True - if (self.central - self.side - self.bandwidth/2. <= other - <= self.central - self.side + self.bandwidth/2.): - return True - return False - - if isinstance(other, (tuple, list)) and len(other) == 3: - return ((self.central - self.side - self.bandwidth/2. <= - other[0] - other[1] - other[2]/2. and - self.central - self.side + self.bandwidth/2. >= - other[0] - other[1] + other[2]/2.) or - (self.central + self.side - self.bandwidth/2. <= - other[0] + other[1] - other[2]/2. and - self.central + self.side + self.bandwidth/2. >= - other[0] + other[1] + other[2]/2.)) - - with suppress(AttributeError): - if self.unit != other.unit: - raise NotImplementedError("Can't compare frequency ranges with different units.") - return ((self.central - self.side - self.bandwidth/2. <= - other.central - other.side - other.bandwidth/2. and - self.central - self.side + self.bandwidth/2. >= - other.central - other.side + other.bandwidth/2.) or - (self.central + self.side - self.bandwidth/2. <= - other.central + other.side - other.bandwidth/2. and - self.central + self.side + self.bandwidth/2. >= - other.central + other.side + other.bandwidth/2.)) - - return False - - def distance(self, value): - """Get the distance from value.""" - if self == value: - try: - left_side_dist = abs(value.central - value.side - (self.central - self.side)) - right_side_dist = abs(value.central + value.side - (self.central + self.side)) - return min(left_side_dist, right_side_dist) - except AttributeError: - if isinstance(value, (tuple, list)): - return abs((value[0] - value[1]) - (self.central - self.side)) - - left_side_dist = abs(value - (self.central - self.side)) - right_side_dist = abs(value - (self.central + self.side)) - return min(left_side_dist, right_side_dist) - else: - return np.inf - - @classmethod - def convert(cls, frq): - """Convert `frq` to this type if possible.""" - if isinstance(frq, dict): - return cls(**frq) - return frq - - -class FrequencyRangeBase(NamedTuple): - """Base class for frequency ranges. - - This is needed because of this bug: https://bugs.python.org/issue41629 - """ - - central: float - bandwidth: float - unit: str = "GHz" - - -class FrequencyRange(FrequencyRangeBase): - """The Frequency range class. - - The elements of the range are central and bandwidth values, and optionally - a unit (defaults to GHz). No clever unit conversion is done here, it's just - used for checking that two ranges are comparable. - - This type is used for passive microwave sensors. - - """ - - def __eq__(self, other): - """Return if two channel frequencies are equal. - - Args: - other (tuple or scalar): (central frq, band width frq) or scalar frq - - Return: - True if other is a scalar and min <= other <= max, or if other is - a tuple equal to self, False otherwise. - - """ - if other is None: - return False - if isinstance(other, numbers.Number): - return other in self - if isinstance(other, (tuple, list)) and len(other) == 2: - return self[:2] == other - return super().__eq__(other) - - def __ne__(self, other): - """Return the opposite of `__eq__`.""" - return not self == other - - def __lt__(self, other): - """Compare to another frequency.""" - if other is None: - return False - return super().__lt__(other) - - def __gt__(self, other): - """Compare to another frequency.""" - if other is None: - return True - return super().__gt__(other) - - def __hash__(self): - """Hash this tuple.""" - return tuple.__hash__(self) - - def __str__(self): - """Format for print out.""" - return "{0.central} {0.unit} ({0.bandwidth} {0.unit})".format(self) - - def __contains__(self, other): - """Check if this range contains *other*.""" - if other is None: - return False - if isinstance(other, numbers.Number): - return self.central - self.bandwidth/2. <= other <= self.central + self.bandwidth/2. - - with suppress(AttributeError): - if self.unit != other.unit: - raise NotImplementedError("Can't compare frequency ranges with different units.") - return (self.central - self.bandwidth/2. <= other.central - other.bandwidth/2. and - self.central + self.bandwidth/2. >= other.central + other.bandwidth/2.) - return False - - def distance(self, value): - """Get the distance from value.""" - if self == value: - try: - return abs(value.central - self.central) - except AttributeError: - if isinstance(value, (tuple, list)): - return abs(value[0] - self.central) - return abs(value - self.central) - else: - return np.inf - - @classmethod - def convert(cls, frq): - """Convert `frq` to this type if possible.""" - if isinstance(frq, dict): - return cls(**frq) - return frq - - class MHS_AMSUB_AAPPL1CFile(AAPPL1BaseFileHandler): """Reader for AMSU-B/MHS L1C files created from the AAPP software.""" diff --git a/satpy/tests/test_dataset.py b/satpy/tests/test_dataset.py index 3cbc937b3b..290317e73e 100644 --- a/satpy/tests/test_dataset.py +++ b/satpy/tests/test_dataset.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- -# Copyright (c) 2015-2021 Satpy developers +# Copyright (c) 2015-2022 Satpy developers # # This file is part of satpy. # @@ -24,6 +24,7 @@ import pytest from satpy.dataset.dataid import DataID, DataQuery, ModifierTuple, WavelengthRange, minimal_default_keys_config +from satpy.readers.pmw_channels_definitions import FrequencyDoubleSideBand, FrequencyQuadrupleSideBand, FrequencyRange from satpy.tests.utils import make_cid, make_dataid, make_dsq @@ -688,10 +689,70 @@ def test_seviri_hrv_has_priority_over_vis008(self): assert res[0].name == "HRV" -def test_frequency_double_side_band_class_method_convert(): +def test_frequency_quadruple_side_band_class_method_convert(): """Test the frequency double side band object: test the class method convert.""" - from satpy.readers.aapp_mhs_amsub_l1c import FrequencyDoubleSideBand + frq_dsb = FrequencyQuadrupleSideBand(57, 0.322, 0.05, 0.036) + + res = frq_dsb.convert(57.37) + assert res == 57.37 + + res = frq_dsb.convert({'central': 57.0, 'side': 0.322, 'sideside': 0.05, 'bandwidth': 0.036}) + assert res == FrequencyQuadrupleSideBand(57, 0.322, 0.05, 0.036) + + +def test_frequency_quadruple_side_band_channel_str(): + """Test the frequency quadruple side band object: test the band description.""" + frq_dsb1 = FrequencyQuadrupleSideBand(57.0, 0.322, 0.05, 0.036) + frq_dsb2 = FrequencyQuadrupleSideBand(57000, 322, 50, 36, 'MHz') + + assert str(frq_dsb1) == "57.0 GHz (0.322_0.05_0.036 GHz)" + assert str(frq_dsb2) == "57000 MHz (322_50_36 MHz)" + + +def test_frequency_quadruple_side_band_channel_equality(): + """Test the frequency quadruple side band object: check if two bands are 'equal'.""" + frq_dsb = FrequencyQuadrupleSideBand(57, 0.322, 0.05, 0.036) + assert frq_dsb is not None + assert 57 != frq_dsb + assert 57.372 == frq_dsb + assert 56.646 == frq_dsb + assert 56.71 == frq_dsb + + assert frq_dsb != FrequencyQuadrupleSideBand(57, 0.322, 0.1, 0.040) + + frq_dsb = None + assert FrequencyQuadrupleSideBand(57, 0.322, 0.05, 0.036) != frq_dsb + assert frq_dsb < FrequencyQuadrupleSideBand(57, 0.322, 0.05, 0.04) + + +def test_frequency_quadruple_side_band_channel_distances(): + """Test the frequency quadruple side band object: get the distance between two bands.""" + frq_dsb = FrequencyQuadrupleSideBand(57, 0.322, 0.05, 0.036) + mydist = frq_dsb.distance(57.372) + assert mydist == 0.0 + + mydist = frq_dsb.distance(57) + assert mydist == np.inf + + +def test_frequency_quadruple_side_band_channel_containment(): + """Test the frequency quadruple side band object: check if one band contains another.""" + frq_dsb = FrequencyQuadrupleSideBand(57, 0.322, 0.05, 0.036) + + assert 57 not in frq_dsb + assert 57.373 in frq_dsb + with pytest.raises(NotImplementedError): + assert frq_dsb in FrequencyQuadrupleSideBand(57, 0.322, 0.05, 0.05) + + frq_dsb = None + assert (frq_dsb in FrequencyQuadrupleSideBand(57, 0.322, 0.05, 0.05)) is False + + assert '57' not in FrequencyQuadrupleSideBand(57, 0.322, 0.05, 0.05) + + +def test_frequency_double_side_band_class_method_convert(): + """Test the frequency double side band object: test the class method convert.""" frq_dsb = FrequencyDoubleSideBand(183, 7, 2) res = frq_dsb.convert(185) @@ -703,8 +764,6 @@ def test_frequency_double_side_band_class_method_convert(): def test_frequency_double_side_band_channel_str(): """Test the frequency double side band object: test the band description.""" - from satpy.readers.aapp_mhs_amsub_l1c import FrequencyDoubleSideBand - frq_dsb1 = FrequencyDoubleSideBand(183, 7, 2) frq_dsb2 = FrequencyDoubleSideBand(183000, 7000, 2000, 'MHz') @@ -714,8 +773,6 @@ def test_frequency_double_side_band_channel_str(): def test_frequency_double_side_band_channel_equality(): """Test the frequency double side band object: check if two bands are 'equal'.""" - from satpy.readers.aapp_mhs_amsub_l1c import FrequencyDoubleSideBand - frq_dsb = FrequencyDoubleSideBand(183, 7, 2) assert frq_dsb is not None assert 183 != frq_dsb @@ -735,8 +792,6 @@ def test_frequency_double_side_band_channel_equality(): def test_frequency_double_side_band_channel_distances(): """Test the frequency double side band object: get the distance between two bands.""" - from satpy.readers.aapp_mhs_amsub_l1c import FrequencyDoubleSideBand - frq_dsb = FrequencyDoubleSideBand(183, 7, 2) mydist = frq_dsb.distance(175.5) assert mydist == 0.5 @@ -762,8 +817,6 @@ def test_frequency_double_side_band_channel_distances(): def test_frequency_double_side_band_channel_containment(): """Test the frequency double side band object: check if one band contains another.""" - from satpy.readers.aapp_mhs_amsub_l1c import FrequencyDoubleSideBand - frq_dsb = FrequencyDoubleSideBand(183, 7, 2) assert 175.5 in frq_dsb @@ -781,8 +834,6 @@ def test_frequency_double_side_band_channel_containment(): def test_frequency_range_class_method_convert(): """Test the frequency range object: test the class method convert.""" - from satpy.readers.aapp_mhs_amsub_l1c import FrequencyRange - frq_dsb = FrequencyRange(89, 2) res = frq_dsb.convert(89) @@ -794,8 +845,6 @@ def test_frequency_range_class_method_convert(): def test_frequency_range_channel_equality(): """Test the frequency range object: check if two bands are 'equal'.""" - from satpy.readers.aapp_mhs_amsub_l1c import FrequencyRange - frqr = FrequencyRange(2, 1) assert frqr is not None assert 1.7 == frqr @@ -807,8 +856,6 @@ def test_frequency_range_channel_equality(): def test_frequency_range_channel_containment(): """Test the frequency range object: channel containment.""" - from satpy.readers.aapp_mhs_amsub_l1c import FrequencyRange - frqr = FrequencyRange(2, 1) assert 1.7 in frqr assert 2.8 not in frqr @@ -824,8 +871,6 @@ def test_frequency_range_channel_containment(): def test_frequency_range_channel_distances(): """Test the frequency range object: derive distances between bands.""" - from satpy.readers.aapp_mhs_amsub_l1c import FrequencyRange - frqr = FrequencyRange(190.0, 2) mydist = frqr.distance(FrequencyRange(190, 2)) @@ -840,8 +885,6 @@ def test_frequency_range_channel_distances(): def test_wavelength_range(): """Test the wavelength range object.""" - from satpy.dataset.dataid import WavelengthRange - wr = WavelengthRange(1, 2, 3) assert 1.2 == wr assert .9 != wr @@ -873,8 +916,6 @@ def test_wavelength_range(): def test_wavelength_range_cf_roundtrip(): """Test the wavelength range object roundtrip to cf.""" - from satpy.dataset.dataid import WavelengthRange - wr = WavelengthRange(1, 2, 3) assert WavelengthRange.from_cf(wr.to_cf()) == wr From 05afe37c6d2bada2671c0ba75241633771912257 Mon Sep 17 00:00:00 2001 From: "Adam.Dybbroe" Date: Tue, 14 Jun 2022 11:14:59 +0200 Subject: [PATCH 03/25] Refactor frequency handling Signed-off-by: Adam.Dybbroe --- satpy/readers/pmw_channels_definitions.py | 415 ++++++++++++++++++++++ 1 file changed, 415 insertions(+) create mode 100644 satpy/readers/pmw_channels_definitions.py diff --git a/satpy/readers/pmw_channels_definitions.py b/satpy/readers/pmw_channels_definitions.py new file mode 100644 index 0000000000..ad2a6460b7 --- /dev/null +++ b/satpy/readers/pmw_channels_definitions.py @@ -0,0 +1,415 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +# Copyright (c) 2022 Satpy Developers + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +"""Passive Microwave instrument and channel specific features.""" + +import numbers +from contextlib import suppress +from typing import NamedTuple + +import numpy as np + + +class FrequencyQuadrupleSideBandBase(NamedTuple): + """Base class for a frequency quadruple side band. + + Frequency Quadruple Side Band is supposed to describe the special type of + bands commonly used in temperature sounding from Passive Microwave + Sensors. When the absorption band being observed is symmetrical it is + advantageous (giving better NeDT) to sense in a band both right and left of + the central absorption frequency. But to avoid (CO2) absorption lines + symmetrically positioned on each side of the main absorption band it is + common to split the side bands in two 'side-side' bands. + + This is needed because of this bug: https://bugs.python.org/issue41629 + + """ + + central: float + side: float + sideside: float + bandwidth: float + unit: str = "GHz" + + +class FrequencyQuadrupleSideBand(FrequencyQuadrupleSideBandBase): + """The frequency quadruple side band class. + + The elements of the quadruple-side-band type frequency band are the + central frquency, the relative (main) side band frequency (relative to the + center - left and right), the sub-side band frequency (relative to the + offset side-band(s)) and their bandwidths. Optionally a unit (defaults to + GHz) may be specified. No clever unit conversion is done here, it's just + used for checking that two ranges are comparable. + + Frequency Quadruple Side Band is supposed to describe the special type of + bands commonly used in temperature sounding from Passive Microwave + Sensors. When the absorption band being observed is symmetrical it is + advantageous (giving better NeDT) to sense in a band both right and left of + the central absorption frequency. But to avoid (CO2) absorption lines + symmetrically positioned on each side of the main absorption band it is + common to split the side bands in two 'side-side' bands. + + """ + + def __eq__(self, other): + """Return if two channel frequencies are equal. + + Args: + other (tuple or scalar): (central frq, side band frq, side-side band frq, + and band width frq) or scalar frq + + Return: + True if other is a scalar and min <= other <= max, or if other is + a tuple equal to self, False otherwise. + + """ + if other is None: + return False + if isinstance(other, numbers.Number): + return other in self + if isinstance(other, (tuple, list)) and len(other) == 4: + return other in self + return super().__eq__(other) + + def __ne__(self, other): + """Return the opposite of `__eq__`.""" + return not self == other + + def __lt__(self, other): + """Compare to another frequency.""" + if other is None: + return False + return super().__lt__(other) + + def __gt__(self, other): + """Compare to another frequency.""" + if other is None: + return True + return super().__gt__(other) + + def __hash__(self): + """Hash this tuple.""" + return tuple.__hash__(self) + + def __str__(self): + """Format for print out.""" + return "{0.central} {0.unit} ({0.side}_{0.sideside}_{0.bandwidth} {0.unit})".format(self) + + def __contains__(self, other): + """Check if this quadruple-side-band 'contains' *other*.""" + if other is None: + return False + + # The four centrals: + central_left_left = self.central - self.side - self.sideside + central_left_right = self.central - self.side + self.sideside + central_right_left = self.central + self.side - self.sideside + central_right_right = self.central + self.side + self.sideside + + if isinstance(other, numbers.Number): + if _is_inside_interval(other, central_left_left, self.bandwidth): + return True + if _is_inside_interval(other, central_left_right, self.bandwidth): + return True + if _is_inside_interval(other, central_right_left, self.bandwidth): + return True + if _is_inside_interval(other, central_right_right, self.bandwidth): + return True + + return False + + if isinstance(other, (tuple, list)) and len(other) == 5: + raise NotImplementedError("Can't check if one frequency quadruple side band is contained in another.") + + with suppress(AttributeError): + if self.unit != other.unit: + raise NotImplementedError("Can't compare frequency ranges with different units.") + + return False + + def distance(self, value): + """Get the distance from value.""" + left_left = self.central - self.side - self.sideside + right_right = self.central + self.side + self.sideside + + if self == value: + try: + left_side_dist = abs(value.central - value.side - value.sideside - left_left) + right_side_dist = abs(value.central + value.side + value.sideside - right_right) + return min(left_side_dist, right_side_dist) + except AttributeError: + if isinstance(value, (tuple, list)): + raise NotImplementedError('Distance to a quadruple side band frequency not supported for this type') + + left_side_dist = abs(value - left_left) + right_side_dist = abs(value - right_right) + return min(left_side_dist, right_side_dist) + else: + return np.inf + + @classmethod + def convert(cls, frq): + """Convert `frq` to this type if possible.""" + if isinstance(frq, dict): + return cls(**frq) + return frq + + +class FrequencyDoubleSideBandBase(NamedTuple): + """Base class for a frequency double side band. + + Frequency Double Side Band is supposed to describe the special type of bands + commonly used in humidty sounding from Passive Microwave Sensors. When the + absorption band being observed is symmetrical it is advantageous (giving + better NeDT) to sense in a band both right and left of the central + absorption frequency. + + This is needed because of this bug: https://bugs.python.org/issue41629 + + """ + + central: float + side: float + bandwidth: float + unit: str = "GHz" + + +class FrequencyDoubleSideBand(FrequencyDoubleSideBandBase): + """The frequency double side band class. + + The elements of the double-side-band type frequency band are the central + frquency, the relative side band frequency (relative to the center - left + and right) and their bandwidths, and optionally a unit (defaults to + GHz). No clever unit conversion is done here, it's just used for checking + that two ranges are comparable. + + Frequency Double Side Band is supposed to describe the special type of bands + commonly used in humidty sounding from Passive Microwave Sensors. When the + absorption band being observed is symmetrical it is advantageous (giving + better NeDT) to sense in a band both right and left of the central + absorption frequency. + + """ + + def __eq__(self, other): + """Return if two channel frequencies are equal. + + Args: + other (tuple or scalar): (central frq, side band frq and band width frq) or scalar frq + + Return: + True if other is a scalar and min <= other <= max, or if other is + a tuple equal to self, False otherwise. + + """ + if other is None: + return False + if isinstance(other, numbers.Number): + return other in self + if isinstance(other, (tuple, list)) and len(other) == 3: + return other in self + return super().__eq__(other) + + def __ne__(self, other): + """Return the opposite of `__eq__`.""" + return not self == other + + def __lt__(self, other): + """Compare to another frequency.""" + if other is None: + return False + return super().__lt__(other) + + def __gt__(self, other): + """Compare to another frequency.""" + if other is None: + return True + return super().__gt__(other) + + def __hash__(self): + """Hash this tuple.""" + return tuple.__hash__(self) + + def __str__(self): + """Format for print out.""" + return "{0.central} {0.unit} ({0.side}_{0.bandwidth} {0.unit})".format(self) + + def __contains__(self, other): + """Check if this double-side-band 'contains' *other*.""" + if other is None: + return False + if isinstance(other, numbers.Number): + if (self.central + self.side - self.bandwidth/2. <= other + <= self.central + self.side + self.bandwidth/2.): + return True + if (self.central - self.side - self.bandwidth/2. <= other + <= self.central - self.side + self.bandwidth/2.): + return True + return False + + if isinstance(other, (tuple, list)) and len(other) == 3: + return ((self.central - self.side - self.bandwidth/2. <= + other[0] - other[1] - other[2]/2. and + self.central - self.side + self.bandwidth/2. >= + other[0] - other[1] + other[2]/2.) or + (self.central + self.side - self.bandwidth/2. <= + other[0] + other[1] - other[2]/2. and + self.central + self.side + self.bandwidth/2. >= + other[0] + other[1] + other[2]/2.)) + + with suppress(AttributeError): + if self.unit != other.unit: + raise NotImplementedError("Can't compare frequency ranges with different units.") + return ((self.central - self.side - self.bandwidth/2. <= + other.central - other.side - other.bandwidth/2. and + self.central - self.side + self.bandwidth/2. >= + other.central - other.side + other.bandwidth/2.) or + (self.central + self.side - self.bandwidth/2. <= + other.central + other.side - other.bandwidth/2. and + self.central + self.side + self.bandwidth/2. >= + other.central + other.side + other.bandwidth/2.)) + + return False + + def distance(self, value): + """Get the distance from value.""" + if self == value: + try: + left_side_dist = abs(value.central - value.side - (self.central - self.side)) + right_side_dist = abs(value.central + value.side - (self.central + self.side)) + return min(left_side_dist, right_side_dist) + except AttributeError: + if isinstance(value, (tuple, list)): + return abs((value[0] - value[1]) - (self.central - self.side)) + + left_side_dist = abs(value - (self.central - self.side)) + right_side_dist = abs(value - (self.central + self.side)) + return min(left_side_dist, right_side_dist) + else: + return np.inf + + @classmethod + def convert(cls, frq): + """Convert `frq` to this type if possible.""" + if isinstance(frq, dict): + return cls(**frq) + return frq + + +class FrequencyRangeBase(NamedTuple): + """Base class for frequency ranges. + + This is needed because of this bug: https://bugs.python.org/issue41629 + """ + + central: float + bandwidth: float + unit: str = "GHz" + + +class FrequencyRange(FrequencyRangeBase): + """The Frequency range class. + + The elements of the range are central and bandwidth values, and optionally + a unit (defaults to GHz). No clever unit conversion is done here, it's just + used for checking that two ranges are comparable. + + This type is used for passive microwave sensors. + + """ + + def __eq__(self, other): + """Return if two channel frequencies are equal. + + Args: + other (tuple or scalar): (central frq, band width frq) or scalar frq + + Return: + True if other is a scalar and min <= other <= max, or if other is + a tuple equal to self, False otherwise. + + """ + if other is None: + return False + if isinstance(other, numbers.Number): + return other in self + if isinstance(other, (tuple, list)) and len(other) == 2: + return self[:2] == other + return super().__eq__(other) + + def __ne__(self, other): + """Return the opposite of `__eq__`.""" + return not self == other + + def __lt__(self, other): + """Compare to another frequency.""" + if other is None: + return False + return super().__lt__(other) + + def __gt__(self, other): + """Compare to another frequency.""" + if other is None: + return True + return super().__gt__(other) + + def __hash__(self): + """Hash this tuple.""" + return tuple.__hash__(self) + + def __str__(self): + """Format for print out.""" + return "{0.central} {0.unit} ({0.bandwidth} {0.unit})".format(self) + + def __contains__(self, other): + """Check if this range contains *other*.""" + if other is None: + return False + if isinstance(other, numbers.Number): + return self.central - self.bandwidth/2. <= other <= self.central + self.bandwidth/2. + + with suppress(AttributeError): + if self.unit != other.unit: + raise NotImplementedError("Can't compare frequency ranges with different units.") + return (self.central - self.bandwidth/2. <= other.central - other.bandwidth/2. and + self.central + self.bandwidth/2. >= other.central + other.bandwidth/2.) + return False + + def distance(self, value): + """Get the distance from value.""" + if self == value: + try: + return abs(value.central - self.central) + except AttributeError: + if isinstance(value, (tuple, list)): + return abs(value[0] - self.central) + return abs(value - self.central) + else: + return np.inf + + @classmethod + def convert(cls, frq): + """Convert `frq` to this type if possible.""" + if isinstance(frq, dict): + return cls(**frq) + return frq + + +def _is_inside_interval(value, central, width): + return (central - width/2 <= value <= central + width/2) From da2d8438c856e379fb35dcda5769899c6e80a7cb Mon Sep 17 00:00:00 2001 From: "Adam.Dybbroe" Date: Tue, 14 Jun 2022 22:03:58 +0200 Subject: [PATCH 04/25] Fix module import Signed-off-by: Adam.Dybbroe --- satpy/tests/test_yaml_reader.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/satpy/tests/test_yaml_reader.py b/satpy/tests/test_yaml_reader.py index a6b8a2cb30..f35391ff1d 100644 --- a/satpy/tests/test_yaml_reader.py +++ b/satpy/tests/test_yaml_reader.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- -# Copyright (c) 2015-2019, 2021 Satpy developers +# Copyright (c) 2015-2022 Satpy developers # # This file is part of satpy. # @@ -30,8 +30,8 @@ import satpy.readers.yaml_reader as yr from satpy.dataset import DataQuery from satpy.dataset.dataid import ModifierTuple -from satpy.readers.aapp_mhs_amsub_l1c import FrequencyDoubleSideBand, FrequencyRange from satpy.readers.file_handlers import BaseFileHandler +from satpy.readers.pmw_channels_definitions import FrequencyDoubleSideBand, FrequencyRange from satpy.tests.utils import make_dataid MHS_YAML_READER_DICT = { From c0d4b3471118106af544b9b6f7bfdeed5a308916 Mon Sep 17 00:00:00 2001 From: "Adam.Dybbroe" Date: Thu, 23 Jun 2022 13:10:07 +0200 Subject: [PATCH 05/25] Fix resolution and add support for navigation and other aux data Signed-off-by: Adam.Dybbroe --- satpy/etc/readers/mws_l1b_nc.yaml | 100 +++++++++++++++++++------- satpy/readers/mws_l1b.py | 113 ++++++++++++++++++++++-------- 2 files changed, 158 insertions(+), 55 deletions(-) diff --git a/satpy/etc/readers/mws_l1b_nc.yaml b/satpy/etc/readers/mws_l1b_nc.yaml index 0f3aeffcbc..911eeeaab6 100644 --- a/satpy/etc/readers/mws_l1b_nc.yaml +++ b/satpy/etc/readers/mws_l1b_nc.yaml @@ -1,6 +1,8 @@ reader: name: mws_l1b_nc - description: EPS-SG l1b Reader for reading the MWS (Microwave Sounder) level-1 data + short_name: MWS L1B RAD NetCDF4 + long_name: EPS-SG MWS L1B Radiance (NetCDF4) + description: Reader for the EPS-SG l1b MWS (Microwave Sounder) level-1 files in netCDF4. reader: !!python/name:satpy.readers.yaml_reader.FileYAMLReader sensors: [mws,] default_channels: [] @@ -36,7 +38,7 @@ datasets: bandwidth: 0.270 unit: GHz polarization: 'QH' - resolution: 40000 + resolution: 17000 calibration: brightness_temperature: standard_name: toa_brightness_temperature @@ -51,7 +53,7 @@ datasets: bandwidth: 0.180 unit: GHz polarization: 'QH' - resolution: 40000 + resolution: 17000 calibration: brightness_temperature: standard_name: toa_brightness_temperature @@ -66,7 +68,7 @@ datasets: bandwidth: 0.180 unit: GHz polarization: 'QH' - resolution: 20000 + resolution: 17000 calibration: brightness_temperature: standard_name: toa_brightness_temperature @@ -81,7 +83,7 @@ datasets: bandwidth: 0.400 unit: GHz polarization: 'QH' - resolution: 20000 + resolution: 17000 calibration: brightness_temperature: standard_name: toa_brightness_temperature @@ -97,7 +99,7 @@ datasets: bandwidth: 0.140 unit: GHz polarization: 'QH' - resolution: 20000 + resolution: 17000 calibration: brightness_temperature: standard_name: toa_brightness_temperature @@ -113,7 +115,7 @@ datasets: bandwidth: 0.170 unit: GHz polarization: 'QH' - resolution: 20000 + resolution: 17000 calibration: brightness_temperature: standard_name: toa_brightness_temperature @@ -129,7 +131,7 @@ datasets: bandwidth: 0.142 unit: GHz polarization: 'QH' - resolution: 20000 + resolution: 17000 calibration: brightness_temperature: standard_name: toa_brightness_temperature @@ -144,7 +146,7 @@ datasets: bandwidth: 0.400 unit: GHz polarization: 'QH' - resolution: 20000 + resolution: 17000 calibration: brightness_temperature: standard_name: toa_brightness_temperature @@ -159,7 +161,7 @@ datasets: bandwidth: 0.400 unit: GHz polarization: 'QH' - resolution: 20000 + resolution: 17000 calibration: brightness_temperature: standard_name: toa_brightness_temperature @@ -174,7 +176,7 @@ datasets: bandwidth: 0.330 unit: GHz polarization: 'QH' - resolution: 20000 + resolution: 17000 calibration: brightness_temperature: standard_name: toa_brightness_temperature @@ -189,7 +191,7 @@ datasets: bandwidth: 0.330 unit: GHz polarization: 'QH' - resolution: 20000 + resolution: 17000 calibration: brightness_temperature: standard_name: toa_brightness_temperature @@ -206,7 +208,7 @@ datasets: bandwidth: 0.078 unit: GHz polarization: 'QH' - resolution: 20000 + resolution: 17000 calibration: brightness_temperature: standard_name: toa_brightness_temperature @@ -224,7 +226,7 @@ datasets: bandwidth: 0.036 unit: GHz polarization: 'QH' - resolution: 20000 + resolution: 17000 calibration: brightness_temperature: standard_name: toa_brightness_temperature @@ -242,7 +244,7 @@ datasets: bandwidth: 0.016 unit: GHz polarization: 'QH' - resolution: 20000 + resolution: 17000 calibration: brightness_temperature: standard_name: toa_brightness_temperature @@ -260,7 +262,7 @@ datasets: bandwidth: 0.008 unit: GHz polarization: 'QH' - resolution: 20000 + resolution: 17000 calibration: brightness_temperature: standard_name: toa_brightness_temperature @@ -278,7 +280,7 @@ datasets: bandwidth: 0.004 unit: GHz polarization: 'QH' - resolution: 20000 + resolution: 17000 calibration: brightness_temperature: standard_name: toa_brightness_temperature @@ -325,7 +327,7 @@ datasets: bandwidth: 2.0 unit: GHz polarization: 'QV' - resolution: 20000 + resolution: 17000 calibration: brightness_temperature: standard_name: toa_brightness_temperature @@ -341,7 +343,7 @@ datasets: bandwidth: 2.0 unit: GHz polarization: 'QV' - resolution: 20000 + resolution: 17000 calibration: brightness_temperature: standard_name: toa_brightness_temperature @@ -357,7 +359,7 @@ datasets: bandwidth: 1.0 unit: GHz polarization: 'QV' - resolution: 20000 + resolution: 17000 calibration: brightness_temperature: standard_name: toa_brightness_temperature @@ -373,7 +375,7 @@ datasets: bandwidth: 1.0 unit: GHz polarization: 'QV' - resolution: 20000 + resolution: 17000 calibration: brightness_temperature: standard_name: toa_brightness_temperature @@ -389,7 +391,7 @@ datasets: bandwidth: 0.5 unit: GHz polarization: 'QV' - resolution: 20000 + resolution: 17000 calibration: brightness_temperature: standard_name: toa_brightness_temperature @@ -413,21 +415,71 @@ datasets: - mws_lat file_type: mws_l1b_nc +# --- Coordinates --- mws_lat: name: mws_lat - resolution: 40000 + resolution: 17000 file_type: mws_l1b_nc standard_name: latitude units: degrees_north mws_lon: name: mws_lon - resolution: 40000 + resolution: 17000 file_type: mws_l1b_nc standard_name: longitude units: degrees_east +# --- Navigation data --- + + solar_azimuth: + name: solar_azimuth + standard_name: solar_azimuth_angle + file_type: mws_l1b_nc + #file_key: data/navigation/mws_solar_azimuth_angle + coordinates: + - mws_lon + - mws_lat + solar_zenith: + name: solar_zenith + standard_name: solar_zenith_angle + file_type: mws_l1b_nc + coordinates: + - mws_lon + - mws_lat + satellite_azimuth: + name: satellite_azimuth + standard_name: satellite_azimuth_angle + file_type: mws_l1b_nc + coordinates: + - mws_lon + - mws_lat + satellite_zenith: + name: satellite_zenith + standard_name: satellite_zenith_angle + file_type: mws_l1b_nc + coordinates: + - mws_lon + - mws_lat + +# --- Land surface data --- + surface_type: + name: surface_type + standard_name: surface_type + file_type: mws_l1b_nc + coordinates: + - mws_lon + - mws_lat + terrain_elevation: + name: terrain_elevation + standard_name: terrain_elevation + file_type: mws_l1b_nc + coordinates: + - mws_lon + - mws_lat + + file_types: mws_l1b_nc: # EPS-SG_MWS-1B-RAD.nc diff --git a/satpy/readers/mws_l1b.py b/satpy/readers/mws_l1b.py index 772c7d74d1..ca04396eca 100644 --- a/satpy/readers/mws_l1b.py +++ b/satpy/readers/mws_l1b.py @@ -25,7 +25,9 @@ """ import logging +from datetime import datetime +import dask.array as da import numpy as np from netCDF4 import default_fillvals @@ -38,24 +40,26 @@ # parameter name and values are the paths to the variable inside the netcdf AUX_DATA = { - 'subsatellite_latitude': 'state/platform/subsatellite_latitude', - 'subsatellite_longitude': 'state/platform/subsatellite_longitude', - 'platform_altitude': 'state/platform/platform_altitude', - 'subsolar_latitude': 'state/celestial/subsolar_latitude', - 'subsolar_longitude': 'state/celestial/subsolar_longitude', - 'earth_sun_distance': 'state/celestial/earth_sun_distance', - 'sun_satellite_distance': 'state/celestial/sun_satellite_distance', - 'time': 'time', - 'swath_number': 'data/swath_number', - 'swath_direction': 'data/swath_direction', + 'scantime_utc': 'data/navigation/mws_scantime_utc', + 'solar_azimuth': 'data/navigation/mws_solar_azimuth_angle', + 'solar_zenith': 'data/navigation/mws_solar_zenith_angle', + 'satellite_azimuth': 'data/navigation/mws_satellite_azimuth_angle', + 'satellite_zenith': 'data/navigation/mws_satellite_zenith_angle', + 'surface_type': 'data/navigation/mws_surface_type', + 'terrain_elevation': 'data/navigation/mws_terrain_elevation', 'mws_lat': 'data/navigation/mws_lat', 'mws_lon': 'data/navigation/mws_lon', } MWS_CHANNEL_NAMES_TO_NUMBER = {'1': 1, '2': 2, '3': 3, '4': 4, - '5': 5, '6': 6, '7': 7, '8': 8} + '5': 5, '6': 6, '7': 7, '8': 8, + '9': 9, '10': 10, '11': 11, '12': 12, + '13': 13, '14': 14, '15': 15, '16': 16, + '17': 17, '18': 18, '19': 19, '20': 20, + '21': 21, '22': 22, '23': 23, '24': 24} -MWS_CHANNEL_NAMES = ['1', '2', '3', '4', '5', '6'] +MWS_CHANNEL_NAMES = list(MWS_CHANNEL_NAMES_TO_NUMBER.keys()) +MWS_CHANNELS = set(MWS_CHANNEL_NAMES_TO_NUMBER.keys()) def get_channel_index_from_name(chname): @@ -84,9 +88,6 @@ class MWSL1BFile(NetCDF4FileHandler): """ - # FIXME! - # - # After launch: translate to Metop-X instead? _platform_name_translate = { "SGA1": "Metop-SG-A1", "SGA2": "Metop-SG-A2", @@ -109,24 +110,56 @@ def __init__(self, filename, filename_info, filetype_info): @property def start_time(self): """Get start time.""" - return self.filename_info['start_time'] + return datetime.strptime(self['/attr/sensing_start_time_utc'], + '%Y-%m-%d %H:%M:%S.%f') @property def end_time(self): """Get end time.""" - return self.filename_info['end_time'] + return datetime.strptime(self['/attr/sensing_end_time_utc'], + '%Y-%m-%d %H:%M:%S.%f') - def get_dataset(self, key, info=None): + @property + def sensor(self): + """Get the sensor name.""" + return self['/attr/instrument'] + + @property + def platform_name(self): + """Get the platform name.""" + return self._platform_name_translate.get(self['/attr/spacecraft']) + + @property + def sub_satellite_longitude_start(self): + """Get the longitude of sub-satellite point at start of the product.""" + return self['status/satellite/subsat_longitude_start'].data.item() + + @property + def sub_satellite_latitude_start(self): + """Get the latitude of sub-satellite point at start of the product.""" + return self['status/satellite/subsat_latitude_start'].data.item() + + @property + def sub_satellite_longitude_end(self): + """Get the longitude of sub-satellite point at end of the product.""" + return self['status/satellite/subsat_longitude_end'].data.item() + + @property + def sub_satellite_latitude_end(self): + """Get the latitude of sub-satellite point at end of the product.""" + return self['status/satellite/subsat_latitude_end'].data.item() + + def get_dataset(self, dataset_id, info=None): """Load a dataset.""" - logger.debug('Reading {} from {}'.format(key['name'], self.filename)) + logger.debug('Reading {} from {}'.format(dataset_id['name'], self.filename)) - if _get_aux_data_name_from_dsname(key['name']) is not None: - return self._get_dataset_aux_data(key['name'], info=info) - elif any(lb in key['name'] for lb in {"1", "2", "3", "4"}): - return self._get_dataset_channel(key, info=info) + if _get_aux_data_name_from_dsname(dataset_id['name']) is not None: + return self._get_dataset_aux_data(dataset_id['name'], info=info) + elif any(lb in dataset_id['name'] for lb in MWS_CHANNELS): + return self._get_dataset_channel(dataset_id, info=info) else: raise ValueError("Unknown dataset key, not a channel, quality or auxiliary data: " - f"{key['name']:s}") + f"{dataset_id['name']:s}") def _standardize_dims(self, variable): """Standardize dims to y, x.""" @@ -166,17 +199,32 @@ def _get_dataset_channel(self, key, info=None): # Manage the attributes of the dataset data.attrs.setdefault('units', None) data.attrs.update(info) - # variable.attrs.update(self._get_global_attributes()) # FIXME! See VII reader + + i = getattr(data, 'attrs', {}) + i.update(info) + i.update({ + "platform_name": self.platform_name, + "sensor": self.sensor, + "orbital_parameters": {'sub_satellite_latitude_start': self.sub_satellite_latitude_start, + 'sub_satellite_longitude_start': self.sub_satellite_longitude_start, + 'sub_satellite_latitude_end': self.sub_satellite_latitude_end, + 'sub_satellite_longitude_end': self.sub_satellite_longitude_end}, + }) + i.update(key.to_dict()) + data.attrs.update(i) return data def _get_dataset_aux_data(self, dsname, info=None): """Get the auxiliary data arrays using the index map.""" - # Geolocation: - if dsname in ['mws_lat', 'mws_lon']: + # Geolocation and navigation data: + if dsname in ['mws_lat', 'mws_lon', + 'solar_azimuth', 'solar_zenith', + 'satellite_azimuth', 'satellite_zenith', + 'surface_type', 'terrain_elevation']: var_key = AUX_DATA.get(dsname) else: - raise NotImplementedError("Only lons and lats supported - no other auxillary data yet...") + raise NotImplementedError("Dataset %s not supported..." % dsname) try: variable = self[var_key] @@ -185,10 +233,13 @@ def _get_dataset_aux_data(self, dsname, info=None): return None # Scale the data: - variable.data = variable.data * variable.attrs['scale_factor'] + variable.attrs['add_offset'] + missing_value = variable.attrs['missing_value'] + if 'scale_factor' in variable.attrs and 'add_offset' in variable.attrs: + variable.data = da.where(variable.data == missing_value, np.nan, + variable.data * variable.attrs['scale_factor'] + variable.attrs['add_offset']) # Manage the attributes of the dataset variable.attrs.setdefault('units', None) - variable.attrs.update(info) - # variable.attrs.update(self._get_global_attributes()) # FIXME! See VII reader + if info: + variable.attrs.update(info) return variable From f1b4cf1ece9b2d2a6997b12bbb47ecefc52af1d0 Mon Sep 17 00:00:00 2001 From: "Adam.Dybbroe" Date: Thu, 23 Jun 2022 13:10:27 +0200 Subject: [PATCH 06/25] Update list of supported readers Signed-off-by: Adam.Dybbroe --- doc/source/index.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/doc/source/index.rst b/doc/source/index.rst index 5d7da435c2..d7b19561ca 100644 --- a/doc/source/index.rst +++ b/doc/source/index.rst @@ -292,6 +292,9 @@ the base Satpy installation. * - AAPP L1C AMSU-B format - `aapp_amsub_l1c` - Beta + * - EPS-SG MWS level-1c netCDF format + - `mws_l1b_nc` + - Beta * - Arctica-M (N1) MSU-GS/A data in HDF5 format - `msu_gsa_l1b` - Beta From af49fc834a61f99b49e2337a0ef4326fd9d8fe8a Mon Sep 17 00:00:00 2001 From: "Adam.Dybbroe" Date: Tue, 23 Aug 2022 15:21:43 +0200 Subject: [PATCH 07/25] Adding tests for the reader Signed-off-by: Adam.Dybbroe --- satpy/readers/mws_l1b.py | 6 +- satpy/tests/reader_tests/test_mws_l1b_nc.py | 188 ++++++++++++++++++++ 2 files changed, 189 insertions(+), 5 deletions(-) create mode 100644 satpy/tests/reader_tests/test_mws_l1b_nc.py diff --git a/satpy/readers/mws_l1b.py b/satpy/readers/mws_l1b.py index ca04396eca..f7387ecffb 100644 --- a/satpy/readers/mws_l1b.py +++ b/satpy/readers/mws_l1b.py @@ -1,11 +1,7 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- -# Copyright (c) 2022 Adam.Dybbroe - -# Author(s): - -# Adam.Dybbroe +# Copyright (c) 2022 Pytroll Developers # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by diff --git a/satpy/tests/reader_tests/test_mws_l1b_nc.py b/satpy/tests/reader_tests/test_mws_l1b_nc.py new file mode 100644 index 0000000000..4841c02fdc --- /dev/null +++ b/satpy/tests/reader_tests/test_mws_l1b_nc.py @@ -0,0 +1,188 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +# Copyright (c) 2022 Pytroll Developers + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +"""The mws_l1b_nc reader tests. + +This module tests the reading of the MWS l1b netCDF format data as per version v4B issued 22 November 2021. + +""" + +from datetime import datetime + +import numpy as np +import pytest +from netCDF4 import Dataset + +from satpy.readers.mws_l1b import MWSL1BFile + +N_CHANNELS = 24 +N_CHANNELS_OS = 2 +N_SCANS = 2637 +N_FOVS = 95 +N_FOVS_CAL = 5 +N_PRTS = 6 + + +@pytest.fixture +def reader(fake_file): + """Return reader of mws level-1b data.""" + return MWSL1BFile( + filename=fake_file, + filename_info={ + 'sensing_start_time': ( + datetime.fromisoformat('2000-01-01T01:00:00') + ), + 'sensing_end_time': ( + datetime.fromisoformat('2000-01-01T02:00:00') + ), + 'creation_time': ( + datetime.fromisoformat('2000-01-01T03:00:00') + ), + }, + filetype_info={ + 'longitude': 'data/navigation_data/mws_lon', + 'latitude': 'data/navigation_data/mws_lat', + 'solar_azimuth': 'data/navigation/mws_solar_azimuth_angle', + 'solar_zenith': 'data/navigation/mws_solar_zenith_angle', + } + ) + + +@pytest.fixture +def fake_file(tmp_path): + """Return file path to level-1b file.""" + file_path = tmp_path / 'test_file_mws_l1b.nc' + writer = MWSL1BFakeFileWriter(file_path) + writer.write() + yield file_path + + +class MWSL1BFakeFileWriter: + """Writer class of fake mws level-1b data.""" + + def __init__(self, file_path): + """Init.""" + self.file_path = file_path + + def write(self): + """Write fake data to file.""" + with Dataset(self.file_path, 'w') as dataset: + self._write_attributes(dataset) + self._write_status_group(dataset) + data_group = dataset.createGroup('data') + self._write_navigation_data_group(data_group) + + @staticmethod + def _write_attributes(dataset): + """Write attributes.""" + dataset.sensing_start_time_utc = "2000-01-02 03:04:05.000" + dataset.sensing_end_time_utc = "2000-01-02 04:05:06.000" + dataset.instrument = "MWS" + dataset.spacecraft = "SGA1" + + @staticmethod + def _write_status_group(dataset): + """Write the status group.""" + group = dataset.createGroup('/status/satellite') + subsat_latitude_start = group.createVariable( + 'subsat_latitude_start', "f4" + ) + subsat_latitude_start[:] = 52.19 + + subsat_longitude_start = group.createVariable( + 'subsat_longitude_start', "f4" + ) + subsat_longitude_start[:] = 23.26 + + subsat_latitude_end = group.createVariable( + 'subsat_latitude_end', "f4" + ) + subsat_latitude_end[:] = 60.00 + + subsat_longitude_end = group.createVariable( + 'subsat_longitude_end', "f4" + ) + subsat_longitude_end[:] = 2.47 + + @staticmethod + def _write_navigation_data_group(dataset): + """Write the navigation data group.""" + group = dataset.createGroup('navigation') + group.createDimension('n_schannels', N_CHANNELS) + group.createDimension('n_schannels_os', N_CHANNELS_OS) + group.createDimension('n_scans', N_SCANS) + group.createDimension('n_fovs', N_FOVS) + group.createDimension('n_prts', N_PRTS) + group.createDimension('n_fovs_cal', N_FOVS_CAL) + + dimensions = ('n_scans', 'n_fovs') + shape = (N_SCANS, N_FOVS) + longitude = group.createVariable( + 'mws_lon', + np.float32, + dimensions=dimensions, + ) + longitude[:] = np.ones(shape) + latitude = group.createVariable( + 'mws_lat', + np.float32, + dimensions=dimensions, + ) + latitude[:] = 2. * np.ones(shape) + azimuth = group.createVariable( + 'mws_solar_azimuth_angle', + np.float32, + dimensions=dimensions, + ) + azimuth[:] = 3. * np.ones(shape) + + +class TestMwsL1bNCFileHandler: + """Test the MWSL1BFile reader.""" + + def test_start_time(self, reader): + """Test acquiring the start time.""" + assert reader.start_time == datetime(2000, 1, 2, 3, 4, 5) + + def test_end_time(self, reader): + """Test acquiring the end time.""" + assert reader.end_time == datetime(2000, 1, 2, 4, 5, 6) + + def test_sensor(self, reader): + """Test sensor.""" + assert reader.sensor == "MWS" + + def test_platform_name(self, reader): + """Test getting the platform name.""" + assert reader.platform_name == "Metop-SG-A1" + + def test_sub_satellite_longitude_start(self, reader): + """Test getting the longitude of sub-satellite point at start of the product.""" + np.testing.assert_allclose(reader.sub_satellite_longitude_start, 23.26) + + def test_sub_satellite_latitude_start(self, reader): + """Test getting the latitude of sub-satellite point at start of the product.""" + np.testing.assert_allclose(reader.sub_satellite_latitude_start, 52.19) + + def test_sub_satellite_longitude_end(self, reader): + """Test getting the longitude of sub-satellite point at end of the product.""" + np.testing.assert_allclose(reader.sub_satellite_longitude_end, 2.47) + + def test_sub_satellite_latitude_end(self, reader): + """Test getting the latitude of sub-satellite point at end of the product.""" + np.testing.assert_allclose(reader.sub_satellite_latitude_end, 60.0) From 27c500817bb26c81f0a23c9d0e0c3d46f02bf263 Mon Sep 17 00:00:00 2001 From: "Adam.Dybbroe" Date: Tue, 23 Aug 2022 15:23:38 +0200 Subject: [PATCH 08/25] Remove line never used - module not meant to be run as script Signed-off-by: Adam.Dybbroe --- satpy/readers/mws_l1b.py | 1 - 1 file changed, 1 deletion(-) diff --git a/satpy/readers/mws_l1b.py b/satpy/readers/mws_l1b.py index f7387ecffb..8a6b3b3675 100644 --- a/satpy/readers/mws_l1b.py +++ b/satpy/readers/mws_l1b.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python # -*- coding: utf-8 -*- # Copyright (c) 2022 Pytroll Developers From 4c0a81fd345ced512f8adda54a57fe2b7dd152ef Mon Sep 17 00:00:00 2001 From: "Adam.Dybbroe" Date: Tue, 23 Aug 2022 17:00:12 +0200 Subject: [PATCH 09/25] Improve and bugfix documentation, and follow feedback from reviewer Signed-off-by: Adam.Dybbroe --- satpy/readers/mws_l1b.py | 6 +--- satpy/readers/pmw_channels_definitions.py | 35 +++++++++++++++------ satpy/tests/reader_tests/test_mws_l1b_nc.py | 3 -- 3 files changed, 27 insertions(+), 17 deletions(-) diff --git a/satpy/readers/mws_l1b.py b/satpy/readers/mws_l1b.py index 8a6b3b3675..0763e66d4f 100644 --- a/satpy/readers/mws_l1b.py +++ b/satpy/readers/mws_l1b.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - # Copyright (c) 2022 Pytroll Developers # This program is free software: you can redistribute it and/or modify @@ -62,7 +60,7 @@ def get_channel_index_from_name(chname): chindex = MWS_CHANNEL_NAMES_TO_NUMBER.get(chname, 0) - 1 if 0 <= chindex < 24: return chindex - raise AttributeError("Channel name %s not supported: " % chname) + raise AttributeError("Channel name {ch} not supported: ".format(ch=chname)) def _get_aux_data_name_from_dsname(dsname): @@ -70,8 +68,6 @@ def _get_aux_data_name_from_dsname(dsname): if len(aux_data_name) > 0: return aux_data_name[0] - return None - class MWSL1BFile(NetCDF4FileHandler): """Class implementing the EPS-SG-A1 MWS L1b Filehandler. diff --git a/satpy/readers/pmw_channels_definitions.py b/satpy/readers/pmw_channels_definitions.py index ad2a6460b7..576ba4f8d0 100644 --- a/satpy/readers/pmw_channels_definitions.py +++ b/satpy/readers/pmw_channels_definitions.py @@ -75,8 +75,9 @@ def __eq__(self, other): and band width frq) or scalar frq Return: - True if other is a scalar and min <= other <= max, or if other is - a tuple equal to self, False otherwise. + True if other is a scalar and min <= other <= max, or if other is a + tuple equal to self, or if other is a number contained by self. + False otherwise. """ if other is None: @@ -144,7 +145,20 @@ def __contains__(self, other): return False def distance(self, value): - """Get the distance from value.""" + """Get the distance to the quadruple side band. + + Determining the distance in frequency space between two quadruple side + bands can be quite ambiguous, as such bands are in effect a set of + 4 narrow bands. To keep it as simple as possible we have until further + decided to define the distance between such two bands by calculating + the distances between the outermost left-sides of the two bands and the + distances of the outermost right sides and then minimising those two. + + If the frequency entered is a single value the distance will be the + minimum of the distances to the two outermost sides of the quadruple + side band. + + """ left_left = self.central - self.side - self.sideside right_right = self.central + self.side + self.sideside @@ -155,7 +169,8 @@ def distance(self, value): return min(left_side_dist, right_side_dist) except AttributeError: if isinstance(value, (tuple, list)): - raise NotImplementedError('Distance to a quadruple side band frequency not supported for this type') + msg = 'Distance to a quadruple side band frequency not supported for this type' + raise NotImplementedError(msg) left_side_dist = abs(value - left_left) right_side_dist = abs(value - right_right) @@ -214,8 +229,9 @@ def __eq__(self, other): other (tuple or scalar): (central frq, side band frq and band width frq) or scalar frq Return: - True if other is a scalar and min <= other <= max, or if other is - a tuple equal to self, False otherwise. + True if other is a scalar and min <= other <= max, or if other is a + tuple equal to self, or if other is a number contained by self. + False otherwise. """ if other is None: @@ -335,14 +351,15 @@ class FrequencyRange(FrequencyRangeBase): """ def __eq__(self, other): - """Return if two channel frequencies are equal. + """Check wether two channel frequencies are equal. Args: other (tuple or scalar): (central frq, band width frq) or scalar frq Return: - True if other is a scalar and min <= other <= max, or if other is - a tuple equal to self, False otherwise. + True if other is a scalar and min <= other <= max, or if other is a + tuple equal to self, or if other is a number contained by self. + False otherwise. """ if other is None: diff --git a/satpy/tests/reader_tests/test_mws_l1b_nc.py b/satpy/tests/reader_tests/test_mws_l1b_nc.py index 4841c02fdc..3d75f8ac16 100644 --- a/satpy/tests/reader_tests/test_mws_l1b_nc.py +++ b/satpy/tests/reader_tests/test_mws_l1b_nc.py @@ -1,6 +1,3 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - # Copyright (c) 2022 Pytroll Developers # This program is free software: you can redistribute it and/or modify From 123c8fbffd9bf027429d7b9b7be80b6b407f7c96 Mon Sep 17 00:00:00 2001 From: "Adam.Dybbroe" Date: Wed, 24 Aug 2022 14:37:26 +0200 Subject: [PATCH 10/25] Refactor, add tests and change string representations of bands Signed-off-by: Adam.Dybbroe --- satpy/readers/mws_l1b.py | 2 +- satpy/readers/pmw_channels_definitions.py | 18 ++-------- satpy/tests/reader_tests/test_mws_l1b_nc.py | 18 +++++++++- satpy/tests/test_dataset.py | 37 +++++++++++++-------- 4 files changed, 44 insertions(+), 31 deletions(-) diff --git a/satpy/readers/mws_l1b.py b/satpy/readers/mws_l1b.py index 0763e66d4f..0207c404dd 100644 --- a/satpy/readers/mws_l1b.py +++ b/satpy/readers/mws_l1b.py @@ -60,7 +60,7 @@ def get_channel_index_from_name(chname): chindex = MWS_CHANNEL_NAMES_TO_NUMBER.get(chname, 0) - 1 if 0 <= chindex < 24: return chindex - raise AttributeError("Channel name {ch} not supported: ".format(ch=chname)) + raise AttributeError(f"Channel name '{chname}' not supported") def _get_aux_data_name_from_dsname(dsname): diff --git a/satpy/readers/pmw_channels_definitions.py b/satpy/readers/pmw_channels_definitions.py index 576ba4f8d0..70c29d48ae 100644 --- a/satpy/readers/pmw_channels_definitions.py +++ b/satpy/readers/pmw_channels_definitions.py @@ -88,10 +88,6 @@ def __eq__(self, other): return other in self return super().__eq__(other) - def __ne__(self, other): - """Return the opposite of `__eq__`.""" - return not self == other - def __lt__(self, other): """Compare to another frequency.""" if other is None: @@ -110,7 +106,7 @@ def __hash__(self): def __str__(self): """Format for print out.""" - return "{0.central} {0.unit} ({0.side}_{0.sideside}_{0.bandwidth} {0.unit})".format(self) + return f"central={self.central} {self.unit} ±{self.side} ±{self.sideside} width={self.bandwidth} {self.unit}" def __contains__(self, other): """Check if this quadruple-side-band 'contains' *other*.""" @@ -242,10 +238,6 @@ def __eq__(self, other): return other in self return super().__eq__(other) - def __ne__(self, other): - """Return the opposite of `__eq__`.""" - return not self == other - def __lt__(self, other): """Compare to another frequency.""" if other is None: @@ -264,7 +256,7 @@ def __hash__(self): def __str__(self): """Format for print out.""" - return "{0.central} {0.unit} ({0.side}_{0.bandwidth} {0.unit})".format(self) + return f"central={self.central} {self.unit} ±{self.side} width={self.bandwidth} {self.unit}" def __contains__(self, other): """Check if this double-side-band 'contains' *other*.""" @@ -370,10 +362,6 @@ def __eq__(self, other): return self[:2] == other return super().__eq__(other) - def __ne__(self, other): - """Return the opposite of `__eq__`.""" - return not self == other - def __lt__(self, other): """Compare to another frequency.""" if other is None: @@ -392,7 +380,7 @@ def __hash__(self): def __str__(self): """Format for print out.""" - return "{0.central} {0.unit} ({0.bandwidth} {0.unit})".format(self) + return f"central={self.central} {self.unit} width={self.bandwidth} {self.unit}" def __contains__(self, other): """Check if this range contains *other*.""" diff --git a/satpy/tests/reader_tests/test_mws_l1b_nc.py b/satpy/tests/reader_tests/test_mws_l1b_nc.py index 3d75f8ac16..362352b376 100644 --- a/satpy/tests/reader_tests/test_mws_l1b_nc.py +++ b/satpy/tests/reader_tests/test_mws_l1b_nc.py @@ -25,7 +25,7 @@ import pytest from netCDF4 import Dataset -from satpy.readers.mws_l1b import MWSL1BFile +from satpy.readers.mws_l1b import MWSL1BFile, get_channel_index_from_name N_CHANNELS = 24 N_CHANNELS_OS = 2 @@ -183,3 +183,19 @@ def test_sub_satellite_longitude_end(self, reader): def test_sub_satellite_latitude_end(self, reader): """Test getting the latitude of sub-satellite point at end of the product.""" np.testing.assert_allclose(reader.sub_satellite_latitude_end, 60.0) + + +@pytest.mark.parametrize("name, index", [('1', 0), ('2', 1), ('24', 23)]) +def test_get_channel_index_from_name(name, index): + """Test getting the MWS channel index from the channel name.""" + ch_idx = get_channel_index_from_name(name) + assert ch_idx == index + + +def test_get_channel_index_from_name_throw_exception(): + """Test that an excpetion is thrown when getting the MWS channel index from an unsupported name.""" + with pytest.raises(Exception) as excinfo: + _ = get_channel_index_from_name('channel 1') + + assert str(excinfo.value) == "Channel name 'channel 1' not supported" + assert excinfo.type == AttributeError diff --git a/satpy/tests/test_dataset.py b/satpy/tests/test_dataset.py index 290317e73e..d6f3ac83ac 100644 --- a/satpy/tests/test_dataset.py +++ b/satpy/tests/test_dataset.py @@ -705,8 +705,8 @@ def test_frequency_quadruple_side_band_channel_str(): frq_dsb1 = FrequencyQuadrupleSideBand(57.0, 0.322, 0.05, 0.036) frq_dsb2 = FrequencyQuadrupleSideBand(57000, 322, 50, 36, 'MHz') - assert str(frq_dsb1) == "57.0 GHz (0.322_0.05_0.036 GHz)" - assert str(frq_dsb2) == "57000 MHz (322_50_36 MHz)" + assert str(frq_dsb1) == "central=57.0 GHz ±0.322 ±0.05 width=0.036 GHz" + assert str(frq_dsb2) == "central=57000 MHz ±322 ±50 width=36 MHz" def test_frequency_quadruple_side_band_channel_equality(): @@ -767,8 +767,8 @@ def test_frequency_double_side_band_channel_str(): frq_dsb1 = FrequencyDoubleSideBand(183, 7, 2) frq_dsb2 = FrequencyDoubleSideBand(183000, 7000, 2000, 'MHz') - assert str(frq_dsb1) == "183 GHz (7_2 GHz)" - assert str(frq_dsb2) == "183000 MHz (7000_2000 MHz)" + assert str(frq_dsb1) == "central=183 GHz ±7 width=2 GHz" + assert str(frq_dsb2) == "central=183000 MHz ±7000 width=2000 MHz" def test_frequency_double_side_band_channel_equality(): @@ -817,32 +817,41 @@ def test_frequency_double_side_band_channel_distances(): def test_frequency_double_side_band_channel_containment(): """Test the frequency double side band object: check if one band contains another.""" - frq_dsb = FrequencyDoubleSideBand(183, 7, 2) + frq_range = FrequencyDoubleSideBand(183, 7, 2) - assert 175.5 in frq_dsb - assert frq_dsb in FrequencyDoubleSideBand(183, 6.5, 3) - assert frq_dsb not in FrequencyDoubleSideBand(183, 4, 2) + assert 175.5 in frq_range + assert frq_range in FrequencyDoubleSideBand(183, 6.5, 3) + assert frq_range not in FrequencyDoubleSideBand(183, 4, 2) with pytest.raises(NotImplementedError): - assert frq_dsb in FrequencyDoubleSideBand(183, 6.5, 3, 'MHz') + assert frq_range in FrequencyDoubleSideBand(183, 6.5, 3, 'MHz') - frq_dsb = None - assert (frq_dsb in FrequencyDoubleSideBand(183, 3, 2)) is False + frq_range = None + assert (frq_range in FrequencyDoubleSideBand(183, 3, 2)) is False assert '183' not in FrequencyDoubleSideBand(183, 3, 2) def test_frequency_range_class_method_convert(): """Test the frequency range object: test the class method convert.""" - frq_dsb = FrequencyRange(89, 2) + frq_range = FrequencyRange(89, 2) - res = frq_dsb.convert(89) + res = frq_range.convert(89) assert res == 89 - res = frq_dsb.convert({'central': 89, 'bandwidth': 2}) + res = frq_range.convert({'central': 89, 'bandwidth': 2}) assert res == FrequencyRange(89, 2) +def test_frequency_range_class_method_str(): + """Test the frequency range object: test the band description.""" + frq_range1 = FrequencyRange(89, 2) + frq_range2 = FrequencyRange(89000, 2000, 'MHz') + + assert str(frq_range1) == "central=89 GHz width=2 GHz" + assert str(frq_range2) == "central=89000 MHz width=2000 MHz" + + def test_frequency_range_channel_equality(): """Test the frequency range object: check if two bands are 'equal'.""" frqr = FrequencyRange(2, 1) From 6b447da967748eec5dbf4e87417d18931fc9e7d2 Mon Sep 17 00:00:00 2001 From: "Adam.Dybbroe" Date: Wed, 24 Aug 2022 16:43:37 +0200 Subject: [PATCH 11/25] Improve test coverage and enhance attribute handling Signed-off-by: Adam.Dybbroe --- satpy/readers/mws_l1b.py | 69 ++++++++++++++++++--- satpy/tests/reader_tests/test_mws_l1b_nc.py | 67 ++++++++++++++++++++ 2 files changed, 126 insertions(+), 10 deletions(-) diff --git a/satpy/readers/mws_l1b.py b/satpy/readers/mws_l1b.py index 0207c404dd..1d44249a59 100644 --- a/satpy/readers/mws_l1b.py +++ b/satpy/readers/mws_l1b.py @@ -145,14 +145,20 @@ def get_dataset(self, dataset_id, info=None): logger.debug('Reading {} from {}'.format(dataset_id['name'], self.filename)) if _get_aux_data_name_from_dsname(dataset_id['name']) is not None: - return self._get_dataset_aux_data(dataset_id['name'], info=info) + variable = self._get_dataset_aux_data(dataset_id['name'], info=info) elif any(lb in dataset_id['name'] for lb in MWS_CHANNELS): - return self._get_dataset_channel(dataset_id, info=info) + variable = self._get_dataset_channel(dataset_id, info=info) else: raise ValueError("Unknown dataset key, not a channel, quality or auxiliary data: " f"{dataset_id['name']:s}") - def _standardize_dims(self, variable): + variable = self._manage_attributes(variable, info) + variable = self._drop_coords(variable) + variable = self._standardize_dims(variable) + return variable + + @staticmethod + def _standardize_dims(variable): """Standardize dims to y, x.""" if 'n_scans' in variable.dims: variable = variable.rename({'n_fovs': 'x', 'n_scans': 'y'}) @@ -160,6 +166,21 @@ def _standardize_dims(self, variable): variable = variable.transpose('y', 'x') return variable + @staticmethod + def _drop_coords(variable): + """Drop coords that are not in dims.""" + for coord in variable.coords: + if coord not in variable.dims: + variable = variable.drop_vars(coord) + return variable + + def _manage_attributes(self, variable, dataset_info): + """Manage attributes of the dataset.""" + variable.attrs.setdefault('units', None) + variable.attrs.update(dataset_info) + variable.attrs.update(self._get_global_attributes()) + return variable + def _get_dataset_channel(self, key, info=None): """Load dataset corresponding to channel measurement. @@ -224,13 +245,41 @@ def _get_dataset_aux_data(self, dsname, info=None): return None # Scale the data: - missing_value = variable.attrs['missing_value'] if 'scale_factor' in variable.attrs and 'add_offset' in variable.attrs: - variable.data = da.where(variable.data == missing_value, np.nan, - variable.data * variable.attrs['scale_factor'] + variable.attrs['add_offset']) + if 'missing_value' in variable.attrs: + missing_value = variable.attrs['missing_value'] + variable.data = da.where(variable.data == missing_value, np.nan, + variable.data * variable.attrs['scale_factor'] + variable.attrs['add_offset']) + else: + variable.data = variable.data * variable.attrs['scale_factor'] + variable.attrs['add_offset'] - # Manage the attributes of the dataset - variable.attrs.setdefault('units', None) - if info: - variable.attrs.update(info) return variable + + def _get_global_attributes(self): + """Create a dictionary of global attributes.""" + return { + 'filename': self.filename, + 'start_time': self.start_time, + 'end_time': self.end_time, + 'spacecraft_name': self.platform_name, + 'sensor': self.sensor, + 'filename_start_time': self.filename_info['sensing_start_time'], + 'filename_end_time': self.filename_info['sensing_end_time'], + 'platform_name': self.platform_name, + 'quality_group': self._get_quality_attributes(), + } + + def _get_quality_attributes(self): + """Get quality attributes.""" + quality_group = self['quality'] + quality_dict = {} + for key in quality_group: + # Add the values (as Numpy array) of each variable in the group + # where possible + try: + quality_dict[key] = quality_group[key].values + except ValueError: + quality_dict[key] = None + # Add the attributes of the quality group + quality_dict.update(quality_group.attrs) + return quality_dict diff --git a/satpy/tests/reader_tests/test_mws_l1b_nc.py b/satpy/tests/reader_tests/test_mws_l1b_nc.py index 362352b376..86d7a233e2 100644 --- a/satpy/tests/reader_tests/test_mws_l1b_nc.py +++ b/satpy/tests/reader_tests/test_mws_l1b_nc.py @@ -19,10 +19,13 @@ """ +import logging from datetime import datetime +from unittest.mock import patch import numpy as np import pytest +import xarray as xr from netCDF4 import Dataset from satpy.readers.mws_l1b import MWSL1BFile, get_channel_index_from_name @@ -81,6 +84,7 @@ def write(self): with Dataset(self.file_path, 'w') as dataset: self._write_attributes(dataset) self._write_status_group(dataset) + self._write_quality_group(dataset) data_group = dataset.createGroup('data') self._write_navigation_data_group(data_group) @@ -116,6 +120,16 @@ def _write_status_group(dataset): ) subsat_longitude_end[:] = 2.47 + @staticmethod + def _write_quality_group(dataset): + """Write the quality group.""" + group = dataset.createGroup('quality') + group.overall_quality_flag = 0 + duration_of_product = group.createVariable( + 'duration_of_product', "f4" + ) + duration_of_product[:] = 1000. + @staticmethod def _write_navigation_data_group(dataset): """Write the navigation data group.""" @@ -184,6 +198,59 @@ def test_sub_satellite_latitude_end(self, reader): """Test getting the latitude of sub-satellite point at end of the product.""" np.testing.assert_allclose(reader.sub_satellite_latitude_end, 60.0) + def test_get_dataset_raise_exception_if_data_not_exist(self, reader): + """Test get dataset return none if data does not exist.""" + dataset_id = {'name': 'unknown'} + dataset_info = {'file_key': 'non/existing/data'} + + with pytest.raises(ValueError) as exec_info: + _ = reader.get_dataset(dataset_id, dataset_info) + + assert str(exec_info.value) == 'Unknown dataset key, not a channel, quality or auxiliary data: unknown' + + def test_get_dataset_logs_debug_message(self, caplog, fake_file, reader): + """Test get dataset return none if data does not exist.""" + dataset_id = {'name': 'mws_lon'} + dataset_info = {'file_key': 'data/navigation_data/mws_lon'} + + with caplog.at_level(logging.DEBUG): + _ = reader.get_dataset(dataset_id, dataset_info) + + log_output = "Reading mws_lon from {filename}".format(filename=str(fake_file)) + assert log_output in caplog.text + + @staticmethod + def test_drop_coords(reader): + """Test drop coordinates.""" + coords = "dummy" + data = xr.DataArray( + np.ones(10), + dims=('y'), + coords={coords: 0}, + ) + assert coords in data.coords + data = reader._drop_coords(data) + assert coords not in data.coords + + @patch( + 'satpy.readers.mws_l1b.MWSL1BFile._get_global_attributes', + return_value={"mocked_global_attributes": True}, + ) + def test_manage_attributes(self, mock, reader): + """Test manage attributes.""" + variable = xr.DataArray( + np.ones(N_SCANS), + attrs={"season": "summer"}, + ) + dataset_info = {'name': '1', 'units': 'K'} + variable = reader._manage_attributes(variable, dataset_info) + assert variable.attrs == { + 'season': 'summer', + 'units': 'K', + 'name': '1', + 'mocked_global_attributes': True, + } + @pytest.mark.parametrize("name, index", [('1', 0), ('2', 1), ('24', 23)]) def test_get_channel_index_from_name(name, index): From f3463a6d70feae66fb28fa239b58de7d5a9455e1 Mon Sep 17 00:00:00 2001 From: "Adam.Dybbroe" Date: Fri, 26 Aug 2022 11:31:48 +0200 Subject: [PATCH 12/25] Refactor Signed-off-by: Adam.Dybbroe --- satpy/readers/pmw_channels_definitions.py | 78 ++++++++++++----------- 1 file changed, 41 insertions(+), 37 deletions(-) diff --git a/satpy/readers/pmw_channels_definitions.py b/satpy/readers/pmw_channels_definitions.py index 70c29d48ae..17979a5296 100644 --- a/satpy/readers/pmw_channels_definitions.py +++ b/satpy/readers/pmw_channels_definitions.py @@ -119,15 +119,12 @@ def __contains__(self, other): central_right_left = self.central + self.side - self.sideside central_right_right = self.central + self.side + self.sideside + four_centrals = [central_left_left, central_left_right, + central_right_left, central_right_right] if isinstance(other, numbers.Number): - if _is_inside_interval(other, central_left_left, self.bandwidth): - return True - if _is_inside_interval(other, central_left_right, self.bandwidth): - return True - if _is_inside_interval(other, central_right_left, self.bandwidth): - return True - if _is_inside_interval(other, central_right_right, self.bandwidth): - return True + for central in four_centrals: + if _is_inside_interval(other, central, self.bandwidth): + return True return False @@ -162,7 +159,6 @@ def distance(self, value): try: left_side_dist = abs(value.central - value.side - value.sideside - left_left) right_side_dist = abs(value.central + value.side + value.sideside - right_right) - return min(left_side_dist, right_side_dist) except AttributeError: if isinstance(value, (tuple, list)): msg = 'Distance to a quadruple side band frequency not supported for this type' @@ -170,7 +166,8 @@ def distance(self, value): left_side_dist = abs(value - left_left) right_side_dist = abs(value - right_right) - return min(left_side_dist, right_side_dist) + + return min(left_side_dist, right_side_dist) else: return np.inf @@ -262,37 +259,44 @@ def __contains__(self, other): """Check if this double-side-band 'contains' *other*.""" if other is None: return False + + leftside = self.central - self.side + rightside = self.central + self.side + if isinstance(other, numbers.Number): - if (self.central + self.side - self.bandwidth/2. <= other - <= self.central + self.side + self.bandwidth/2.): + if self._check_band_contains_other((leftside, self.bandwidth), (other, 0)): return True - if (self.central - self.side - self.bandwidth/2. <= other - <= self.central - self.side + self.bandwidth/2.): - return True - return False + return self._check_band_contains_other((rightside, self.bandwidth), (other, 0)) + other_leftside, other_rightside, other_bandwidth = 0, 0, 0 if isinstance(other, (tuple, list)) and len(other) == 3: - return ((self.central - self.side - self.bandwidth/2. <= - other[0] - other[1] - other[2]/2. and - self.central - self.side + self.bandwidth/2. >= - other[0] - other[1] + other[2]/2.) or - (self.central + self.side - self.bandwidth/2. <= - other[0] + other[1] - other[2]/2. and - self.central + self.side + self.bandwidth/2. >= - other[0] + other[1] + other[2]/2.)) + other_leftside = other[0] - other[1] + other_rightside = other[0] + other[1] + other_bandwidth = other[2] + else: + with suppress(AttributeError): + if self.unit != other.unit: + raise NotImplementedError("Can't compare frequency ranges with different units.") + other_leftside = other.central - other.side + other_rightside = other.central + other.side + other_bandwidth = other.bandwidth + + if self._check_band_contains_other((leftside, self.bandwidth), (other_leftside, other_bandwidth)): + return True + return self._check_band_contains_other((rightside, self.bandwidth), (other_rightside, other_bandwidth)) - with suppress(AttributeError): - if self.unit != other.unit: - raise NotImplementedError("Can't compare frequency ranges with different units.") - return ((self.central - self.side - self.bandwidth/2. <= - other.central - other.side - other.bandwidth/2. and - self.central - self.side + self.bandwidth/2. >= - other.central - other.side + other.bandwidth/2.) or - (self.central + self.side - self.bandwidth/2. <= - other.central + other.side - other.bandwidth/2. and - self.central + self.side + self.bandwidth/2. >= - other.central + other.side + other.bandwidth/2.)) + @staticmethod + def _check_band_contains_other(band, other_band): + """Check that a band contains another band. + A band is here defined as a tuple of a central frequency and a bandwidth. + """ + central1, width1 = band + central_other, width_other = other_band + + if ((central1 - width1/2. <= central_other - width_other/2.) and + (central1 + width1/2. >= central_other + width_other/2.)): + return True return False def distance(self, value): @@ -301,14 +305,14 @@ def distance(self, value): try: left_side_dist = abs(value.central - value.side - (self.central - self.side)) right_side_dist = abs(value.central + value.side - (self.central + self.side)) - return min(left_side_dist, right_side_dist) except AttributeError: if isinstance(value, (tuple, list)): return abs((value[0] - value[1]) - (self.central - self.side)) left_side_dist = abs(value - (self.central - self.side)) right_side_dist = abs(value - (self.central + self.side)) - return min(left_side_dist, right_side_dist) + + return min(left_side_dist, right_side_dist) else: return np.inf From 3b7bd7003ac41fd7cf6fa1c5b60cdfa620fc8cc1 Mon Sep 17 00:00:00 2001 From: "Adam.Dybbroe" Date: Fri, 26 Aug 2022 16:13:03 +0200 Subject: [PATCH 13/25] Refactoring and adding test Signed-off-by: Adam.Dybbroe --- satpy/etc/readers/mws_l1b_nc.yaml | 31 ++++++++++- satpy/readers/mws_l1b.py | 41 ++++++++------ satpy/tests/reader_tests/test_mws_l1b_nc.py | 60 +++++++++++++++------ 3 files changed, 99 insertions(+), 33 deletions(-) diff --git a/satpy/etc/readers/mws_l1b_nc.yaml b/satpy/etc/readers/mws_l1b_nc.yaml index 07b5b6b450..ce7fe527de 100644 --- a/satpy/etc/readers/mws_l1b_nc.yaml +++ b/satpy/etc/readers/mws_l1b_nc.yaml @@ -47,6 +47,7 @@ datasets: - mws_lon - mws_lat file_type: mws_l1b_nc + file_key: data/calibration/mws_toa_brightness_temperature '2': name: '2' frequency_range: @@ -62,6 +63,7 @@ datasets: - mws_lon - mws_lat file_type: mws_l1b_nc + file_key: data/calibration/mws_toa_brightness_temperature '3': name: '3' frequency_range: @@ -77,6 +79,7 @@ datasets: - mws_lon - mws_lat file_type: mws_l1b_nc + file_key: data/calibration/mws_toa_brightness_temperature '4': name: '4' frequency_range: @@ -92,6 +95,7 @@ datasets: - mws_lon - mws_lat file_type: mws_l1b_nc + file_key: data/calibration/mws_toa_brightness_temperature '5': name: '5' frequency_double_sideband: @@ -108,6 +112,7 @@ datasets: - mws_lon - mws_lat file_type: mws_l1b_nc + file_key: data/calibration/mws_toa_brightness_temperature '6': name: '6' frequency_double_sideband: @@ -124,6 +129,7 @@ datasets: - mws_lon - mws_lat file_type: mws_l1b_nc + file_key: data/calibration/mws_toa_brightness_temperature '7': name: '7' frequency_double_sideband: @@ -140,6 +146,7 @@ datasets: - mws_lon - mws_lat file_type: mws_l1b_nc + file_key: data/calibration/mws_toa_brightness_temperature '8': name: '8' frequency_range: @@ -155,6 +162,7 @@ datasets: - mws_lon - mws_lat file_type: mws_l1b_nc + file_key: data/calibration/mws_toa_brightness_temperature '9': name: '9' frequency_range: @@ -170,6 +178,7 @@ datasets: - mws_lon - mws_lat file_type: mws_l1b_nc + file_key: data/calibration/mws_toa_brightness_temperature '10': name: '10' frequency_range: @@ -185,6 +194,7 @@ datasets: - mws_lon - mws_lat file_type: mws_l1b_nc + file_key: data/calibration/mws_toa_brightness_temperature '11': name: '11' frequency_range: @@ -200,6 +210,7 @@ datasets: - mws_lon - mws_lat file_type: mws_l1b_nc + file_key: data/calibration/mws_toa_brightness_temperature '12': #57.290344±0.217 name: '12' @@ -217,6 +228,7 @@ datasets: - mws_lon - mws_lat file_type: mws_l1b_nc + file_key: data/calibration/mws_toa_brightness_temperature '13': #57.290344±0.3222±0.048 name: '13' @@ -235,6 +247,7 @@ datasets: - mws_lon - mws_lat file_type: mws_l1b_nc + file_key: data/calibration/mws_toa_brightness_temperature '14': #57.290344±0.3222±0.022 name: '14' @@ -253,6 +266,7 @@ datasets: - mws_lon - mws_lat file_type: mws_l1b_nc + file_key: data/calibration/mws_toa_brightness_temperature '15': #57.290344±0.3222±0.010 name: '15' @@ -271,6 +285,7 @@ datasets: - mws_lon - mws_lat file_type: mws_l1b_nc + file_key: data/calibration/mws_toa_brightness_temperature '16': #57.290344±0.3222±0.0045 name: '16' @@ -289,6 +304,7 @@ datasets: - mws_lon - mws_lat file_type: mws_l1b_nc + file_key: data/calibration/mws_toa_brightness_temperature '17': name: '17' frequency_range: @@ -304,6 +320,7 @@ datasets: - mws_lon - mws_lat file_type: mws_l1b_nc + file_key: data/calibration/mws_toa_brightness_temperature '18': name: '18' # FIXME! Is this a souble side band or what? MWS-18; 164–167; 2 x 1350; QH @@ -320,6 +337,7 @@ datasets: - mws_lon - mws_lat file_type: mws_l1b_nc + file_key: data/calibration/mws_toa_brightness_temperature '19': name: '19' frequency_double_sideband: @@ -336,6 +354,7 @@ datasets: - mws_lon - mws_lat file_type: mws_l1b_nc + file_key: data/calibration/mws_toa_brightness_temperature '20': name: '20' frequency_double_sideband: @@ -352,6 +371,7 @@ datasets: - mws_lon - mws_lat file_type: mws_l1b_nc + file_key: data/calibration/mws_toa_brightness_temperature '21': name: '21' frequency_double_sideband: @@ -368,6 +388,7 @@ datasets: - mws_lon - mws_lat file_type: mws_l1b_nc + file_key: data/calibration/mws_toa_brightness_temperature '22': name: '22' frequency_double_sideband: @@ -384,6 +405,7 @@ datasets: - mws_lon - mws_lat file_type: mws_l1b_nc + file_key: data/calibration/mws_toa_brightness_temperature '23': name: '23' frequency_double_sideband: @@ -400,6 +422,7 @@ datasets: - mws_lon - mws_lat file_type: mws_l1b_nc + file_key: data/calibration/mws_toa_brightness_temperature '24': name: '24' frequency_range: @@ -415,6 +438,7 @@ datasets: - mws_lon - mws_lat file_type: mws_l1b_nc + file_key: data/calibration/mws_toa_brightness_temperature # --- Coordinates --- @@ -422,6 +446,7 @@ datasets: name: mws_lat resolution: 17000 file_type: mws_l1b_nc + file_key: data/navigation/mws_lat standard_name: latitude units: degrees_north @@ -429,6 +454,7 @@ datasets: name: mws_lon resolution: 17000 file_type: mws_l1b_nc + file_key: data/navigation/mws_lat standard_name: longitude units: degrees_east @@ -438,7 +464,7 @@ datasets: name: solar_azimuth standard_name: solar_azimuth_angle file_type: mws_l1b_nc - #file_key: data/navigation/mws_solar_azimuth_angle + file_key: data/navigation/mws_solar_azimuth_angle coordinates: - mws_lon - mws_lat @@ -446,6 +472,7 @@ datasets: name: solar_zenith standard_name: solar_zenith_angle file_type: mws_l1b_nc + file_key: data/navigation/mws_solar_zenith_angle coordinates: - mws_lon - mws_lat @@ -453,6 +480,7 @@ datasets: name: satellite_azimuth standard_name: satellite_azimuth_angle file_type: mws_l1b_nc + file_key: data/navigation/mws_satellite_azimuth_angle coordinates: - mws_lon - mws_lat @@ -460,6 +488,7 @@ datasets: name: satellite_zenith standard_name: satellite_zenith_angle file_type: mws_l1b_nc + file_key: data/navigation/mws_satellite_zenith_angle coordinates: - mws_lon - mws_lat diff --git a/satpy/readers/mws_l1b.py b/satpy/readers/mws_l1b.py index 1d44249a59..7a55f91f6b 100644 --- a/satpy/readers/mws_l1b.py +++ b/satpy/readers/mws_l1b.py @@ -140,19 +140,21 @@ def sub_satellite_latitude_end(self): """Get the latitude of sub-satellite point at end of the product.""" return self['status/satellite/subsat_latitude_end'].data.item() - def get_dataset(self, dataset_id, info=None): - """Load a dataset.""" + def get_dataset(self, dataset_id, dataset_info): + """Get dataset using file_key in dataset_info.""" logger.debug('Reading {} from {}'.format(dataset_id['name'], self.filename)) + var_key = dataset_info['file_key'] if _get_aux_data_name_from_dsname(dataset_id['name']) is not None: - variable = self._get_dataset_aux_data(dataset_id['name'], info=info) + variable = self._get_dataset_aux_data(dataset_id['name'], info=dataset_info) elif any(lb in dataset_id['name'] for lb in MWS_CHANNELS): - variable = self._get_dataset_channel(dataset_id, info=info) + logger.debug(f'Reading in file to get dataset with key {var_key}.') + variable = self._get_dataset_channel(dataset_id, dataset_info) else: - raise ValueError("Unknown dataset key, not a channel, quality or auxiliary data: " - f"{dataset_id['name']:s}") + logger.warning(f'Could not find key {var_key} in NetCDF file, no valid Dataset created') # noqa: E501 + return None - variable = self._manage_attributes(variable, info) + variable = self._manage_attributes(variable, dataset_info) variable = self._drop_coords(variable) variable = self._standardize_dims(variable) return variable @@ -181,7 +183,7 @@ def _manage_attributes(self, variable, dataset_info): variable.attrs.update(self._get_global_attributes()) return variable - def _get_dataset_channel(self, key, info=None): + def _get_dataset_channel(self, key, dataset_info): """Load dataset corresponding to channel measurement. Load a dataset when the key refers to a measurand, whether uncalibrated @@ -190,7 +192,7 @@ def _get_dataset_channel(self, key, info=None): """ # Get the dataset # Get metadata for given dataset - grp_pth = 'data/calibration/mws_toa_brightness_temperature' + grp_pth = dataset_info['file_key'] channel_index = get_channel_index_from_name(key['name']) data = self[grp_pth][:, :, channel_index] @@ -200,6 +202,7 @@ def _get_dataset_channel(self, key, info=None): "FillValue", default_fillvals.get(data.dtype.str[1:], np.nan)) vr = attrs.get("valid_range", [-np.inf, np.inf]) + if key['calibration'] == "counts": attrs["_FillValue"] = fv nfv = fv @@ -210,11 +213,11 @@ def _get_dataset_channel(self, key, info=None): # Manage the attributes of the dataset data.attrs.setdefault('units', None) - data.attrs.update(info) + data.attrs.update(dataset_info) - i = getattr(data, 'attrs', {}) - i.update(info) - i.update({ + dataset_attrs = getattr(data, 'attrs', {}) + dataset_attrs.update(dataset_info) + dataset_attrs.update({ "platform_name": self.platform_name, "sensor": self.sensor, "orbital_parameters": {'sub_satellite_latitude_start': self.sub_satellite_latitude_start, @@ -222,9 +225,13 @@ def _get_dataset_channel(self, key, info=None): 'sub_satellite_latitude_end': self.sub_satellite_latitude_end, 'sub_satellite_longitude_end': self.sub_satellite_longitude_end}, }) - i.update(key.to_dict()) - data.attrs.update(i) + try: + dataset_attrs.update(key.to_dict()) + except AttributeError: + dataset_attrs.update(key) + + data.attrs.update(dataset_attrs) return data def _get_dataset_aux_data(self, dsname, info=None): @@ -263,8 +270,8 @@ def _get_global_attributes(self): 'end_time': self.end_time, 'spacecraft_name': self.platform_name, 'sensor': self.sensor, - 'filename_start_time': self.filename_info['sensing_start_time'], - 'filename_end_time': self.filename_info['sensing_end_time'], + 'filename_start_time': self.filename_info['start_time'], + 'filename_end_time': self.filename_info['end_time'], 'platform_name': self.platform_name, 'quality_group': self._get_quality_attributes(), } diff --git a/satpy/tests/reader_tests/test_mws_l1b_nc.py b/satpy/tests/reader_tests/test_mws_l1b_nc.py index 86d7a233e2..ca8f3c6489 100644 --- a/satpy/tests/reader_tests/test_mws_l1b_nc.py +++ b/satpy/tests/reader_tests/test_mws_l1b_nc.py @@ -44,10 +44,10 @@ def reader(fake_file): return MWSL1BFile( filename=fake_file, filename_info={ - 'sensing_start_time': ( + 'start_time': ( datetime.fromisoformat('2000-01-01T01:00:00') ), - 'sensing_end_time': ( + 'end_time': ( datetime.fromisoformat('2000-01-01T02:00:00') ), 'creation_time': ( @@ -59,6 +59,8 @@ def reader(fake_file): 'latitude': 'data/navigation_data/mws_lat', 'solar_azimuth': 'data/navigation/mws_solar_azimuth_angle', 'solar_zenith': 'data/navigation/mws_solar_zenith_angle', + 'satellite_azimuth': 'data/navigation/mws_satellite_azimuth_angle', + 'satellite_zenith': 'data/navigation/mws_satellite_zenith_angle', } ) @@ -86,7 +88,9 @@ def write(self): self._write_status_group(dataset) self._write_quality_group(dataset) data_group = dataset.createGroup('data') + self._create_scan_dimensions(data_group) self._write_navigation_data_group(data_group) + self._write_calibration_data_group(data_group) @staticmethod def _write_attributes(dataset): @@ -134,13 +138,6 @@ def _write_quality_group(dataset): def _write_navigation_data_group(dataset): """Write the navigation data group.""" group = dataset.createGroup('navigation') - group.createDimension('n_schannels', N_CHANNELS) - group.createDimension('n_schannels_os', N_CHANNELS_OS) - group.createDimension('n_scans', N_SCANS) - group.createDimension('n_fovs', N_FOVS) - group.createDimension('n_prts', N_PRTS) - group.createDimension('n_fovs_cal', N_FOVS_CAL) - dimensions = ('n_scans', 'n_fovs') shape = (N_SCANS, N_FOVS) longitude = group.createVariable( @@ -162,6 +159,25 @@ def _write_navigation_data_group(dataset): ) azimuth[:] = 3. * np.ones(shape) + @staticmethod + def _create_scan_dimensions(dataset): + """Create the scan/fovs dimensions.""" + dataset.createDimension('n_channels', N_CHANNELS) + dataset.createDimension('n_channels_os', N_CHANNELS_OS) + dataset.createDimension('n_scans', N_SCANS) + dataset.createDimension('n_fovs', N_FOVS) + dataset.createDimension('n_prts', N_PRTS) + dataset.createDimension('n_fovs_cal', N_FOVS_CAL) + + @staticmethod + def _write_calibration_data_group(dataset): + """Write the measurement data group.""" + group = dataset.createGroup('calibration') + toa_bt = group.createVariable( + 'mws_toa_brightness_temperature', np.float32, dimensions=('n_scans', 'n_fovs', 'n_channels',) + ) + toa_bt[:] = 240.0 * np.ones((N_SCANS, N_FOVS, N_CHANNELS)) + class TestMwsL1bNCFileHandler: """Test the MWSL1BFile reader.""" @@ -198,15 +214,29 @@ def test_sub_satellite_latitude_end(self, reader): """Test getting the latitude of sub-satellite point at end of the product.""" np.testing.assert_allclose(reader.sub_satellite_latitude_end, 60.0) - def test_get_dataset_raise_exception_if_data_not_exist(self, reader): + def test_get_dataset_get_channeldata(self, reader): + """Test getting channel data.""" + dataset_id = {'name': '1', 'units': 'K', + 'calibration': 'brightness_temperature'} + dataset_info = {'file_key': 'data/calibration/mws_toa_brightness_temperature'} + + dataset = reader.get_dataset(dataset_id, dataset_info) + + expected_bt = np.array([[240., 240., 240., 240., 240.], + [240., 240., 240., 240., 240.], + [240., 240., 240., 240., 240.], + [240., 240., 240., 240., 240.], + [240., 240., 240., 240., 240.]], dtype=np.float32) + + toa_bt = dataset[0:5, 0:5].data.compute() + np.testing.assert_allclose(toa_bt, expected_bt) + + def test_get_dataset_return_none_if_data_not_exist(self, reader): """Test get dataset return none if data does not exist.""" dataset_id = {'name': 'unknown'} dataset_info = {'file_key': 'non/existing/data'} - - with pytest.raises(ValueError) as exec_info: - _ = reader.get_dataset(dataset_id, dataset_info) - - assert str(exec_info.value) == 'Unknown dataset key, not a channel, quality or auxiliary data: unknown' + dataset = reader.get_dataset(dataset_id, dataset_info) + assert dataset is None def test_get_dataset_logs_debug_message(self, caplog, fake_file, reader): """Test get dataset return none if data does not exist.""" From 8e1296380a925ccd4f3cbaa6dc3cee81ed96d053 Mon Sep 17 00:00:00 2001 From: "Adam.Dybbroe" Date: Fri, 26 Aug 2022 17:34:03 +0200 Subject: [PATCH 14/25] Improve test coverage Signed-off-by: Adam.Dybbroe --- satpy/readers/mws_l1b.py | 10 ++--- satpy/tests/reader_tests/test_mws_l1b_nc.py | 49 ++++++++++++++++++++- 2 files changed, 52 insertions(+), 7 deletions(-) diff --git a/satpy/readers/mws_l1b.py b/satpy/readers/mws_l1b.py index 7a55f91f6b..3487318df2 100644 --- a/satpy/readers/mws_l1b.py +++ b/satpy/readers/mws_l1b.py @@ -146,7 +146,7 @@ def get_dataset(self, dataset_id, dataset_info): var_key = dataset_info['file_key'] if _get_aux_data_name_from_dsname(dataset_id['name']) is not None: - variable = self._get_dataset_aux_data(dataset_id['name'], info=dataset_info) + variable = self._get_dataset_aux_data(dataset_id['name']) elif any(lb in dataset_id['name'] for lb in MWS_CHANNELS): logger.debug(f'Reading in file to get dataset with key {var_key}.') variable = self._get_dataset_channel(dataset_id, dataset_info) @@ -187,8 +187,8 @@ def _get_dataset_channel(self, key, dataset_info): """Load dataset corresponding to channel measurement. Load a dataset when the key refers to a measurand, whether uncalibrated - (counts) or calibrated in terms of brightness temperature, radiance, or - reflectance. + (counts) or calibrated in terms of brightness temperature or radiance. + """ # Get the dataset # Get metadata for given dataset @@ -234,7 +234,7 @@ def _get_dataset_channel(self, key, dataset_info): data.attrs.update(dataset_attrs) return data - def _get_dataset_aux_data(self, dsname, info=None): + def _get_dataset_aux_data(self, dsname): """Get the auxiliary data arrays using the index map.""" # Geolocation and navigation data: if dsname in ['mws_lat', 'mws_lon', @@ -243,7 +243,7 @@ def _get_dataset_aux_data(self, dsname, info=None): 'surface_type', 'terrain_elevation']: var_key = AUX_DATA.get(dsname) else: - raise NotImplementedError("Dataset %s not supported..." % dsname) + raise NotImplementedError(f"Dataset '{dsname}' not supported!") try: variable = self[var_key] diff --git a/satpy/tests/reader_tests/test_mws_l1b_nc.py b/satpy/tests/reader_tests/test_mws_l1b_nc.py index ca8f3c6489..d9ed55fc79 100644 --- a/satpy/tests/reader_tests/test_mws_l1b_nc.py +++ b/satpy/tests/reader_tests/test_mws_l1b_nc.py @@ -91,6 +91,7 @@ def write(self): self._create_scan_dimensions(data_group) self._write_navigation_data_group(data_group) self._write_calibration_data_group(data_group) + self._write_measurement_data_group(data_group) @staticmethod def _write_attributes(dataset): @@ -171,13 +172,22 @@ def _create_scan_dimensions(dataset): @staticmethod def _write_calibration_data_group(dataset): - """Write the measurement data group.""" + """Write the calibration data group.""" group = dataset.createGroup('calibration') toa_bt = group.createVariable( 'mws_toa_brightness_temperature', np.float32, dimensions=('n_scans', 'n_fovs', 'n_channels',) ) toa_bt[:] = 240.0 * np.ones((N_SCANS, N_FOVS, N_CHANNELS)) + @staticmethod + def _write_measurement_data_group(dataset): + """Write the measurement data group.""" + group = dataset.createGroup('measurement') + counts = group.createVariable( + 'mws_earth_view_counts', np.int32, dimensions=('n_scans', 'n_fovs', 'n_channels',) + ) + counts[:] = 24100 * np.ones((N_SCANS, N_FOVS, N_CHANNELS), dtype=np.int32) + class TestMwsL1bNCFileHandler: """Test the MWSL1BFile reader.""" @@ -214,7 +224,19 @@ def test_sub_satellite_latitude_end(self, reader): """Test getting the latitude of sub-satellite point at end of the product.""" np.testing.assert_allclose(reader.sub_satellite_latitude_end, 60.0) - def test_get_dataset_get_channeldata(self, reader): + def test_get_dataset_get_channeldata_counts(self, reader): + """Test getting channel data.""" + dataset_id = {'name': '1', 'units': None, + 'calibration': 'counts'} + dataset_info = {'file_key': 'data/measurement/mws_earth_view_counts'} + + dataset = reader.get_dataset(dataset_id, dataset_info) + expected_bt = np.array([[24100, 24100], + [24100, 24100]], dtype=np.int32) + count = dataset[10:12, 12:14].data.compute() + np.testing.assert_allclose(count, expected_bt) + + def test_get_dataset_get_channeldata_bts(self, reader): """Test getting channel data.""" dataset_id = {'name': '1', 'units': 'K', 'calibration': 'brightness_temperature'} @@ -249,6 +271,29 @@ def test_get_dataset_logs_debug_message(self, caplog, fake_file, reader): log_output = "Reading mws_lon from {filename}".format(filename=str(fake_file)) assert log_output in caplog.text + def test_get_dataset_aux_data_not_supported(self, reader): + """Test get auxillary dataset not supported.""" + dataset_id = {'name': 'scantime_utc'} + dataset_info = {'file_key': 'non/existing'} + + with pytest.raises(NotImplementedError) as exec_info: + _ = reader.get_dataset(dataset_id, dataset_info) + + assert str(exec_info.value) == "Dataset 'scantime_utc' not supported!" + + @pytest.mark.parametrize('dims', ( + ('n_scans', 'n_fovs'), + ('x', 'y'), + )) + def test_standardize_dims(self, reader, dims): + """Test standardize dims.""" + variable = xr.DataArray( + np.arange(6).reshape(2, 3), + dims=dims, + ) + standardized = reader._standardize_dims(variable) + assert standardized.dims == ('y', 'x') + @staticmethod def test_drop_coords(reader): """Test drop coordinates.""" From f739ca6ef440b747fa13b1ed6be74b086e2ab98d Mon Sep 17 00:00:00 2001 From: "Adam.Dybbroe" Date: Sat, 27 Aug 2022 22:43:39 +0200 Subject: [PATCH 15/25] Remove redundant code, all datasets in the file will have a missing_value Signed-off-by: Adam.Dybbroe --- satpy/readers/mws_l1b.py | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/satpy/readers/mws_l1b.py b/satpy/readers/mws_l1b.py index 3487318df2..7bd9aff649 100644 --- a/satpy/readers/mws_l1b.py +++ b/satpy/readers/mws_l1b.py @@ -253,12 +253,9 @@ def _get_dataset_aux_data(self, dsname): # Scale the data: if 'scale_factor' in variable.attrs and 'add_offset' in variable.attrs: - if 'missing_value' in variable.attrs: - missing_value = variable.attrs['missing_value'] - variable.data = da.where(variable.data == missing_value, np.nan, - variable.data * variable.attrs['scale_factor'] + variable.attrs['add_offset']) - else: - variable.data = variable.data * variable.attrs['scale_factor'] + variable.attrs['add_offset'] + missing_value = variable.attrs['missing_value'] + variable.data = da.where(variable.data == missing_value, np.nan, + variable.data * variable.attrs['scale_factor'] + variable.attrs['add_offset']) return variable @@ -287,6 +284,6 @@ def _get_quality_attributes(self): quality_dict[key] = quality_group[key].values except ValueError: quality_dict[key] = None - # Add the attributes of the quality group + quality_dict.update(quality_group.attrs) return quality_dict From 10e5d60704025e6620bef502a79fa3da20c9993b Mon Sep 17 00:00:00 2001 From: "Adam.Dybbroe" Date: Sat, 27 Aug 2022 22:44:01 +0200 Subject: [PATCH 16/25] Improve documentation Signed-off-by: Adam.Dybbroe --- satpy/readers/pmw_channels_definitions.py | 35 ++++++++++++++++++----- 1 file changed, 28 insertions(+), 7 deletions(-) diff --git a/satpy/readers/pmw_channels_definitions.py b/satpy/readers/pmw_channels_definitions.py index 17979a5296..1bdf31a211 100644 --- a/satpy/readers/pmw_channels_definitions.py +++ b/satpy/readers/pmw_channels_definitions.py @@ -141,16 +141,20 @@ def distance(self, value): """Get the distance to the quadruple side band. Determining the distance in frequency space between two quadruple side - bands can be quite ambiguous, as such bands are in effect a set of - 4 narrow bands. To keep it as simple as possible we have until further - decided to define the distance between such two bands by calculating - the distances between the outermost left-sides of the two bands and the - distances of the outermost right sides and then minimising those two. + bands can be quite ambiguous, as such bands are in effect a set of 4 + narrow bands, two on each side of the main absorption band, and on each + side, one on each side of the secondary absorption lines. To keep it as + simple as possible we have until further decided to define the distance + between such two bands to infinity if they are determined to be equal. - If the frequency entered is a single value the distance will be the + If the frequency entered is a single value, the distance will be the minimum of the distances to the two outermost sides of the quadruple side band. + If the frequency entered is a tuple or list and the two quadruple + frequency bands are contained in each other (equal) the distance will + always be zero. + """ left_left = self.central - self.side - self.sideside right_right = self.central + self.side + self.sideside @@ -300,7 +304,24 @@ def _check_band_contains_other(band, other_band): return False def distance(self, value): - """Get the distance from value.""" + """Get the distance to the double side band. + + Determining the distance in frequency space between two double side + bands can be quite ambiguous, as such bands are in effect a set of 2 + narrow bands, one on each side of the absorption line. To keep it + as simple as possible we have until further decided to set the + distance between such two bands to infitiy if neither of them are + contained in the other. + + If the frequency entered is a single value and this frequency falls + inside one of the side bands, the distance will be the minimum of the + distances to the two outermost sides of the double side band. However, + is such a single frequency value falls outside one of the two side + bands, the distance will be set to infitiy. + + If the frequency entered is a tuple the distance will either be 0 (if + one is containde in the other) or infinity. + """ if self == value: try: left_side_dist = abs(value.central - value.side - (self.central - self.side)) From ea6155f9027df1f9eb7ba0505f8bc33d6eb760c0 Mon Sep 17 00:00:00 2001 From: "Adam.Dybbroe" Date: Sat, 27 Aug 2022 22:45:30 +0200 Subject: [PATCH 17/25] Increase test coverage Signed-off-by: Adam.Dybbroe --- satpy/tests/reader_tests/test_mws_l1b_nc.py | 20 ++++++- satpy/tests/test_dataset.py | 63 ++++++++++++--------- 2 files changed, 56 insertions(+), 27 deletions(-) diff --git a/satpy/tests/reader_tests/test_mws_l1b_nc.py b/satpy/tests/reader_tests/test_mws_l1b_nc.py index d9ed55fc79..0363acfbdf 100644 --- a/satpy/tests/reader_tests/test_mws_l1b_nc.py +++ b/satpy/tests/reader_tests/test_mws_l1b_nc.py @@ -133,7 +133,7 @@ def _write_quality_group(dataset): duration_of_product = group.createVariable( 'duration_of_product', "f4" ) - duration_of_product[:] = 1000. + duration_of_product[:] = 5944. @staticmethod def _write_navigation_data_group(dataset): @@ -307,6 +307,24 @@ def test_drop_coords(reader): data = reader._drop_coords(data) assert coords not in data.coords + def test_get_global_attributes(self, reader): + """Test get global attributes.""" + attributes = reader._get_global_attributes() + assert attributes == { + 'filename': reader.filename, + 'start_time': datetime(2000, 1, 2, 3, 4, 5), + 'end_time': datetime(2000, 1, 2, 4, 5, 6), + 'spacecraft_name': 'Metop-SG-A1', + 'sensor': 'MWS', + 'filename_start_time': datetime(2000, 1, 1, 1, 0), + 'filename_end_time': datetime(2000, 1, 1, 2, 0), + 'platform_name': 'Metop-SG-A1', + 'quality_group': { + 'duration_of_product': np.array(5944., dtype=np.float32), + 'overall_quality_flag': 0, + } + } + @patch( 'satpy.readers.mws_l1b.MWSL1BFile._get_global_attributes', return_value={"mocked_global_attributes": True}, diff --git a/satpy/tests/test_dataset.py b/satpy/tests/test_dataset.py index d6f3ac83ac..e46ab4c6cf 100644 --- a/satpy/tests/test_dataset.py +++ b/satpy/tests/test_dataset.py @@ -691,62 +691,72 @@ def test_seviri_hrv_has_priority_over_vis008(self): def test_frequency_quadruple_side_band_class_method_convert(): """Test the frequency double side band object: test the class method convert.""" - frq_dsb = FrequencyQuadrupleSideBand(57, 0.322, 0.05, 0.036) + frq_qdsb = FrequencyQuadrupleSideBand(57, 0.322, 0.05, 0.036) - res = frq_dsb.convert(57.37) + res = frq_qdsb.convert(57.37) assert res == 57.37 - res = frq_dsb.convert({'central': 57.0, 'side': 0.322, 'sideside': 0.05, 'bandwidth': 0.036}) + res = frq_qdsb.convert({'central': 57.0, 'side': 0.322, 'sideside': 0.05, 'bandwidth': 0.036}) assert res == FrequencyQuadrupleSideBand(57, 0.322, 0.05, 0.036) def test_frequency_quadruple_side_band_channel_str(): """Test the frequency quadruple side band object: test the band description.""" - frq_dsb1 = FrequencyQuadrupleSideBand(57.0, 0.322, 0.05, 0.036) - frq_dsb2 = FrequencyQuadrupleSideBand(57000, 322, 50, 36, 'MHz') + frq_qdsb1 = FrequencyQuadrupleSideBand(57.0, 0.322, 0.05, 0.036) + frq_qdsb2 = FrequencyQuadrupleSideBand(57000, 322, 50, 36, 'MHz') - assert str(frq_dsb1) == "central=57.0 GHz ±0.322 ±0.05 width=0.036 GHz" - assert str(frq_dsb2) == "central=57000 MHz ±322 ±50 width=36 MHz" + assert str(frq_qdsb1) == "central=57.0 GHz ±0.322 ±0.05 width=0.036 GHz" + assert str(frq_qdsb2) == "central=57000 MHz ±322 ±50 width=36 MHz" def test_frequency_quadruple_side_band_channel_equality(): """Test the frequency quadruple side band object: check if two bands are 'equal'.""" - frq_dsb = FrequencyQuadrupleSideBand(57, 0.322, 0.05, 0.036) - assert frq_dsb is not None - assert 57 != frq_dsb - assert 57.372 == frq_dsb - assert 56.646 == frq_dsb - assert 56.71 == frq_dsb + frq_qdsb = FrequencyQuadrupleSideBand(57, 0.322, 0.05, 0.036) + assert frq_qdsb is not None + assert frq_qdsb < FrequencyQuadrupleSideBand(57, 0.322, 0.05, 0.04) + assert frq_qdsb < FrequencyQuadrupleSideBand(58, 0.322, 0.05, 0.036) + assert frq_qdsb < ((58, 0.322, 0.05, 0.036)) - assert frq_dsb != FrequencyQuadrupleSideBand(57, 0.322, 0.1, 0.040) + assert 57 != frq_qdsb + assert 57.372 == frq_qdsb + assert 56.646 == frq_qdsb + assert 56.71 == frq_qdsb - frq_dsb = None - assert FrequencyQuadrupleSideBand(57, 0.322, 0.05, 0.036) != frq_dsb - assert frq_dsb < FrequencyQuadrupleSideBand(57, 0.322, 0.05, 0.04) + assert frq_qdsb != FrequencyQuadrupleSideBand(57, 0.322, 0.1, 0.040) + + frq_qdsb = None + assert FrequencyQuadrupleSideBand(57, 0.322, 0.05, 0.036) != frq_qdsb + assert frq_qdsb < FrequencyQuadrupleSideBand(57, 0.322, 0.05, 0.04) def test_frequency_quadruple_side_band_channel_distances(): """Test the frequency quadruple side band object: get the distance between two bands.""" - frq_dsb = FrequencyQuadrupleSideBand(57, 0.322, 0.05, 0.036) - mydist = frq_dsb.distance(57.372) + frq_qdsb = FrequencyQuadrupleSideBand(57, 0.322, 0.05, 0.036) + mydist = frq_qdsb.distance(57.372) assert mydist == 0.0 - mydist = frq_dsb.distance(57) + mydist = frq_qdsb.distance(57.38) + np.testing.assert_almost_equal(mydist, 0.008) + + mydist = frq_qdsb.distance(57) assert mydist == np.inf + mydist = frq_qdsb.distance((57, 0.322, 0.05, 0.018)) + assert mydist == 0.0 + def test_frequency_quadruple_side_band_channel_containment(): """Test the frequency quadruple side band object: check if one band contains another.""" - frq_dsb = FrequencyQuadrupleSideBand(57, 0.322, 0.05, 0.036) + frq_qdsb = FrequencyQuadrupleSideBand(57, 0.322, 0.05, 0.036) - assert 57 not in frq_dsb - assert 57.373 in frq_dsb + assert 57 not in frq_qdsb + assert 57.373 in frq_qdsb with pytest.raises(NotImplementedError): - assert frq_dsb in FrequencyQuadrupleSideBand(57, 0.322, 0.05, 0.05) + assert frq_qdsb in FrequencyQuadrupleSideBand(57, 0.322, 0.05, 0.05) - frq_dsb = None - assert (frq_dsb in FrequencyQuadrupleSideBand(57, 0.322, 0.05, 0.05)) is False + frq_qdsb = None + assert (frq_qdsb in FrequencyQuadrupleSideBand(57, 0.322, 0.05, 0.05)) is False assert '57' not in FrequencyQuadrupleSideBand(57, 0.322, 0.05, 0.05) @@ -786,6 +796,7 @@ def test_frequency_double_side_band_channel_equality(): assert FrequencyDoubleSideBand(183, 7, 2) != frq_dsb assert frq_dsb < FrequencyDoubleSideBand(183, 7, 2) + assert FrequencyDoubleSideBand(182, 7, 2) < FrequencyDoubleSideBand(183, 7, 2) assert FrequencyDoubleSideBand(184, 7, 2) > FrequencyDoubleSideBand(183, 7, 2) From 6d94e7277dff2ac7b5890443297094adc22f9819 Mon Sep 17 00:00:00 2001 From: "Adam.Dybbroe" Date: Sat, 27 Aug 2022 23:34:59 +0200 Subject: [PATCH 18/25] Bugfix Signed-off-by: Adam.Dybbroe --- satpy/tests/test_dataset.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/satpy/tests/test_dataset.py b/satpy/tests/test_dataset.py index e46ab4c6cf..293e8a83e1 100644 --- a/satpy/tests/test_dataset.py +++ b/satpy/tests/test_dataset.py @@ -742,7 +742,7 @@ def test_frequency_quadruple_side_band_channel_distances(): assert mydist == np.inf mydist = frq_qdsb.distance((57, 0.322, 0.05, 0.018)) - assert mydist == 0.0 + assert mydist == np.inf def test_frequency_quadruple_side_band_channel_containment(): From c1f71ff7385c3a389d2c1ac099e89fea80b846da Mon Sep 17 00:00:00 2001 From: "Adam.Dybbroe" Date: Sun, 28 Aug 2022 13:40:41 +0200 Subject: [PATCH 19/25] Bugfix - Throw exception if expected dataset is not present in file Signed-off-by: Adam.Dybbroe --- satpy/readers/mws_l1b.py | 5 ++--- satpy/tests/reader_tests/test_mws_l1b_nc.py | 15 +++++++++++++++ 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/satpy/readers/mws_l1b.py b/satpy/readers/mws_l1b.py index 7bd9aff649..affb5e1538 100644 --- a/satpy/readers/mws_l1b.py +++ b/satpy/readers/mws_l1b.py @@ -248,15 +248,14 @@ def _get_dataset_aux_data(self, dsname): try: variable = self[var_key] except KeyError: - logger.warning("Could not find key %s in NetCDF file, no valid Dataset created", var_key) - return None + logger.exception("Could not find key %s in NetCDF file, no valid Dataset created", var_key) + raise # Scale the data: if 'scale_factor' in variable.attrs and 'add_offset' in variable.attrs: missing_value = variable.attrs['missing_value'] variable.data = da.where(variable.data == missing_value, np.nan, variable.data * variable.attrs['scale_factor'] + variable.attrs['add_offset']) - return variable def _get_global_attributes(self): diff --git a/satpy/tests/reader_tests/test_mws_l1b_nc.py b/satpy/tests/reader_tests/test_mws_l1b_nc.py index 0363acfbdf..b171a5f8bb 100644 --- a/satpy/tests/reader_tests/test_mws_l1b_nc.py +++ b/satpy/tests/reader_tests/test_mws_l1b_nc.py @@ -281,6 +281,21 @@ def test_get_dataset_aux_data_not_supported(self, reader): assert str(exec_info.value) == "Dataset 'scantime_utc' not supported!" + def test_get_dataset_aux_data_expected_data_missing(self, caplog, reader): + """Test get auxillary dataset which is not present but supposed to be in file.""" + dataset_id = {'name': 'surface_type'} + dataset_info = {'file_key': 'non/existing'} + + with caplog.at_level(logging.ERROR): + with pytest.raises(KeyError) as exec_info: + _ = reader.get_dataset(dataset_id, dataset_info) + + assert str(exec_info.value) == "'data/navigation/mws_surface_type'" + + log_output = ("Could not find key data/navigation/mws_surface_type in NetCDF file," + + " no valid Dataset created") + assert log_output in caplog.text + @pytest.mark.parametrize('dims', ( ('n_scans', 'n_fovs'), ('x', 'y'), From 953fb801f54aff420d57332a4c9840847d365072 Mon Sep 17 00:00:00 2001 From: "Adam.Dybbroe" Date: Sun, 28 Aug 2022 13:41:03 +0200 Subject: [PATCH 20/25] Increase test coverage Signed-off-by: Adam.Dybbroe --- satpy/tests/test_dataset.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/satpy/tests/test_dataset.py b/satpy/tests/test_dataset.py index 293e8a83e1..059865edde 100644 --- a/satpy/tests/test_dataset.py +++ b/satpy/tests/test_dataset.py @@ -716,6 +716,9 @@ def test_frequency_quadruple_side_band_channel_equality(): assert frq_qdsb < FrequencyQuadrupleSideBand(57, 0.322, 0.05, 0.04) assert frq_qdsb < FrequencyQuadrupleSideBand(58, 0.322, 0.05, 0.036) assert frq_qdsb < ((58, 0.322, 0.05, 0.036)) + assert frq_qdsb > FrequencyQuadrupleSideBand(57, 0.322, 0.04, 0.01) + assert frq_qdsb > None + assert (frq_qdsb < None) is False assert 57 != frq_qdsb assert 57.372 == frq_qdsb From cba9a2453bf065387ba0497b33ff858623b02259 Mon Sep 17 00:00:00 2001 From: "Adam.Dybbroe" Date: Wed, 31 Aug 2022 15:25:16 +0200 Subject: [PATCH 21/25] Refactor and use a mixin class with common operations among all Frequency types Signed-off-by: Adam.Dybbroe --- satpy/readers/pmw_channels_definitions.py | 104 +++++++--------------- 1 file changed, 31 insertions(+), 73 deletions(-) diff --git a/satpy/readers/pmw_channels_definitions.py b/satpy/readers/pmw_channels_definitions.py index 1bdf31a211..ab57965d1b 100644 --- a/satpy/readers/pmw_channels_definitions.py +++ b/satpy/readers/pmw_channels_definitions.py @@ -25,6 +25,33 @@ import numpy as np +class FrequencyBandBaseArithmetics: + """Mixin class with basic frequency comparison operations.""" + + def __lt__(self, other): + """Compare to another frequency.""" + if other is None: + return False + return super().__lt__(other) + + def __gt__(self, other): + """Compare to another frequency.""" + if other is None: + return True + return super().__gt__(other) + + def __hash__(self): + """Hash this tuple.""" + return tuple.__hash__(self) + + @classmethod + def convert(cls, frq): + """Convert `frq` to this type if possible.""" + if isinstance(frq, dict): + return cls(**frq) + return frq + + class FrequencyQuadrupleSideBandBase(NamedTuple): """Base class for a frequency quadruple side band. @@ -47,7 +74,7 @@ class FrequencyQuadrupleSideBandBase(NamedTuple): unit: str = "GHz" -class FrequencyQuadrupleSideBand(FrequencyQuadrupleSideBandBase): +class FrequencyQuadrupleSideBand(FrequencyBandBaseArithmetics, FrequencyQuadrupleSideBandBase): """The frequency quadruple side band class. The elements of the quadruple-side-band type frequency band are the @@ -88,22 +115,6 @@ def __eq__(self, other): return other in self return super().__eq__(other) - def __lt__(self, other): - """Compare to another frequency.""" - if other is None: - return False - return super().__lt__(other) - - def __gt__(self, other): - """Compare to another frequency.""" - if other is None: - return True - return super().__gt__(other) - - def __hash__(self): - """Hash this tuple.""" - return tuple.__hash__(self) - def __str__(self): """Format for print out.""" return f"central={self.central} {self.unit} ±{self.side} ±{self.sideside} width={self.bandwidth} {self.unit}" @@ -175,13 +186,6 @@ def distance(self, value): else: return np.inf - @classmethod - def convert(cls, frq): - """Convert `frq` to this type if possible.""" - if isinstance(frq, dict): - return cls(**frq) - return frq - class FrequencyDoubleSideBandBase(NamedTuple): """Base class for a frequency double side band. @@ -202,7 +206,7 @@ class FrequencyDoubleSideBandBase(NamedTuple): unit: str = "GHz" -class FrequencyDoubleSideBand(FrequencyDoubleSideBandBase): +class FrequencyDoubleSideBand(FrequencyBandBaseArithmetics, FrequencyDoubleSideBandBase): """The frequency double side band class. The elements of the double-side-band type frequency band are the central @@ -239,22 +243,6 @@ def __eq__(self, other): return other in self return super().__eq__(other) - def __lt__(self, other): - """Compare to another frequency.""" - if other is None: - return False - return super().__lt__(other) - - def __gt__(self, other): - """Compare to another frequency.""" - if other is None: - return True - return super().__gt__(other) - - def __hash__(self): - """Hash this tuple.""" - return tuple.__hash__(self) - def __str__(self): """Format for print out.""" return f"central={self.central} {self.unit} ±{self.side} width={self.bandwidth} {self.unit}" @@ -337,13 +325,6 @@ def distance(self, value): else: return np.inf - @classmethod - def convert(cls, frq): - """Convert `frq` to this type if possible.""" - if isinstance(frq, dict): - return cls(**frq) - return frq - class FrequencyRangeBase(NamedTuple): """Base class for frequency ranges. @@ -356,7 +337,7 @@ class FrequencyRangeBase(NamedTuple): unit: str = "GHz" -class FrequencyRange(FrequencyRangeBase): +class FrequencyRange(FrequencyBandBaseArithmetics, FrequencyRangeBase): """The Frequency range class. The elements of the range are central and bandwidth values, and optionally @@ -387,22 +368,6 @@ def __eq__(self, other): return self[:2] == other return super().__eq__(other) - def __lt__(self, other): - """Compare to another frequency.""" - if other is None: - return False - return super().__lt__(other) - - def __gt__(self, other): - """Compare to another frequency.""" - if other is None: - return True - return super().__gt__(other) - - def __hash__(self): - """Hash this tuple.""" - return tuple.__hash__(self) - def __str__(self): """Format for print out.""" return f"central={self.central} {self.unit} width={self.bandwidth} {self.unit}" @@ -433,13 +398,6 @@ def distance(self, value): else: return np.inf - @classmethod - def convert(cls, frq): - """Convert `frq` to this type if possible.""" - if isinstance(frq, dict): - return cls(**frq) - return frq - def _is_inside_interval(value, central, width): - return (central - width/2 <= value <= central + width/2) + return central - width/2 <= value <= central + width/2 From b54129fda4c3da3b7ba8fc19faee237fffc0a69a Mon Sep 17 00:00:00 2001 From: "Adam.Dybbroe" Date: Wed, 31 Aug 2022 16:23:31 +0200 Subject: [PATCH 22/25] Move back the hash'es to the child classes Signed-off-by: Adam.Dybbroe --- satpy/readers/pmw_channels_definitions.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/satpy/readers/pmw_channels_definitions.py b/satpy/readers/pmw_channels_definitions.py index ab57965d1b..3c347e2a82 100644 --- a/satpy/readers/pmw_channels_definitions.py +++ b/satpy/readers/pmw_channels_definitions.py @@ -40,10 +40,6 @@ def __gt__(self, other): return True return super().__gt__(other) - def __hash__(self): - """Hash this tuple.""" - return tuple.__hash__(self) - @classmethod def convert(cls, frq): """Convert `frq` to this type if possible.""" @@ -119,6 +115,10 @@ def __str__(self): """Format for print out.""" return f"central={self.central} {self.unit} ±{self.side} ±{self.sideside} width={self.bandwidth} {self.unit}" + def __hash__(self): + """Hash this tuple.""" + return tuple.__hash__(self) + def __contains__(self, other): """Check if this quadruple-side-band 'contains' *other*.""" if other is None: @@ -247,6 +247,10 @@ def __str__(self): """Format for print out.""" return f"central={self.central} {self.unit} ±{self.side} width={self.bandwidth} {self.unit}" + def __hash__(self): + """Hash this tuple.""" + return tuple.__hash__(self) + def __contains__(self, other): """Check if this double-side-band 'contains' *other*.""" if other is None: @@ -372,6 +376,10 @@ def __str__(self): """Format for print out.""" return f"central={self.central} {self.unit} width={self.bandwidth} {self.unit}" + def __hash__(self): + """Hash this tuple.""" + return tuple.__hash__(self) + def __contains__(self, other): """Check if this range contains *other*.""" if other is None: From afd235801bade1661df49e8665ccb19548342ef0 Mon Sep 17 00:00:00 2001 From: "Adam.Dybbroe" Date: Fri, 2 Sep 2022 11:07:01 +0200 Subject: [PATCH 23/25] Remove redundant code and add test cases and coverage Signed-off-by: Adam.Dybbroe --- satpy/readers/pmw_channels_definitions.py | 4 ---- satpy/tests/test_dataset.py | 10 ++++++++++ 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/satpy/readers/pmw_channels_definitions.py b/satpy/readers/pmw_channels_definitions.py index 3c347e2a82..f1c10b8459 100644 --- a/satpy/readers/pmw_channels_definitions.py +++ b/satpy/readers/pmw_channels_definitions.py @@ -175,10 +175,6 @@ def distance(self, value): left_side_dist = abs(value.central - value.side - value.sideside - left_left) right_side_dist = abs(value.central + value.side + value.sideside - right_right) except AttributeError: - if isinstance(value, (tuple, list)): - msg = 'Distance to a quadruple side band frequency not supported for this type' - raise NotImplementedError(msg) - left_side_dist = abs(value - left_left) right_side_dist = abs(value - right_right) diff --git a/satpy/tests/test_dataset.py b/satpy/tests/test_dataset.py index 059865edde..b79c5b53e4 100644 --- a/satpy/tests/test_dataset.py +++ b/satpy/tests/test_dataset.py @@ -735,9 +735,19 @@ def test_frequency_quadruple_side_band_channel_equality(): def test_frequency_quadruple_side_band_channel_distances(): """Test the frequency quadruple side band object: get the distance between two bands.""" frq_qdsb = FrequencyQuadrupleSideBand(57, 0.322, 0.05, 0.036) + mydist = frq_qdsb.distance([57, 0.322, 0.05, 0.036]) + + frq_dict = {'central': 57, 'side': 0.322, 'sideside': 0.05, + 'bandwidth': 0.036, 'unit': 'GHz'} + mydist = frq_qdsb.distance(frq_dict) + assert mydist == np.inf + mydist = frq_qdsb.distance(57.372) assert mydist == 0.0 + mydist = frq_qdsb.distance(FrequencyQuadrupleSideBand(57, 0.322, 0.05, 0.036)) + assert mydist == 0.0 + mydist = frq_qdsb.distance(57.38) np.testing.assert_almost_equal(mydist, 0.008) From 6183e1ace70392a64588fd05bb1da68c88657ab1 Mon Sep 17 00:00:00 2001 From: Adam Dybbroe Date: Fri, 2 Sep 2022 11:55:49 +0200 Subject: [PATCH 24/25] Update satpy/readers/mws_l1b.py Simplify - reuse already created list Co-authored-by: Martin Raspaud --- satpy/readers/mws_l1b.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/satpy/readers/mws_l1b.py b/satpy/readers/mws_l1b.py index affb5e1538..5a8f94962e 100644 --- a/satpy/readers/mws_l1b.py +++ b/satpy/readers/mws_l1b.py @@ -52,7 +52,7 @@ '21': 21, '22': 22, '23': 23, '24': 24} MWS_CHANNEL_NAMES = list(MWS_CHANNEL_NAMES_TO_NUMBER.keys()) -MWS_CHANNELS = set(MWS_CHANNEL_NAMES_TO_NUMBER.keys()) +MWS_CHANNELS = set(MWS_CHANNEL_NAMES) def get_channel_index_from_name(chname): From d03fc0a32fa2f6039a1fbac85c6ea60bf945d693 Mon Sep 17 00:00:00 2001 From: "Adam.Dybbroe" Date: Mon, 5 Sep 2022 13:20:01 +0200 Subject: [PATCH 25/25] Add test coverage and test scaling the data upon reading Signed-off-by: Adam.Dybbroe --- satpy/readers/mws_l1b.py | 1 + satpy/tests/reader_tests/test_mws_l1b_nc.py | 30 ++++++++++++++++++--- 2 files changed, 28 insertions(+), 3 deletions(-) diff --git a/satpy/readers/mws_l1b.py b/satpy/readers/mws_l1b.py index affb5e1538..517dd51866 100644 --- a/satpy/readers/mws_l1b.py +++ b/satpy/readers/mws_l1b.py @@ -256,6 +256,7 @@ def _get_dataset_aux_data(self, dsname): missing_value = variable.attrs['missing_value'] variable.data = da.where(variable.data == missing_value, np.nan, variable.data * variable.attrs['scale_factor'] + variable.attrs['add_offset']) + return variable def _get_global_attributes(self): diff --git a/satpy/tests/reader_tests/test_mws_l1b_nc.py b/satpy/tests/reader_tests/test_mws_l1b_nc.py index b171a5f8bb..bca79df8ad 100644 --- a/satpy/tests/reader_tests/test_mws_l1b_nc.py +++ b/satpy/tests/reader_tests/test_mws_l1b_nc.py @@ -143,22 +143,27 @@ def _write_navigation_data_group(dataset): shape = (N_SCANS, N_FOVS) longitude = group.createVariable( 'mws_lon', - np.float32, + np.int32, dimensions=dimensions, ) - longitude[:] = np.ones(shape) + longitude.scale_factor = 1.0E-4 + longitude.add_offset = 0.0 + longitude.missing_value = np.array((-2147483648), np.int32) + longitude[:] = 35.7535 * np.ones(shape) + latitude = group.createVariable( 'mws_lat', np.float32, dimensions=dimensions, ) latitude[:] = 2. * np.ones(shape) + azimuth = group.createVariable( 'mws_solar_azimuth_angle', np.float32, dimensions=dimensions, ) - azimuth[:] = 3. * np.ones(shape) + azimuth[:] = 179. * np.ones(shape) @staticmethod def _create_scan_dimensions(dataset): @@ -177,6 +182,9 @@ def _write_calibration_data_group(dataset): toa_bt = group.createVariable( 'mws_toa_brightness_temperature', np.float32, dimensions=('n_scans', 'n_fovs', 'n_channels',) ) + toa_bt.scale_factor = 1.0 # 1.0E-8 + toa_bt.add_offset = 0.0 + toa_bt.missing_value = -2147483648 toa_bt[:] = 240.0 * np.ones((N_SCANS, N_FOVS, N_CHANNELS)) @staticmethod @@ -260,6 +268,22 @@ def test_get_dataset_return_none_if_data_not_exist(self, reader): dataset = reader.get_dataset(dataset_id, dataset_info) assert dataset is None + def test_get_navigation_longitudes(self, caplog, fake_file, reader): + """Test get the longitudes.""" + dataset_id = {'name': 'mws_lon'} + dataset_info = {'file_key': 'data/navigation_data/mws_lon'} + + dataset = reader.get_dataset(dataset_id, dataset_info) + + expected_lons = np.array([[35.753498, 35.753498, 35.753498, 35.753498, 35.753498], + [35.753498, 35.753498, 35.753498, 35.753498, 35.753498], + [35.753498, 35.753498, 35.753498, 35.753498, 35.753498], + [35.753498, 35.753498, 35.753498, 35.753498, 35.753498], + [35.753498, 35.753498, 35.753498, 35.753498, 35.753498]], dtype=np.float32) + + longitudes = dataset[0:5, 0:5].data.compute() + np.testing.assert_allclose(longitudes, expected_lons) + def test_get_dataset_logs_debug_message(self, caplog, fake_file, reader): """Test get dataset return none if data does not exist.""" dataset_id = {'name': 'mws_lon'}