From 826122a2dd67b4332b28bfcdf945c6dc7e9cc565 Mon Sep 17 00:00:00 2001 From: Joleen Feltz Date: Sun, 21 Mar 2021 09:47:15 -0500 Subject: [PATCH 01/14] Add reader_kwarg to omit limb correction on ATMS sensors. --- satpy/readers/mirs.py | 31 ++++++++++++++++++++++++------- 1 file changed, 24 insertions(+), 7 deletions(-) diff --git a/satpy/readers/mirs.py b/satpy/readers/mirs.py index 10934e1281..d2d4a88531 100644 --- a/satpy/readers/mirs.py +++ b/satpy/readers/mirs.py @@ -190,13 +190,29 @@ def limb_correct_atms_bt(bt_data, surf_type_mask, coeff_fns, ds_info): class MiRSL2ncHandler(BaseFileHandler): - """MiRS handler for NetCDF4 files using xarray.""" + """MiRS handler for NetCDF4 files using xarray. - def __init__(self, filename, filename_info, filetype_info): + The MiRS retrieval algorithm runs on multiple + sensors. For the ATMS sensors, a limb correction + is applied by default. In order to change that + behavior, use the keyword argument limb_correction=False + + + from satpy import Scene, find_files_and_readers + + filenames = find_files_and_readers(base_dir, reader="mirs") + scene = satpy.Scene(filenames, + reader_kwargs={'limb_correction': False}) + + """ + + def __init__(self, filename, filename_info, filetype_info, + limb_correction=True): """Init method.""" super(MiRSL2ncHandler, self).__init__(filename, filename_info, - filetype_info) + filetype_info, + ) self.nc = xr.open_dataset(self.filename, decode_cf=True, @@ -212,6 +228,7 @@ def __init__(self, filename, filename_info, filetype_info): self.platform_name = self._get_platform_name self.sensor = self._get_sensor + self.limb_correction = limb_correction @property def platform_shortname(self): @@ -346,15 +363,15 @@ def get_dataset(self, ds_id, ds_info): data = self['BT'] data = data.rename(new_name_or_name_dict=ds_info["name"]) - if self.sensor.lower() != "atms": - LOG.info("Limb Correction will not be applied to non-ATMS BTs") - data = data[:, :, idx] - else: + if self.sensor.lower() == "atms" and self.limb_correction: sfc_type_mask = self['Sfc_type'] data = limb_correct_atms_bt(data, sfc_type_mask, self._get_coeff_filenames, ds_info) self.nc = self.nc.merge(data) + else: + LOG.info("No Limb Correction applied.") + data = data[:, :, idx] else: data = self[ds_id['name']] From 79218d767af1699ad02a28c18d367fc8a5e5492f Mon Sep 17 00:00:00 2001 From: Joleen Feltz Date: Mon, 22 Mar 2021 05:31:29 -0500 Subject: [PATCH 02/14] Fix docstring in mirs reader class --- satpy/readers/mirs.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/satpy/readers/mirs.py b/satpy/readers/mirs.py index d2d4a88531..ea11c7c4a9 100644 --- a/satpy/readers/mirs.py +++ b/satpy/readers/mirs.py @@ -198,11 +198,10 @@ class MiRSL2ncHandler(BaseFileHandler): behavior, use the keyword argument limb_correction=False - from satpy import Scene, find_files_and_readers + from satpy import Scene, find_files_and_readers - filenames = find_files_and_readers(base_dir, reader="mirs") - scene = satpy.Scene(filenames, - reader_kwargs={'limb_correction': False}) + filenames = find_files_and_readers(base_dir, reader="mirs") + scene = Scene(filenames, reader_kwargs={'limb_correction': False}) """ From 1cdbc0c3d0b794708523d813a666987c8a3ffe73 Mon Sep 17 00:00:00 2001 From: Joleen Feltz Date: Mon, 22 Mar 2021 16:12:07 -0500 Subject: [PATCH 03/14] add check for calling limb correction only when sensor is atms --- satpy/tests/reader_tests/test_mirs.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/satpy/tests/reader_tests/test_mirs.py b/satpy/tests/reader_tests/test_mirs.py index bca09d3f90..1bc70e7918 100644 --- a/satpy/tests/reader_tests/test_mirs.py +++ b/satpy/tests/reader_tests/test_mirs.py @@ -82,7 +82,7 @@ def fake_coeff_from_fn(fn): return coeff_str -def _get_datasets_with_attributes(): +def _get_datasets_with_attributes(**kwargs): """Represent files with two resolution of variables in them (ex. OCEAN).""" bt = xr.DataArray(np.linspace(1830, 3930, N_SCANLINE * N_FOV * N_CHANNEL). reshape(N_SCANLINE, N_FOV, N_CHANNEL), @@ -91,7 +91,8 @@ def _get_datasets_with_attributes(): 'coordinates': "Longitude Latitude Freq", 'scale_factor': 0.01, '_FillValue': -999, - 'valid_range': [0, 50000]}) + 'valid_range': [0, 50000]}, + dims=('Scanline', 'Field_of_view', 'Channel')) rr = xr.DataArray(np.random.randint(100, 500, size=(N_SCANLINE, N_FOV)), attrs={'long_name': "Rain Rate (mm/hr)", 'units': "mm/hr", @@ -275,8 +276,14 @@ def test_basic_load(self, filenames, loadable_ids, platform_name): fd.side_effect = fake_coeff_from_fn loaded_data_arrs = r.load(loadable_ids) assert loaded_data_arrs + for data_id, data_arr in loaded_data_arrs.items(): if data_id['name'] not in ['latitude', 'longitude']: self._check_area(data_arr) self._check_fill(data_arr) self._check_attrs(data_arr, platform_name) + + if data_arr.attrs['sensor'] == 'atms': + fd.assert_called() + else: + fd.assert_not_called() From e783a8c8a7d554ce6a480a4490a84ad3d28657b9 Mon Sep 17 00:00:00 2001 From: Joleen Feltz Date: Sat, 27 Mar 2021 01:03:59 -0500 Subject: [PATCH 04/14] Fix docstring --- satpy/readers/mirs.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/satpy/readers/mirs.py b/satpy/readers/mirs.py index ea11c7c4a9..a8dab41b26 100644 --- a/satpy/readers/mirs.py +++ b/satpy/readers/mirs.py @@ -195,13 +195,13 @@ class MiRSL2ncHandler(BaseFileHandler): The MiRS retrieval algorithm runs on multiple sensors. For the ATMS sensors, a limb correction is applied by default. In order to change that - behavior, use the keyword argument limb_correction=False + behavior, use the keyword argument ``limb_correction=False``:: - from satpy import Scene, find_files_and_readers + from satpy import Scene, find_files_and_readers - filenames = find_files_and_readers(base_dir, reader="mirs") - scene = Scene(filenames, reader_kwargs={'limb_correction': False}) + filenames = find_files_and_readers(base_dir, reader="mirs") + scene = Scene(filenames, reader_kwargs={'limb_correction': False}) """ From fc610c8442c0e23c30c34a97285694ca02b44337 Mon Sep 17 00:00:00 2001 From: Joleen Feltz Date: Sat, 27 Mar 2021 10:22:21 -0500 Subject: [PATCH 05/14] Add a kwarg test for limb_correction. --- satpy/tests/reader_tests/test_mirs.py | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/satpy/tests/reader_tests/test_mirs.py b/satpy/tests/reader_tests/test_mirs.py index 1bc70e7918..0ec614845b 100644 --- a/satpy/tests/reader_tests/test_mirs.py +++ b/satpy/tests/reader_tests/test_mirs.py @@ -287,3 +287,28 @@ def test_basic_load(self, filenames, loadable_ids, platform_name): fd.assert_called() else: fd.assert_not_called() + + @pytest.mark.parametrize( + ("filenames", "loadable_ids", "platform_name"), + [ + ([AWIPS_FILE], TEST_VARS, "metop-a"), + ([NPP_MIRS_L2_SWATH], TEST_VARS, "npp"), + ([OTHER_MIRS_L2_SWATH], TEST_VARS, "gpm"), + ] + ) + def test_kwarg_load(self, filenames, loadable_ids, platform_name): + """Test the limb_correction kwarg when filehandler is loaded.""" + from satpy.readers import load_reader + with mock.patch('satpy.readers.mirs.xr.open_dataset') as od: + od.side_effect = fake_open_dataset + r = load_reader(self.reader_configs) + loadables = r.select_files_from_pathnames(filenames) + r.create_filehandlers(loadables, fh_kwargs={"limb_correction": False}) + + with mock.patch('satpy.readers.mirs.read_atms_coeff_to_string') as \ + fd, mock.patch('satpy.readers.mirs.retrieve'): + fd.side_effect = fake_coeff_from_fn + loaded_data_arrs = r.load(loadable_ids) + assert loaded_data_arrs + + fd.assert_not_called() From fdba5febe0c7ea13160dd2d94eacb14e748dd039 Mon Sep 17 00:00:00 2001 From: Joleen Feltz Date: Sat, 27 Mar 2021 10:53:07 -0500 Subject: [PATCH 06/14] Remove unused variables and add a noaa-20 test Remove platform name from parameterize in last test for kwargs because they are not used in that test. Add a test for the NOAA-20 platform when the limb_correction assertions are tested and platform name is tested. --- satpy/tests/reader_tests/test_mirs.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/satpy/tests/reader_tests/test_mirs.py b/satpy/tests/reader_tests/test_mirs.py index 0ec614845b..7618c143b7 100644 --- a/satpy/tests/reader_tests/test_mirs.py +++ b/satpy/tests/reader_tests/test_mirs.py @@ -27,6 +27,7 @@ AWIPS_FILE = "IMG_SX.M2.D17037.S1601.E1607.B0000001.WE.HR.ORB.nc" NPP_MIRS_L2_SWATH = "NPR-MIRS-IMG_v11r6_npp_s201702061601000_e201702061607000_c202012201658410.nc" +N20_MIRS_L2_SWATH = "NPR-MIRS-IMG_v11r4_n20_s201702061601000_e201702061607000_c202012201658410.nc" OTHER_MIRS_L2_SWATH = "NPR-MIRS-IMG_v11r4_gpm_s201702061601000_e201702061607000_c202010080001310.nc" EXAMPLE_FILES = [AWIPS_FILE, NPP_MIRS_L2_SWATH, OTHER_MIRS_L2_SWATH] @@ -260,6 +261,7 @@ def _check_attrs(data_arr, platform_name): [ ([AWIPS_FILE], TEST_VARS, "metop-a"), ([NPP_MIRS_L2_SWATH], TEST_VARS, "npp"), + ([N20_MIRS_L2_SWATH], TEST_VARS, "noaa-20"), ([OTHER_MIRS_L2_SWATH], TEST_VARS, "gpm"), ] ) @@ -289,14 +291,15 @@ def test_basic_load(self, filenames, loadable_ids, platform_name): fd.assert_not_called() @pytest.mark.parametrize( - ("filenames", "loadable_ids", "platform_name"), + ("filenames", "loadable_ids"), [ - ([AWIPS_FILE], TEST_VARS, "metop-a"), - ([NPP_MIRS_L2_SWATH], TEST_VARS, "npp"), - ([OTHER_MIRS_L2_SWATH], TEST_VARS, "gpm"), + ([AWIPS_FILE], TEST_VARS), + ([NPP_MIRS_L2_SWATH], TEST_VARS), + ([N20_MIRS_L2_SWATH], TEST_VARS), + ([OTHER_MIRS_L2_SWATH], TEST_VARS), ] ) - def test_kwarg_load(self, filenames, loadable_ids, platform_name): + def test_kwarg_load(self, filenames, loadable_ids): """Test the limb_correction kwarg when filehandler is loaded.""" from satpy.readers import load_reader with mock.patch('satpy.readers.mirs.xr.open_dataset') as od: From a66e3bc63323dff2b5ac237fcb90d9953a27e8fd Mon Sep 17 00:00:00 2001 From: Joleen Feltz Date: Sun, 28 Mar 2021 09:01:13 -0500 Subject: [PATCH 07/14] Fold reader_kwarg test into basic_load --- satpy/tests/reader_tests/test_mirs.py | 56 ++++++++++----------------- 1 file changed, 20 insertions(+), 36 deletions(-) diff --git a/satpy/tests/reader_tests/test_mirs.py b/satpy/tests/reader_tests/test_mirs.py index 7618c143b7..58f89112a2 100644 --- a/satpy/tests/reader_tests/test_mirs.py +++ b/satpy/tests/reader_tests/test_mirs.py @@ -257,22 +257,28 @@ def _check_attrs(data_arr, platform_name): assert attrs['end_time'] == END_TIME @pytest.mark.parametrize( - ("filenames", "loadable_ids", "platform_name"), + ("filenames", "loadable_ids", "platform_name", "reader_kw"), [ - ([AWIPS_FILE], TEST_VARS, "metop-a"), - ([NPP_MIRS_L2_SWATH], TEST_VARS, "npp"), - ([N20_MIRS_L2_SWATH], TEST_VARS, "noaa-20"), - ([OTHER_MIRS_L2_SWATH], TEST_VARS, "gpm"), + ([AWIPS_FILE], TEST_VARS, "metop-a", None), + ([NPP_MIRS_L2_SWATH], TEST_VARS, "npp", None), + ([N20_MIRS_L2_SWATH], TEST_VARS, "noaa-20", None), + ([OTHER_MIRS_L2_SWATH], TEST_VARS, "gpm", None), + + ([AWIPS_FILE], TEST_VARS, "metop-a", {"limb_correction": False}), + ([NPP_MIRS_L2_SWATH], TEST_VARS, "npp", {"limb_correction": False}), + ([N20_MIRS_L2_SWATH], TEST_VARS, "noaa-20", {"limb_correction": False}), + ([OTHER_MIRS_L2_SWATH], TEST_VARS, "gpm", {"limb_correction": False}), ] ) - def test_basic_load(self, filenames, loadable_ids, platform_name): + def test_basic_load(self, filenames, loadable_ids, + platform_name, reader_kw): """Test that variables are loaded properly.""" from satpy.readers import load_reader with mock.patch('satpy.readers.mirs.xr.open_dataset') as od: od.side_effect = fake_open_dataset r = load_reader(self.reader_configs) loadables = r.select_files_from_pathnames(filenames) - r.create_filehandlers(loadables) + r.create_filehandlers(loadables, fh_kwargs=reader_kw) with mock.patch('satpy.readers.mirs.read_atms_coeff_to_string') as \ fd, mock.patch('satpy.readers.mirs.retrieve'): fd.side_effect = fake_coeff_from_fn @@ -285,33 +291,11 @@ def test_basic_load(self, filenames, loadable_ids, platform_name): self._check_fill(data_arr) self._check_attrs(data_arr, platform_name) - if data_arr.attrs['sensor'] == 'atms': - fd.assert_called() - else: + if reader_kw and not reader_kw['limb_correction']: fd.assert_not_called() - - @pytest.mark.parametrize( - ("filenames", "loadable_ids"), - [ - ([AWIPS_FILE], TEST_VARS), - ([NPP_MIRS_L2_SWATH], TEST_VARS), - ([N20_MIRS_L2_SWATH], TEST_VARS), - ([OTHER_MIRS_L2_SWATH], TEST_VARS), - ] - ) - def test_kwarg_load(self, filenames, loadable_ids): - """Test the limb_correction kwarg when filehandler is loaded.""" - from satpy.readers import load_reader - with mock.patch('satpy.readers.mirs.xr.open_dataset') as od: - od.side_effect = fake_open_dataset - r = load_reader(self.reader_configs) - loadables = r.select_files_from_pathnames(filenames) - r.create_filehandlers(loadables, fh_kwargs={"limb_correction": False}) - - with mock.patch('satpy.readers.mirs.read_atms_coeff_to_string') as \ - fd, mock.patch('satpy.readers.mirs.retrieve'): - fd.side_effect = fake_coeff_from_fn - loaded_data_arrs = r.load(loadable_ids) - assert loaded_data_arrs - - fd.assert_not_called() + else: + sensor = data_arr.attrs['sensor'] + if sensor == 'atms': + fd.assert_called() + else: + fd.assert_not_called() From 6132c67beb6af3eeadf7be7c62cff98111adaea3 Mon Sep 17 00:00:00 2001 From: Joleen Feltz Date: Tue, 30 Mar 2021 07:05:48 -0500 Subject: [PATCH 08/14] Simply if/then statement, split parameterize for readability. --- satpy/tests/reader_tests/test_mirs.py | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/satpy/tests/reader_tests/test_mirs.py b/satpy/tests/reader_tests/test_mirs.py index 58f89112a2..899c63a098 100644 --- a/satpy/tests/reader_tests/test_mirs.py +++ b/satpy/tests/reader_tests/test_mirs.py @@ -257,19 +257,20 @@ def _check_attrs(data_arr, platform_name): assert attrs['end_time'] == END_TIME @pytest.mark.parametrize( - ("filenames", "loadable_ids", "platform_name", "reader_kw"), + ("filenames", "loadable_ids", "platform_name"), [ - ([AWIPS_FILE], TEST_VARS, "metop-a", None), - ([NPP_MIRS_L2_SWATH], TEST_VARS, "npp", None), - ([N20_MIRS_L2_SWATH], TEST_VARS, "noaa-20", None), - ([OTHER_MIRS_L2_SWATH], TEST_VARS, "gpm", None), - - ([AWIPS_FILE], TEST_VARS, "metop-a", {"limb_correction": False}), - ([NPP_MIRS_L2_SWATH], TEST_VARS, "npp", {"limb_correction": False}), - ([N20_MIRS_L2_SWATH], TEST_VARS, "noaa-20", {"limb_correction": False}), - ([OTHER_MIRS_L2_SWATH], TEST_VARS, "gpm", {"limb_correction": False}), + ([AWIPS_FILE], TEST_VARS, "metop-a"), + ([NPP_MIRS_L2_SWATH], TEST_VARS, "npp"), + ([N20_MIRS_L2_SWATH], TEST_VARS, "noaa-20"), + ([OTHER_MIRS_L2_SWATH], TEST_VARS, "gpm"), + + ([AWIPS_FILE], TEST_VARS, "metop-a"), + ([NPP_MIRS_L2_SWATH], TEST_VARS, "npp"), + ([N20_MIRS_L2_SWATH], TEST_VARS, "noaa-20"), + ([OTHER_MIRS_L2_SWATH], TEST_VARS, "gpm"), ] ) + @pytest.mark.parametrize('reader_kw', [{}, {'limb_correction': False}]) def test_basic_load(self, filenames, loadable_ids, platform_name, reader_kw): """Test that variables are loaded properly.""" @@ -286,6 +287,8 @@ def test_basic_load(self, filenames, loadable_ids, assert loaded_data_arrs for data_id, data_arr in loaded_data_arrs.items(): + sensor = data_arr.attrs['sensor'] + if data_id['name'] not in ['latitude', 'longitude']: self._check_area(data_arr) self._check_fill(data_arr) From f4bdbdab008979f6ae7fa4e76f4ee68f9f8cac2d Mon Sep 17 00:00:00 2001 From: Joleen Feltz Date: Tue, 30 Mar 2021 07:41:56 -0500 Subject: [PATCH 09/14] Simplify assertion if/then statement for limb_correction. --- satpy/tests/reader_tests/test_mirs.py | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/satpy/tests/reader_tests/test_mirs.py b/satpy/tests/reader_tests/test_mirs.py index 899c63a098..c88f720b03 100644 --- a/satpy/tests/reader_tests/test_mirs.py +++ b/satpy/tests/reader_tests/test_mirs.py @@ -294,11 +294,8 @@ def test_basic_load(self, filenames, loadable_ids, self._check_fill(data_arr) self._check_attrs(data_arr, platform_name) - if reader_kw and not reader_kw['limb_correction']: - fd.assert_not_called() + sensor = data_arr.attrs['sensor'] + if reader_kw.get('limb_correction', True) and sensor == 'atms': + fd.assert_called() else: - sensor = data_arr.attrs['sensor'] - if sensor == 'atms': - fd.assert_called() - else: - fd.assert_not_called() + fd.assert_not_called() From b0140d1654413b2bfb8587bf548590fa4c4a0b83 Mon Sep 17 00:00:00 2001 From: Joleen Feltz Date: Tue, 30 Mar 2021 12:36:54 -0500 Subject: [PATCH 10/14] Remove extra line getting the name of the sensor --- satpy/tests/reader_tests/test_mirs.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/satpy/tests/reader_tests/test_mirs.py b/satpy/tests/reader_tests/test_mirs.py index c88f720b03..c995b0b46c 100644 --- a/satpy/tests/reader_tests/test_mirs.py +++ b/satpy/tests/reader_tests/test_mirs.py @@ -287,8 +287,6 @@ def test_basic_load(self, filenames, loadable_ids, assert loaded_data_arrs for data_id, data_arr in loaded_data_arrs.items(): - sensor = data_arr.attrs['sensor'] - if data_id['name'] not in ['latitude', 'longitude']: self._check_area(data_arr) self._check_fill(data_arr) From 9f85942bd911ac4c7a328d4f8304192a6973b9e8 Mon Sep 17 00:00:00 2001 From: Joleen Feltz Date: Fri, 30 Apr 2021 06:14:27 -0500 Subject: [PATCH 11/14] Use valid range when present Fixes a bug in which valid range was ignored. --- satpy/readers/mirs.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/satpy/readers/mirs.py b/satpy/readers/mirs.py index d1efb42420..4b19779e36 100644 --- a/satpy/readers/mirs.py +++ b/satpy/readers/mirs.py @@ -356,6 +356,17 @@ def _fill_data(self, data_arr, attrs): data_arr = data_arr.where(data_arr != fill_value, fill_out) return data_arr, attrs + def _get_valid_range(self, data_arr, attrs): + # handle valid_range + valid_range = attrs.pop('valid_range', None) + if isinstance(valid_range, np.ndarray): + valid_min, valid_max = valid_range + + if valid_min is not None and valid_max is not None: + data_arr = data_arr.where((data_arr >= valid_min) & + (data_arr <= valid_max)) + return data_arr, attrs + def get_dataset(self, ds_id, ds_info): """Get datasets.""" if 'dependencies' in ds_info.keys(): @@ -483,6 +494,7 @@ def __getitem__(self, item): attrs = data.attrs.copy() data, attrs = self._scale_data(data, attrs) data, attrs = self._fill_data(data, attrs) + data, attrs = self._get_valid_range(data, attrs) # 'Freq' dimension causes issues in other processing if 'Freq' in data.coords: From d1a0bb50e8951cc84ceb2901ab8f60eb418680a7 Mon Sep 17 00:00:00 2001 From: Joleen Feltz Date: Fri, 30 Apr 2021 06:49:51 -0500 Subject: [PATCH 12/14] Check to confirm that valid range is no longer in attributes. --- satpy/tests/reader_tests/test_mirs.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/satpy/tests/reader_tests/test_mirs.py b/satpy/tests/reader_tests/test_mirs.py index c995b0b46c..bc111fbb6e 100644 --- a/satpy/tests/reader_tests/test_mirs.py +++ b/satpy/tests/reader_tests/test_mirs.py @@ -247,6 +247,11 @@ def _check_fill(data_arr): # we started with float32, it should stay that way assert data_arr.dtype.type == np.float64 + @staticmethod + def _check_valid_range(data_arr): + # valid_range is popped out of data_arr.attrs when it is applied + assert 'valid_range' not in data_arr.attrs + @staticmethod def _check_attrs(data_arr, platform_name): attrs = data_arr.attrs @@ -290,6 +295,7 @@ def test_basic_load(self, filenames, loadable_ids, if data_id['name'] not in ['latitude', 'longitude']: self._check_area(data_arr) self._check_fill(data_arr) + self._check_valid_range(data_arr) self._check_attrs(data_arr, platform_name) sensor = data_arr.attrs['sensor'] From 582837deb0ffd7258a8806c30121723ed684d6fa Mon Sep 17 00:00:00 2001 From: Joleen Feltz Date: Fri, 30 Apr 2021 13:17:41 -0500 Subject: [PATCH 13/14] Add a test to check valid range was applied correctly. Change get_valid_range=>apply_valid_range and check for case of valid_range not None --- satpy/readers/mirs.py | 6 +++--- satpy/tests/reader_tests/test_mirs.py | 12 ++++++++++-- 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/satpy/readers/mirs.py b/satpy/readers/mirs.py index 4b19779e36..789ff4f07d 100644 --- a/satpy/readers/mirs.py +++ b/satpy/readers/mirs.py @@ -356,10 +356,10 @@ def _fill_data(self, data_arr, attrs): data_arr = data_arr.where(data_arr != fill_value, fill_out) return data_arr, attrs - def _get_valid_range(self, data_arr, attrs): + def _apply_valid_range(self, data_arr, attrs): # handle valid_range valid_range = attrs.pop('valid_range', None) - if isinstance(valid_range, np.ndarray): + if valid_range is not None: valid_min, valid_max = valid_range if valid_min is not None and valid_max is not None: @@ -494,7 +494,7 @@ def __getitem__(self, item): attrs = data.attrs.copy() data, attrs = self._scale_data(data, attrs) data, attrs = self._fill_data(data, attrs) - data, attrs = self._get_valid_range(data, attrs) + data, attrs = self._apply_valid_range(data, attrs) # 'Freq' dimension causes issues in other processing if 'Freq' in data.coords: diff --git a/satpy/tests/reader_tests/test_mirs.py b/satpy/tests/reader_tests/test_mirs.py index bc111fbb6e..aaf10859d8 100644 --- a/satpy/tests/reader_tests/test_mirs.py +++ b/satpy/tests/reader_tests/test_mirs.py @@ -248,9 +248,17 @@ def _check_fill(data_arr): assert data_arr.dtype.type == np.float64 @staticmethod - def _check_valid_range(data_arr): + def _check_valid_range(filename, data_arr): # valid_range is popped out of data_arr.attrs when it is applied assert 'valid_range' not in data_arr.attrs + var_name = data_arr.attrs["name"] + data = fake_open_dataset(filename) + if "btemp" in var_name: + data = data['BT'] + if "valid_range" in data.attrs: + valid_range = data.attrs["valid_range"] + assert data_arr.data.min() > valid_range[0] + assert data_arr.data.max() < valid_range[1] @staticmethod def _check_attrs(data_arr, platform_name): @@ -295,7 +303,7 @@ def test_basic_load(self, filenames, loadable_ids, if data_id['name'] not in ['latitude', 'longitude']: self._check_area(data_arr) self._check_fill(data_arr) - self._check_valid_range(data_arr) + self._check_valid_range(filenames[0], data_arr) self._check_attrs(data_arr, platform_name) sensor = data_arr.attrs['sensor'] From 879fce4bc1ef9f0192b40564164f49f63dd3e7c5 Mon Sep 17 00:00:00 2001 From: Joleen Feltz Date: Fri, 30 Apr 2021 14:47:58 -0500 Subject: [PATCH 14/14] valid_range is inclusive so include both min/max in acceptable values only read fake_data one time, not every time check_valid_range is called. --- satpy/tests/reader_tests/test_mirs.py | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/satpy/tests/reader_tests/test_mirs.py b/satpy/tests/reader_tests/test_mirs.py index aaf10859d8..63d48b37fa 100644 --- a/satpy/tests/reader_tests/test_mirs.py +++ b/satpy/tests/reader_tests/test_mirs.py @@ -248,17 +248,11 @@ def _check_fill(data_arr): assert data_arr.dtype.type == np.float64 @staticmethod - def _check_valid_range(filename, data_arr): + def _check_valid_range(data_arr, test_valid_range): # valid_range is popped out of data_arr.attrs when it is applied assert 'valid_range' not in data_arr.attrs - var_name = data_arr.attrs["name"] - data = fake_open_dataset(filename) - if "btemp" in var_name: - data = data['BT'] - if "valid_range" in data.attrs: - valid_range = data.attrs["valid_range"] - assert data_arr.data.min() > valid_range[0] - assert data_arr.data.max() < valid_range[1] + assert data_arr.data.min() >= test_valid_range[0] + assert data_arr.data.max() <= test_valid_range[1] @staticmethod def _check_attrs(data_arr, platform_name): @@ -299,13 +293,20 @@ def test_basic_load(self, filenames, loadable_ids, loaded_data_arrs = r.load(loadable_ids) assert loaded_data_arrs - for data_id, data_arr in loaded_data_arrs.items(): - if data_id['name'] not in ['latitude', 'longitude']: + test_data = fake_open_dataset(filenames[0]) + for _data_id, data_arr in loaded_data_arrs.items(): + var_name = data_arr.attrs["name"] + if var_name not in ['latitude', 'longitude']: self._check_area(data_arr) self._check_fill(data_arr) - self._check_valid_range(filenames[0], data_arr) self._check_attrs(data_arr, platform_name) + input_fake_data = test_data['BT'] if "btemp" in var_name \ + else test_data[var_name] + if "valid_range" in input_fake_data.attrs: + valid_range = input_fake_data.attrs["valid_range"] + self._check_valid_range(data_arr, valid_range) + sensor = data_arr.attrs['sensor'] if reader_kw.get('limb_correction', True) and sensor == 'atms': fd.assert_called()