From 8b06d5f71f779b830e12b9683b5fb6dcbd8a166d Mon Sep 17 00:00:00 2001 From: Martin Raspaud Date: Tue, 9 Feb 2021 10:52:18 +0100 Subject: [PATCH 1/5] Fix typo --- satpy/readers/satpy_cf_nc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/satpy/readers/satpy_cf_nc.py b/satpy/readers/satpy_cf_nc.py index d9e93955af..17554b8e82 100644 --- a/satpy/readers/satpy_cf_nc.py +++ b/satpy/readers/satpy_cf_nc.py @@ -257,7 +257,7 @@ def _dynamic_datasets(self): yield True, ds_info def _coordinate_datasets(self, configured_datasets=None): - """Add information of coordiante datasets.""" + """Add information of coordinate datasets.""" nc = xr.open_dataset(self.filename, engine=self.engine) for var_name, val in nc.coords.items(): ds_info = dict(val.attrs) From cc6ce4431c50ebbc96c91cd3f7a72c41e8f9c28d Mon Sep 17 00:00:00 2001 From: Martin Raspaud Date: Tue, 9 Feb 2021 10:54:39 +0100 Subject: [PATCH 2/5] Allow using builtin coordinates for swath definition creation --- satpy/readers/yaml_reader.py | 16 +++++++-- satpy/tests/test_yaml_reader.py | 60 +++++++++++++++++++++++++++++++++ 2 files changed, 74 insertions(+), 2 deletions(-) diff --git a/satpy/readers/yaml_reader.py b/satpy/readers/yaml_reader.py index 49e29db15e..6112c2a96c 100644 --- a/satpy/readers/yaml_reader.py +++ b/satpy/readers/yaml_reader.py @@ -823,19 +823,31 @@ def _load_dataset_with_area(self, dsid, coords, **kwargs): if not file_handlers: return - area = self._load_dataset_area(dsid, file_handlers, coords, **kwargs) - try: ds = self._load_dataset_data(file_handlers, dsid, **kwargs) except (KeyError, ValueError) as err: logger.exception("Could not load dataset '%s': %s", dsid, str(err)) return None + coords = self._assign_builtin_coords(coords, ds) + + area = self._load_dataset_area(dsid, file_handlers, coords, **kwargs) + if area is not None: ds.attrs['area'] = area ds = add_crs_xy_coords(ds, area) return ds + @staticmethod + def _assign_builtin_coords(coords, ds): + """Assign builtin coords if needed.""" + if not coords: + coords = [] + for coord in ds.coords.values(): + if coord.attrs['standard_name'] in ['longitude', 'latitude']: + coords.append(coord) + return coords + def _load_ancillary_variables(self, datasets, **kwargs): """Load the ancillary variables of `datasets`.""" all_av_ids = set() diff --git a/satpy/tests/test_yaml_reader.py b/satpy/tests/test_yaml_reader.py index 4364f4e6bc..fc6df18271 100644 --- a/satpy/tests/test_yaml_reader.py +++ b/satpy/tests/test_yaml_reader.py @@ -481,6 +481,66 @@ def test_load_entire_dataset(self, xarray): self.assertIs(proj, xarray.concat.return_value) +class TestFileYAMLReaderLoading(unittest.TestCase): + """Tests for FileYAMLReader.load.""" + + def setUp(self): + """Prepare a reader instance with a fake config.""" + patterns = ['a{something:3s}.bla'] + res_dict = {'reader': {'name': 'fake', + 'sensors': ['canon']}, + 'file_types': {'ftype1': {'name': 'ft1', + 'file_reader': BaseFileHandler, + 'file_patterns': patterns}}, + 'datasets': {'ch1': {'name': 'ch01', + 'wavelength': [0.5, 0.6, 0.7], + 'calibration': 'reflectance', + 'file_type': 'ftype1'}, + }} + + self.config = res_dict + self.reader = yr.FileYAMLReader(res_dict, + filter_parameters={ + 'start_time': datetime(2000, 1, 1), + 'end_time': datetime(2000, 1, 2), + }) + + def test_load_dataset_with_builtin_coords(self): + """Test loading a dataset with builtin coordinates.""" + fake_fh = FakeFH(None, None) + import xarray as xr + import numpy as np + lons = xr.DataArray(np.ones((2, 2)) * 2, + dims=['y', 'x'], + attrs={'standard_name': 'longitude', + 'name': 'longitude'}) + lats = xr.DataArray(np.ones((2, 2)) * 2, + dims=['y', 'x'], + attrs={'standard_name': 'latitude', + 'name': 'latitude'}) + data = xr.DataArray(np.ones((2, 2)), + coords={'longitude': lons, + 'latitude': lats}, + dims=['y', 'x']) + + def _assign_array(dsid, *_args, **_kwargs): + if dsid['name'] == 'longitude': + return lons + elif dsid['name'] == 'latitude': + return lats + else: + return data + + fake_fh.get_dataset.side_effect = _assign_array + self.reader.file_handlers = {'ftype1': [fake_fh]} + + res = self.reader.load(['ch01']) + + assert 'area' in res['ch01'].attrs + np.testing.assert_array_equal(res['ch01'].attrs['area'].lons, lons) + np.testing.assert_array_equal(res['ch01'].attrs['area'].lats, lats) + + class TestFileFileYAMLReaderMultipleFileTypes(unittest.TestCase): """Test units from FileYAMLReader with multiple file types.""" From 403cd349199cfc24af8a723762d06091c6f2a96c Mon Sep 17 00:00:00 2001 From: Martin Raspaud Date: Tue, 9 Feb 2021 13:27:38 +0100 Subject: [PATCH 3/5] Fix failing unit tests --- satpy/readers/yaml_reader.py | 2 +- satpy/tests/test_yaml_reader.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/satpy/readers/yaml_reader.py b/satpy/readers/yaml_reader.py index 6112c2a96c..161a5d8d86 100644 --- a/satpy/readers/yaml_reader.py +++ b/satpy/readers/yaml_reader.py @@ -844,7 +844,7 @@ def _assign_builtin_coords(coords, ds): if not coords: coords = [] for coord in ds.coords.values(): - if coord.attrs['standard_name'] in ['longitude', 'latitude']: + if coord.attrs.get('standard_name') in ['longitude', 'latitude']: coords.append(coord) return coords diff --git a/satpy/tests/test_yaml_reader.py b/satpy/tests/test_yaml_reader.py index fc6df18271..c427f7d0d2 100644 --- a/satpy/tests/test_yaml_reader.py +++ b/satpy/tests/test_yaml_reader.py @@ -528,8 +528,8 @@ def _assign_array(dsid, *_args, **_kwargs): return lons elif dsid['name'] == 'latitude': return lats - else: - return data + + return data fake_fh.get_dataset.side_effect = _assign_array self.reader.file_handlers = {'ftype1': [fake_fh]} From 706a2da07df3e268f936fa835d34154220ce0bc9 Mon Sep 17 00:00:00 2001 From: Martin Raspaud Date: Thu, 11 Feb 2021 20:15:05 +0100 Subject: [PATCH 4/5] Fix satpy nc reader test --- satpy/readers/yaml_reader.py | 58 ++++++++++++-------- satpy/tests/reader_tests/test_satpy_cf_nc.py | 38 +++++++------ satpy/tests/test_yaml_reader.py | 58 ++++++++++++-------- 3 files changed, 90 insertions(+), 64 deletions(-) diff --git a/satpy/readers/yaml_reader.py b/satpy/readers/yaml_reader.py index 161a5d8d86..5b42127258 100644 --- a/satpy/readers/yaml_reader.py +++ b/satpy/readers/yaml_reader.py @@ -775,33 +775,45 @@ def _get_file_handlers(self, dsid): def _make_area_from_coords(self, coords): """Create an appropriate area with the given *coords*.""" if len(coords) == 2: - lon_sn = coords[0].attrs.get('standard_name') - lat_sn = coords[1].attrs.get('standard_name') - if lon_sn == 'longitude' and lat_sn == 'latitude': - key = None - try: - key = (coords[0].data.name, coords[1].data.name) - sdef = self.coords_cache.get(key) - except AttributeError: - sdef = None - if sdef is None: - sdef = SwathDefinition(*coords) - sensor_str = '_'.join(self.info['sensors']) - shape_str = '_'.join(map(str, coords[0].shape)) - sdef.name = "{}_{}_{}_{}".format(sensor_str, shape_str, - coords[0].attrs['name'], - coords[1].attrs['name']) - if key is not None: - self.coords_cache[key] = sdef - return sdef - else: - raise ValueError( - 'Coordinates info object missing standard_name key: ' + - str(coords)) + lats, lons = self._get_lons_lats_from_coords(coords) + + sdef = self._make_swath_definition_from_lons_lats(lons, lats) + return sdef elif len(coords) != 0: raise NameError("Don't know what to do with coordinates " + str( coords)) + def _get_lons_lats_from_coords(self, coords): + """Get lons and lats from the coords list.""" + lons, lats = None, None + for coord in coords: + if coord.attrs.get('standard_name') == 'longitude': + lons = coord + elif coord.attrs.get('standard_name') == 'latitude': + lats = coord + if lons is None or lats is None: + raise ValueError('Missing longitude or latitude coordinate: ' + str(coords)) + return lats, lons + + def _make_swath_definition_from_lons_lats(self, lons, lats): + """Make a swath definition instance from lons and lats.""" + key = None + try: + key = (lons.data.name, lats.data.name) + sdef = self.coords_cache.get(key) + except AttributeError: + sdef = None + if sdef is None: + sdef = SwathDefinition(lons, lats) + sensor_str = '_'.join(self.info['sensors']) + shape_str = '_'.join(map(str, lons.shape)) + sdef.name = "{}_{}_{}_{}".format(sensor_str, shape_str, + lons.attrs.get('name', lons.name), + lats.attrs.get('name', lats.name)) + if key is not None: + self.coords_cache[key] = sdef + return sdef + def _load_dataset_area(self, dsid, file_handlers, coords, **kwargs): """Get the area for *dsid*.""" try: diff --git a/satpy/tests/reader_tests/test_satpy_cf_nc.py b/satpy/tests/reader_tests/test_satpy_cf_nc.py index 15f6434711..747954e246 100644 --- a/satpy/tests/reader_tests/test_satpy_cf_nc.py +++ b/satpy/tests/reader_tests/test_satpy_cf_nc.py @@ -17,11 +17,14 @@ # satpy. If not, see . """Tests for the CF reader.""" -import unittest import os +import unittest +from contextlib import suppress from datetime import datetime -import xarray as xr + import numpy as np +import xarray as xr + from satpy import Scene from satpy.readers.satpy_cf_nc import SatpyCFFileHandler @@ -85,25 +88,24 @@ def setUp(self): def test_write_and_read(self): """Save a file with cf_writer and read the data again.""" - # '{testin}-{sensor}-{start_time:%Y%m%d%H%M%S}-{end_time:%Y%m%d%H%M%S}.nc' filename = 'testingcfwriter{:s}-viirs-mband-20201007075915-20201007080744.nc'.format( datetime.utcnow().strftime('%Y%j%H%M%S')) - self.scene.save_datasets(writer='cf', - filename=filename, - header_attrs={'instrument': 'avhrr'}, - engine='h5netcdf', - flatten_attrs=True, - pretty=True) - scn_ = Scene(reader='satpy_cf_nc', - filenames=[filename]) - scn_.load(['image0', 'image1', 'lat']) - self.assertTrue(np.all(scn_['image0'].data == self.scene['image0'].data)) - self.assertTrue(np.all(scn_['lat'].data == self.scene['lat'].data)) # lat loaded as dataset - self.assertTrue(np.all(scn_['image0'].coords['lon'] == self.scene['lon'].data)) # lon loded as coord try: - os.remove(filename) - except PermissionError: - pass + self.scene.save_datasets(writer='cf', + filename=filename, + header_attrs={'instrument': 'avhrr'}, + engine='h5netcdf', + flatten_attrs=True, + pretty=True) + scn_ = Scene(reader='satpy_cf_nc', + filenames=[filename]) + scn_.load(['image0', 'image1', 'lat']) + self.assertTrue(np.all(scn_['image0'].data == self.scene['image0'].data)) + self.assertTrue(np.all(scn_['lat'].data == self.scene['lat'].data)) # lat loaded as dataset + self.assertTrue(np.all(scn_['image0'].coords['lon'] == self.scene['lon'].data)) # lon loded as coord + finally: + with suppress(PermissionError): + os.remove(filename) def test_fix_modifier_attr(self): """Check that fix modifier can handle empty list as modifier attribute.""" diff --git a/satpy/tests/test_yaml_reader.py b/satpy/tests/test_yaml_reader.py index c427f7d0d2..46d871f1b1 100644 --- a/satpy/tests/test_yaml_reader.py +++ b/satpy/tests/test_yaml_reader.py @@ -28,6 +28,8 @@ from satpy.readers.file_handlers import BaseFileHandler from satpy.dataset import DataQuery from satpy.tests.utils import make_dataid +import xarray as xr +import numpy as np class FakeFH(BaseFileHandler): @@ -504,41 +506,51 @@ def setUp(self): 'start_time': datetime(2000, 1, 1), 'end_time': datetime(2000, 1, 2), }) - - def test_load_dataset_with_builtin_coords(self): - """Test loading a dataset with builtin coordinates.""" fake_fh = FakeFH(None, None) - import xarray as xr - import numpy as np - lons = xr.DataArray(np.ones((2, 2)) * 2, - dims=['y', 'x'], - attrs={'standard_name': 'longitude', - 'name': 'longitude'}) - lats = xr.DataArray(np.ones((2, 2)) * 2, - dims=['y', 'x'], - attrs={'standard_name': 'latitude', - 'name': 'latitude'}) - data = xr.DataArray(np.ones((2, 2)), - coords={'longitude': lons, - 'latitude': lats}, - dims=['y', 'x']) + self.lons = xr.DataArray(np.ones((2, 2)) * 2, + dims=['y', 'x'], + attrs={'standard_name': 'longitude', + 'name': 'longitude'}) + self.lats = xr.DataArray(np.ones((2, 2)) * 2, + dims=['y', 'x'], + attrs={'standard_name': 'latitude', + 'name': 'latitude'}) + self.data = None def _assign_array(dsid, *_args, **_kwargs): if dsid['name'] == 'longitude': - return lons + return self.lons elif dsid['name'] == 'latitude': - return lats + return self.lats - return data + return self.data fake_fh.get_dataset.side_effect = _assign_array self.reader.file_handlers = {'ftype1': [fake_fh]} - res = self.reader.load(['ch01']) + def test_load_dataset_with_builtin_coords(self): + """Test loading a dataset with builtin coordinates.""" + self.data = xr.DataArray(np.ones((2, 2)), + coords={'longitude': self.lons, + 'latitude': self.lats}, + dims=['y', 'x']) + + self._check_area_for_ch01() + def test_load_dataset_with_builtin_coords_in_wrong_order(self): + """Test loading a dataset with builtin coordinates in the wrong order.""" + self.data = xr.DataArray(np.ones((2, 2)), + coords={'latitude': self.lats, + 'longitude': self.lons}, + dims=['y', 'x']) + + self._check_area_for_ch01() + + def _check_area_for_ch01(self): + res = self.reader.load(['ch01']) assert 'area' in res['ch01'].attrs - np.testing.assert_array_equal(res['ch01'].attrs['area'].lons, lons) - np.testing.assert_array_equal(res['ch01'].attrs['area'].lats, lats) + np.testing.assert_array_equal(res['ch01'].attrs['area'].lons, self.lons) + np.testing.assert_array_equal(res['ch01'].attrs['area'].lats, self.lats) class TestFileFileYAMLReaderMultipleFileTypes(unittest.TestCase): From c7e597a25ed40cfa2c89538ef62284de976f4016 Mon Sep 17 00:00:00 2001 From: Martin Raspaud Date: Fri, 12 Feb 2021 07:48:16 +0100 Subject: [PATCH 5/5] Fix style --- satpy/readers/yaml_reader.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/satpy/readers/yaml_reader.py b/satpy/readers/yaml_reader.py index 5b42127258..bfa064b292 100644 --- a/satpy/readers/yaml_reader.py +++ b/satpy/readers/yaml_reader.py @@ -775,7 +775,7 @@ def _get_file_handlers(self, dsid): def _make_area_from_coords(self, coords): """Create an appropriate area with the given *coords*.""" if len(coords) == 2: - lats, lons = self._get_lons_lats_from_coords(coords) + lons, lats = self._get_lons_lats_from_coords(coords) sdef = self._make_swath_definition_from_lons_lats(lons, lats) return sdef @@ -793,7 +793,7 @@ def _get_lons_lats_from_coords(self, coords): lats = coord if lons is None or lats is None: raise ValueError('Missing longitude or latitude coordinate: ' + str(coords)) - return lats, lons + return lons, lats def _make_swath_definition_from_lons_lats(self, lons, lats): """Make a swath definition instance from lons and lats.""" @@ -841,7 +841,7 @@ def _load_dataset_with_area(self, dsid, coords, **kwargs): logger.exception("Could not load dataset '%s': %s", dsid, str(err)) return None - coords = self._assign_builtin_coords(coords, ds) + coords = self._assign_coords_from_dataarray(coords, ds) area = self._load_dataset_area(dsid, file_handlers, coords, **kwargs) @@ -851,8 +851,8 @@ def _load_dataset_with_area(self, dsid, coords, **kwargs): return ds @staticmethod - def _assign_builtin_coords(coords, ds): - """Assign builtin coords if needed.""" + def _assign_coords_from_dataarray(coords, ds): + """Assign coords from the *ds* dataarray if needed.""" if not coords: coords = [] for coord in ds.coords.values():