From 6b5ad9d5128a2891ad89621a58693d3847016415 Mon Sep 17 00:00:00 2001 From: andream Date: Thu, 17 Nov 2022 09:12:06 +0100 Subject: [PATCH 01/26] Adapt fci_l1c_nc.yaml file --- satpy/etc/readers/fci_l1c_nc.yaml | 268 ++++++++++++++++++------------ 1 file changed, 162 insertions(+), 106 deletions(-) diff --git a/satpy/etc/readers/fci_l1c_nc.yaml b/satpy/etc/readers/fci_l1c_nc.yaml index 75e396b56d..598fcb28c2 100644 --- a/satpy/etc/readers/fci_l1c_nc.yaml +++ b/satpy/etc/readers/fci_l1c_nc.yaml @@ -6,7 +6,7 @@ reader: Reader for FCI L1c data in NetCDF4 format. Used to read Meteosat Third Generation (MTG) Flexible Combined Imager (FCI) L1c data. - status: Beta for FDHSI, HRFI not supported yet + status: Beta for full-disc FDHSI and HRFI, RSS not supported yet supports_fsspec: false reader: !!python/name:satpy.readers.yaml_reader.GEOVariableSegmentYAMLReader sensors: [ fci ] @@ -16,7 +16,11 @@ reader: file_types: fci_l1c_fdhsi: file_reader: !!python/name:satpy.readers.fci_l1c_nc.FCIL1cNCFileHandler - file_patterns: [ '{pflag}_{location_indicator},{data_designator},MTI{spacecraft_id:1d}+{data_source}-1C-RRAD-FDHSI-{coverage}-{subsetting}-{component1}-BODY-{component3}-{purpose}-{format}_{oflag}_{originator}_{processing_time:%Y%m%d%H%M%S}_{facility_or_tool}_{environment}_{start_time:%Y%m%d%H%M%S}_{end_time:%Y%m%d%H%M%S}_{processing_mode}_{special_compression}_{disposition_mode}_{repeat_cycle_in_day:>04d}_{count_in_repeat_cycle:>04d}.nc' ] + file_patterns: [ '{pflag}_{location_indicator},{data_designator},MTI{spacecraft_id:1d}+{data_source}-1C-RRAD-FDHSI-FD-{subsetting}-{component1}-BODY-{component3}-{purpose}-{format}_{oflag}_{originator}_{processing_time:%Y%m%d%H%M%S}_{facility_or_tool}_{environment}_{start_time:%Y%m%d%H%M%S}_{end_time:%Y%m%d%H%M%S}_{processing_mode}_{special_compression}_{disposition_mode}_{repeat_cycle_in_day:>04d}_{count_in_repeat_cycle:>04d}.nc' ] + expected_segments: 40 + fci_l1c_hrfi: + file_reader: !!python/name:satpy.readers.fci_l1c_nc.FCIL1cNCFileHandler + file_patterns: [ '{pflag}_{location_indicator},{data_designator},MTI{spacecraft_id:1d}+{data_source}-1C-RRAD-HRFI-FD-{subsetting}-{component1}-BODY-{component3}-{purpose}-{format}_{oflag}_{originator}_{processing_time:%Y%m%d%H%M%S}_{facility_or_tool}_{environment}_{start_time:%Y%m%d%H%M%S}_{end_time:%Y%m%d%H%M%S}_{processing_mode}_{special_compression}_{disposition_mode}_{repeat_cycle_in_day:>04d}_{count_in_repeat_cycle:>04d}.nc' ] expected_segments: 40 @@ -59,7 +63,9 @@ datasets: name: vis_06 sensor: fci wavelength: [0.590, 0.640, 0.690] - resolution: 1000 + resolution: + 500: {file_type: fci_l1c_hrfi} + 1000: {file_type: fci_l1c_fdhsi} calibration: counts: standard_name: counts @@ -70,7 +76,6 @@ datasets: reflectance: standard_name: toa_bidirectional_reflectance units: "%" - file_type: fci_l1c_fdhsi vis_08: name: vis_08 @@ -144,7 +149,9 @@ datasets: name: nir_22 sensor: fci wavelength: [2.200, 2.250, 2.300] - resolution: 1000 + resolution: + 500: {file_type: fci_l1c_hrfi} + 1000: {file_type: fci_l1c_fdhsi} calibration: counts: standard_name: counts @@ -155,13 +162,14 @@ datasets: reflectance: standard_name: toa_bidirectional_reflectance units: "%" - file_type: fci_l1c_fdhsi ir_38: name: ir_38 sensor: fci wavelength: [3.400, 3.800, 4.200] - resolution: 2000 + resolution: + 1000: {file_type: fci_l1c_hrfi} + 2000: {file_type: fci_l1c_fdhsi} calibration: counts: standard_name: counts @@ -172,7 +180,6 @@ datasets: brightness_temperature: standard_name: toa_brightness_temperature units: "K" - file_type: fci_l1c_fdhsi wv_63: name: wv_63 @@ -246,7 +253,9 @@ datasets: name: ir_105 sensor: fci wavelength: [9.800, 10.500, 11.200] - resolution: 2000 + resolution: + 1000: {file_type: fci_l1c_hrfi} + 2000: {file_type: fci_l1c_fdhsi} calibration: counts: standard_name: counts @@ -257,7 +266,6 @@ datasets: brightness_temperature: standard_name: toa_brightness_temperature units: "K" - file_type: fci_l1c_fdhsi ir_123: name: ir_123 @@ -308,8 +316,9 @@ datasets: vis_06_pixel_quality: name: vis_06_pixel_quality sensor: fci - resolution: 1000 - file_type: fci_l1c_fdhsi + resolution: + 500: {file_type: fci_l1c_hrfi} + 1000: {file_type: fci_l1c_fdhsi} vis_08_pixel_quality: name: vis_08_pixel_quality @@ -338,14 +347,16 @@ datasets: nir_22_pixel_quality: name: nir_22_pixel_quality sensor: fci - resolution: 1000 - file_type: fci_l1c_fdhsi + resolution: + 500: {file_type: fci_l1c_hrfi} + 1000: {file_type: fci_l1c_fdhsi} ir_38_pixel_quality: name: ir_38_pixel_quality sensor: fci - resolution: 2000 - file_type: fci_l1c_fdhsi + resolution: + 1000: {file_type: fci_l1c_hrfi} + 2000: {file_type: fci_l1c_fdhsi} wv_63_pixel_quality: name: wv_63_pixel_quality @@ -374,8 +385,9 @@ datasets: ir_105_pixel_quality: name: ir_105_pixel_quality sensor: fci - resolution: 2000 - file_type: fci_l1c_fdhsi + resolution: + 1000: {file_type: fci_l1c_hrfi} + 2000: {file_type: fci_l1c_fdhsi} ir_123_pixel_quality: name: ir_123_pixel_quality @@ -404,8 +416,9 @@ datasets: vis_06_index_map: name: vis_06_index_map sensor: fci - resolution: 1000 - file_type: fci_l1c_fdhsi + resolution: + 500: {file_type: fci_l1c_hrfi} + 1000: {file_type: fci_l1c_fdhsi} vis_08_index_map: name: vis_08_index_map @@ -434,14 +447,16 @@ datasets: nir_22_index_map: name: nir_22_index_map sensor: fci - resolution: 1000 - file_type: fci_l1c_fdhsi + resolution: + 500: {file_type: fci_l1c_hrfi} + 1000: {file_type: fci_l1c_fdhsi} ir_38_index_map: name: ir_38_index_map sensor: fci - resolution: 2000 - file_type: fci_l1c_fdhsi + resolution: + 1000: {file_type: fci_l1c_hrfi} + 2000: {file_type: fci_l1c_fdhsi} wv_63_index_map: name: wv_63_index_map @@ -470,8 +485,9 @@ datasets: ir_105_index_map: name: ir_105_index_map sensor: fci - resolution: 2000 - file_type: fci_l1c_fdhsi + resolution: + 1000: {file_type: fci_l1c_hrfi} + 2000: {file_type: fci_l1c_fdhsi} ir_123_index_map: name: ir_123_index_map @@ -503,8 +519,9 @@ datasets: name: vis_06_time units: s sensor: fci - resolution: 1000 - file_type: fci_l1c_fdhsi + resolution: + 500: {file_type: fci_l1c_hrfi} + 1000: {file_type: fci_l1c_fdhsi} vis_08_time: name: vis_08_time @@ -538,15 +555,17 @@ datasets: name: nir_22_time units: s sensor: fci - resolution: 1000 - file_type: fci_l1c_fdhsi + resolution: + 500: {file_type: fci_l1c_hrfi} + 1000: {file_type: fci_l1c_fdhsi} ir_38_time: name: ir_38_time units: s sensor: fci - resolution: 2000 - file_type: fci_l1c_fdhsi + resolution: + 1000: {file_type: fci_l1c_hrfi} + 2000: {file_type: fci_l1c_fdhsi} wv_63_time: name: wv_63_time @@ -580,8 +599,9 @@ datasets: name: ir_105_time units: s sensor: fci - resolution: 2000 - file_type: fci_l1c_fdhsi + resolution: + 1000: {file_type: fci_l1c_hrfi} + 2000: {file_type: fci_l1c_fdhsi} ir_123_time: name: ir_123_time @@ -612,8 +632,9 @@ datasets: vis_06_swath_direction: name: vis_06_swath_direction sensor: fci - resolution: 1000 - file_type: fci_l1c_fdhsi + resolution: + 500: {file_type: fci_l1c_hrfi} + 1000: {file_type: fci_l1c_fdhsi} vis_08_swath_direction: name: vis_08_swath_direction @@ -642,14 +663,16 @@ datasets: nir_22_swath_direction: name: nir_22_swath_direction sensor: fci - resolution: 1000 - file_type: fci_l1c_fdhsi + resolution: + 500: {file_type: fci_l1c_hrfi} + 1000: {file_type: fci_l1c_fdhsi} ir_38_swath_direction: name: ir_38_swath_direction sensor: fci - resolution: 2000 - file_type: fci_l1c_fdhsi + resolution: + 1000: {file_type: fci_l1c_hrfi} + 2000: {file_type: fci_l1c_fdhsi} wv_63_swath_direction: name: wv_63_swath_direction @@ -678,8 +701,9 @@ datasets: ir_105_swath_direction: name: ir_105_swath_direction sensor: fci - resolution: 2000 - file_type: fci_l1c_fdhsi + resolution: + 1000: {file_type: fci_l1c_hrfi} + 2000: {file_type: fci_l1c_fdhsi} ir_123_swath_direction: name: ir_123_swath_direction @@ -708,8 +732,9 @@ datasets: vis_06_swath_number: name: vis_06_swath_number sensor: fci - resolution: 1000 - file_type: fci_l1c_fdhsi + resolution: + 500: {file_type: fci_l1c_hrfi} + 1000: {file_type: fci_l1c_fdhsi} vis_08_swath_number: name: vis_08_swath_number @@ -738,14 +763,16 @@ datasets: nir_22_swath_number: name: nir_22_swath_number sensor: fci - resolution: 1000 - file_type: fci_l1c_fdhsi + resolution: + 500: {file_type: fci_l1c_hrfi} + 1000: {file_type: fci_l1c_fdhsi} ir_38_swath_number: name: ir_38_swath_number sensor: fci - resolution: 2000 - file_type: fci_l1c_fdhsi + resolution: + 1000: {file_type: fci_l1c_hrfi} + 2000: {file_type: fci_l1c_fdhsi} wv_63_swath_number: name: wv_63_swath_number @@ -774,8 +801,9 @@ datasets: ir_105_swath_number: name: ir_105_swath_number sensor: fci - resolution: 2000 - file_type: fci_l1c_fdhsi + resolution: + 1000: {file_type: fci_l1c_hrfi} + 2000: {file_type: fci_l1c_fdhsi} ir_123_swath_number: name: ir_123_swath_number @@ -807,8 +835,9 @@ datasets: name: vis_06_subsatellite_latitude units: deg sensor: fci - resolution: 1000 - file_type: fci_l1c_fdhsi + resolution: + 500: {file_type: fci_l1c_hrfi} + 1000: {file_type: fci_l1c_fdhsi} vis_08_subsatellite_latitude: name: vis_08_subsatellite_latitude @@ -842,15 +871,17 @@ datasets: name: nir_22_subsatellite_latitude units: deg sensor: fci - resolution: 1000 - file_type: fci_l1c_fdhsi + resolution: + 500: {file_type: fci_l1c_hrfi} + 1000: {file_type: fci_l1c_fdhsi} ir_38_subsatellite_latitude: name: ir_38_subsatellite_latitude units: deg sensor: fci - resolution: 2000 - file_type: fci_l1c_fdhsi + resolution: + 1000: {file_type: fci_l1c_hrfi} + 2000: {file_type: fci_l1c_fdhsi} wv_63_subsatellite_latitude: name: wv_63_subsatellite_latitude @@ -884,8 +915,9 @@ datasets: name: ir_105_subsatellite_latitude units: deg sensor: fci - resolution: 2000 - file_type: fci_l1c_fdhsi + resolution: + 1000: {file_type: fci_l1c_hrfi} + 2000: {file_type: fci_l1c_fdhsi} ir_123_subsatellite_latitude: name: ir_123_subsatellite_latitude @@ -919,8 +951,9 @@ datasets: name: vis_06_subsatellite_longitude units: deg sensor: fci - resolution: 1000 - file_type: fci_l1c_fdhsi + resolution: + 500: {file_type: fci_l1c_hrfi} + 1000: {file_type: fci_l1c_fdhsi} vis_08_subsatellite_longitude: name: vis_08_subsatellite_longitude @@ -954,15 +987,17 @@ datasets: name: nir_22_subsatellite_longitude units: deg sensor: fci - resolution: 1000 - file_type: fci_l1c_fdhsi + resolution: + 500: {file_type: fci_l1c_hrfi} + 1000: {file_type: fci_l1c_fdhsi} ir_38_subsatellite_longitude: name: ir_38_subsatellite_longitude units: deg sensor: fci - resolution: 2000 - file_type: fci_l1c_fdhsi + resolution: + 1000: {file_type: fci_l1c_hrfi} + 2000: {file_type: fci_l1c_fdhsi} wv_63_subsatellite_longitude: name: wv_63_subsatellite_longitude @@ -996,8 +1031,9 @@ datasets: name: ir_105_subsatellite_longitude units: deg sensor: fci - resolution: 2000 - file_type: fci_l1c_fdhsi + resolution: + 1000: {file_type: fci_l1c_hrfi} + 2000: {file_type: fci_l1c_fdhsi} ir_123_subsatellite_longitude: name: ir_123_subsatellite_longitude @@ -1031,8 +1067,9 @@ datasets: name: vis_06_subsolar_latitude units: deg sensor: fci - resolution: 1000 - file_type: fci_l1c_fdhsi + resolution: + 500: {file_type: fci_l1c_hrfi} + 1000: {file_type: fci_l1c_fdhsi} vis_08_subsolar_latitude: name: vis_08_subsolar_latitude @@ -1066,15 +1103,17 @@ datasets: name: nir_22_subsolar_latitude units: deg sensor: fci - resolution: 1000 - file_type: fci_l1c_fdhsi + resolution: + 500: {file_type: fci_l1c_hrfi} + 1000: {file_type: fci_l1c_fdhsi} ir_38_subsolar_latitude: name: ir_38_subsolar_latitude units: deg sensor: fci - resolution: 2000 - file_type: fci_l1c_fdhsi + resolution: + 1000: {file_type: fci_l1c_hrfi} + 2000: {file_type: fci_l1c_fdhsi} wv_63_subsolar_latitude: name: wv_63_subsolar_latitude @@ -1108,8 +1147,9 @@ datasets: name: ir_105_subsolar_latitude units: deg sensor: fci - resolution: 2000 - file_type: fci_l1c_fdhsi + resolution: + 1000: {file_type: fci_l1c_hrfi} + 2000: {file_type: fci_l1c_fdhsi} ir_123_subsolar_latitude: name: ir_123_subsolar_latitude @@ -1143,8 +1183,9 @@ datasets: name: vis_06_subsolar_longitude units: deg sensor: fci - resolution: 1000 - file_type: fci_l1c_fdhsi + resolution: + 500: {file_type: fci_l1c_hrfi} + 1000: {file_type: fci_l1c_fdhsi} vis_08_subsolar_longitude: name: vis_08_subsolar_longitude @@ -1178,15 +1219,17 @@ datasets: name: nir_22_subsolar_longitude units: deg sensor: fci - resolution: 1000 - file_type: fci_l1c_fdhsi + resolution: + 500: {file_type: fci_l1c_hrfi} + 1000: {file_type: fci_l1c_fdhsi} ir_38_subsolar_longitude: name: ir_38_subsolar_longitude units: deg sensor: fci - resolution: 2000 - file_type: fci_l1c_fdhsi + resolution: + 1000: {file_type: fci_l1c_hrfi} + 2000: {file_type: fci_l1c_fdhsi} wv_63_subsolar_longitude: name: wv_63_subsolar_longitude @@ -1220,8 +1263,9 @@ datasets: name: ir_105_subsolar_longitude units: deg sensor: fci - resolution: 2000 - file_type: fci_l1c_fdhsi + resolution: + 1000: {file_type: fci_l1c_hrfi} + 2000: {file_type: fci_l1c_fdhsi} ir_123_subsolar_longitude: name: ir_123_subsolar_longitude @@ -1256,8 +1300,9 @@ datasets: name: vis_06_platform_altitude units: m sensor: fci - resolution: 1000 - file_type: fci_l1c_fdhsi + resolution: + 500: {file_type: fci_l1c_hrfi} + 1000: {file_type: fci_l1c_fdhsi} vis_08_platform_altitude: name: vis_08_platform_altitude @@ -1291,15 +1336,17 @@ datasets: name: nir_22_platform_altitude units: m sensor: fci - resolution: 1000 - file_type: fci_l1c_fdhsi + resolution: + 500: {file_type: fci_l1c_hrfi} + 1000: {file_type: fci_l1c_fdhsi} ir_38_platform_altitude: name: ir_38_platform_altitude units: m sensor: fci - resolution: 2000 - file_type: fci_l1c_fdhsi + resolution: + 1000: {file_type: fci_l1c_hrfi} + 2000: {file_type: fci_l1c_fdhsi} wv_63_platform_altitude: name: wv_63_platform_altitude @@ -1333,8 +1380,9 @@ datasets: name: ir_105_platform_altitude units: m sensor: fci - resolution: 2000 - file_type: fci_l1c_fdhsi + resolution: + 1000: {file_type: fci_l1c_hrfi} + 2000: {file_type: fci_l1c_fdhsi} ir_123_platform_altitude: name: ir_123_platform_altitude @@ -1368,8 +1416,9 @@ datasets: name: vis_06_earth_sun_distance units: km sensor: fci - resolution: 1000 - file_type: fci_l1c_fdhsi + resolution: + 500: {file_type: fci_l1c_hrfi} + 1000: {file_type: fci_l1c_fdhsi} vis_08_earth_sun_distance: name: vis_08_earth_sun_distance @@ -1403,15 +1452,17 @@ datasets: name: nir_22_earth_sun_distance units: km sensor: fci - resolution: 1000 - file_type: fci_l1c_fdhsi + resolution: + 500: {file_type: fci_l1c_hrfi} + 1000: {file_type: fci_l1c_fdhsi} ir_38_earth_sun_distance: name: ir_38_earth_sun_distance units: km sensor: fci - resolution: 2000 - file_type: fci_l1c_fdhsi + resolution: + 1000: {file_type: fci_l1c_hrfi} + 2000: {file_type: fci_l1c_fdhsi} wv_63_earth_sun_distance: name: wv_63_earth_sun_distance @@ -1445,8 +1496,9 @@ datasets: name: ir_105_earth_sun_distance units: km sensor: fci - resolution: 2000 - file_type: fci_l1c_fdhsi + resolution: + 1000: {file_type: fci_l1c_hrfi} + 2000: {file_type: fci_l1c_fdhsi} ir_123_earth_sun_distance: name: ir_123_earth_sun_distance @@ -1480,8 +1532,9 @@ datasets: name: vis_06_sun_satellite_distance units: km sensor: fci - resolution: 1000 - file_type: fci_l1c_fdhsi + resolution: + 500: {file_type: fci_l1c_hrfi} + 1000: {file_type: fci_l1c_fdhsi} vis_08_sun_satellite_distance: name: vis_08_sun_satellite_distance @@ -1515,15 +1568,17 @@ datasets: name: nir_22_sun_satellite_distance units: km sensor: fci - resolution: 1000 - file_type: fci_l1c_fdhsi + resolution: + 500: {file_type: fci_l1c_hrfi} + 1000: {file_type: fci_l1c_fdhsi} ir_38_sun_satellite_distance: name: ir_38_sun_satellite_distance units: km sensor: fci - resolution: 2000 - file_type: fci_l1c_fdhsi + resolution: + 1000: {file_type: fci_l1c_hrfi} + 2000: {file_type: fci_l1c_fdhsi} wv_63_sun_satellite_distance: name: wv_63_sun_satellite_distance @@ -1557,8 +1612,9 @@ datasets: name: ir_105_sun_satellite_distance units: km sensor: fci - resolution: 2000 - file_type: fci_l1c_fdhsi + resolution: + 1000: {file_type: fci_l1c_hrfi} + 2000: {file_type: fci_l1c_fdhsi} ir_123_sun_satellite_distance: name: ir_123_sun_satellite_distance From 78fd9a86c8f75a9911f265e2d11da22ed0e86e8d Mon Sep 17 00:00:00 2001 From: andream Date: Thu, 17 Nov 2022 09:53:56 +0100 Subject: [PATCH 02/26] adapt fci_l1c_nc.py --- satpy/readers/fci_l1c_nc.py | 35 +++++++++++---------- satpy/tests/reader_tests/test_fci_l1c_nc.py | 9 ++++-- 2 files changed, 26 insertions(+), 18 deletions(-) diff --git a/satpy/readers/fci_l1c_nc.py b/satpy/readers/fci_l1c_nc.py index 0d56bd1578..f503a117eb 100644 --- a/satpy/readers/fci_l1c_nc.py +++ b/satpy/readers/fci_l1c_nc.py @@ -204,21 +204,31 @@ def end_time(self): """Get end time.""" return self.filename_info['end_time'] + def get_channel_measured_group_path(self, channel): + """Get the channel's measured group path.""" + if self.filetype_info['file_type'] == 'fci_l1c_hrfi': + channel += '_hr' + measured_group_path = 'data/{}/measured'.format(channel) + + return measured_group_path + def get_segment_position_info(self): """Get the vertical position and size information of the chunk (aka segment) for both 1km and 2km grids. This is used in the GEOVariableSegmentYAMLReader to compute optimal chunk sizes for missing chunks. """ + vis_06_measured_path = self.get_channel_measured_group_path('vis_06') + ir_105_measured_path = self.get_channel_measured_group_path('ir_105') segment_position_info = { - '1km': {'start_position_row': self['data/vis_04/measured/start_position_row'].item(), - 'end_position_row': self['data/vis_04/measured/end_position_row'].item(), - 'segment_height': self['data/vis_04/measured/end_position_row'].item() - - self['data/vis_04/measured/start_position_row'].item() + 1, + '1km': {'start_position_row': self[vis_06_measured_path+'/start_position_row'].item(), + 'end_position_row': self[vis_06_measured_path+'/end_position_row'].item(), + 'segment_height': self[vis_06_measured_path+'/end_position_row'].item() - + self[vis_06_measured_path+'/start_position_row'].item() + 1, 'segment_width': 11136}, - '2km': {'start_position_row': self['data/ir_105/measured/start_position_row'].item(), - 'end_position_row': self['data/ir_105/measured/end_position_row'].item(), - 'segment_height': self['data/ir_105/measured/end_position_row'].item() - - self['data/ir_105/measured/start_position_row'].item() + 1, + '2km': {'start_position_row': self[ir_105_measured_path+'/start_position_row'].item(), + 'end_position_row': self[ir_105_measured_path+'/end_position_row'].item(), + 'segment_height': self[ir_105_measured_path+'/end_position_row'].item() - + self[ir_105_measured_path+'/start_position_row'].item() + 1, 'segment_width': 5568} } @@ -386,13 +396,6 @@ def _get_dataset_aux_data(self, dsname): return aux - @staticmethod - def get_channel_measured_group_path(channel): - """Get the channel's measured group path.""" - measured_group_path = 'data/{}/measured'.format(channel) - - return measured_group_path - def calc_area_extent(self, key): """Calculate area extent for a dataset.""" # if a user requests a pixel quality or index map before the channel data, the @@ -412,7 +415,7 @@ def calc_area_extent(self, key): extents = {} for coord in "xy": - coord_radian = self["data/{:s}/measured/{:s}".format(channel_name, coord)] + coord_radian = self[measured + "/{:s}".format(coord)] # TODO remove this check when old versions of IDPF test data ( 0: diff --git a/satpy/tests/reader_tests/test_fci_l1c_nc.py b/satpy/tests/reader_tests/test_fci_l1c_nc.py index 48203503c4..5b1fc49c97 100644 --- a/satpy/tests/reader_tests/test_fci_l1c_nc.py +++ b/satpy/tests/reader_tests/test_fci_l1c_nc.py @@ -325,6 +325,11 @@ def test_file_pattern(self, reader_configs): "W_XX-EUMETSAT-Darmstadt,IMG+SAT,MTI1+FCI-1C-RRAD-FDHSI-FD--" "CHK-BODY--L2P-NC4E_C_EUMT_20170410114500_GTT_DEV_" "20170410113951_20170410114000_N__C_0070_0070.nc", + # this is an HRFI file + "W_XX-EUMETSAT-Darmstadt,IMG+SAT,MTI1+FCI-1C-RRAD-HRFI-FD--" + "CHK-BODY--L2P-NC4E_C_EUMT_20170410114500_GTT_DEV_" + "20170410113951_20170410114000_N__C_0070_0070.nc", + # this is a TRAIL file, which we don't use/read "W_XX-EUMETSAT-Darmstadt,IMG+SAT,MTI1+FCI-1C-RRAD-FDHSI-FD--" "CHK-TRAIL--L2P-NC4E_C_EUMT_20170410114600_GTT_DEV_" "20170410113000_20170410114000_N__C_0070_0071.nc", @@ -332,8 +337,8 @@ def test_file_pattern(self, reader_configs): reader = load_reader(reader_configs) files = reader.select_files_from_pathnames(filenames) - # only 4 out of 5 above should match - assert len(files) == 4 + # only 5 (4 FDHSI, 1 HRFI) should match, the TRAIL should be ignored + assert len(files) == 5 _chans = {"solar": ["vis_04", "vis_05", "vis_06", "vis_08", "vis_09", "nir_13", "nir_16", "nir_22"], From 035c746b646e9da355cbeb424fa8c92b3ae3c74f Mon Sep 17 00:00:00 2001 From: andream Date: Thu, 17 Nov 2022 10:16:53 +0100 Subject: [PATCH 03/26] refactor FakeFileHandler names --- satpy/tests/reader_tests/test_fci_l1c_nc.py | 51 +++++++++++---------- 1 file changed, 27 insertions(+), 24 deletions(-) diff --git a/satpy/tests/reader_tests/test_fci_l1c_nc.py b/satpy/tests/reader_tests/test_fci_l1c_nc.py index 5b1fc49c97..a95481a681 100644 --- a/satpy/tests/reader_tests/test_fci_l1c_nc.py +++ b/satpy/tests/reader_tests/test_fci_l1c_nc.py @@ -30,7 +30,7 @@ from satpy.tests.reader_tests.test_netcdf_utils import FakeNetCDF4FileHandler -class FakeNetCDF4FileHandler2(FakeNetCDF4FileHandler): +class FakeFCIFileHandlerBase(FakeNetCDF4FileHandler): """Class for faking the NetCDF4 Filehandler.""" def _get_test_calib_for_channel_ir(self, chroot, meas): @@ -143,17 +143,7 @@ def _get_test_content_for_channel(self, pat, ch): return data def _get_test_content_all_channels(self): - chan_patterns = { - "vis_{:>02d}": (4, 5, 6, 8, 9), - "nir_{:>02d}": (13, 16, 22), - "ir_{:>02d}": (38, 87, 97, 105, 123, 133), - "wv_{:>02d}": (63, 73), - } - data = {} - for pat in chan_patterns: - for ch_num in chan_patterns[pat]: - data.update(self._get_test_content_for_channel(pat, ch_num)) - return data + raise NotImplementedError def _get_test_content_areadef(self): data = {} @@ -221,7 +211,24 @@ def get_test_content(self, filename, filename_info, filetype_info): return D -class FakeNetCDF4FileHandler3(FakeNetCDF4FileHandler2): +class FakeFCIFileHandlerFDHSI(FakeFCIFileHandlerBase): + """Mock FDHSI data.""" + + def _get_test_content_all_channels(self): + chan_patterns = { + "vis_{:>02d}": (4, 5, 6, 8, 9), + "nir_{:>02d}": (13, 16, 22), + "ir_{:>02d}": (38, 87, 97, 105, 123, 133), + "wv_{:>02d}": (63, 73), + } + data = {} + for pat in chan_patterns: + for ch_num in chan_patterns[pat]: + data.update(self._get_test_content_for_channel(pat, ch_num)) + return data + + +class FakeFCIFileHandlerWithBadData(FakeFCIFileHandlerFDHSI): """Mock bad data.""" def _get_test_calib_for_channel_ir(self, chroot, meas): @@ -243,7 +250,7 @@ def _get_test_calib_for_channel_vis(self, chroot, meas): return data -class FakeNetCDF4FileHandler4(FakeNetCDF4FileHandler2): +class FakeFCIFileHandlerWithBadIDPFData(FakeFCIFileHandlerFDHSI): """Mock bad data for IDPF TO-DO's.""" def _get_test_calib_for_channel_vis(self, chroot, meas): @@ -287,7 +294,7 @@ class TestFCIL1cNCReader: yaml_file = "fci_l1c_nc.yaml" - _alt_handler = FakeNetCDF4FileHandler2 + _alt_handler = FakeFCIFileHandlerFDHSI @pytest.fixture(autouse=True, scope="class") def fake_handler(self): @@ -306,7 +313,7 @@ def fake_handler(self): class TestFCIL1cNCReaderGoodData(TestFCIL1cNCReader): """Test FCI L1c NetCDF reader.""" - _alt_handler = FakeNetCDF4FileHandler2 + _alt_handler = FakeFCIFileHandlerFDHSI def test_file_pattern(self, reader_configs): """Test file pattern matching.""" @@ -325,10 +332,6 @@ def test_file_pattern(self, reader_configs): "W_XX-EUMETSAT-Darmstadt,IMG+SAT,MTI1+FCI-1C-RRAD-FDHSI-FD--" "CHK-BODY--L2P-NC4E_C_EUMT_20170410114500_GTT_DEV_" "20170410113951_20170410114000_N__C_0070_0070.nc", - # this is an HRFI file - "W_XX-EUMETSAT-Darmstadt,IMG+SAT,MTI1+FCI-1C-RRAD-HRFI-FD--" - "CHK-BODY--L2P-NC4E_C_EUMT_20170410114500_GTT_DEV_" - "20170410113951_20170410114000_N__C_0070_0070.nc", # this is a TRAIL file, which we don't use/read "W_XX-EUMETSAT-Darmstadt,IMG+SAT,MTI1+FCI-1C-RRAD-FDHSI-FD--" "CHK-TRAIL--L2P-NC4E_C_EUMT_20170410114600_GTT_DEV_" @@ -337,8 +340,8 @@ def test_file_pattern(self, reader_configs): reader = load_reader(reader_configs) files = reader.select_files_from_pathnames(filenames) - # only 5 (4 FDHSI, 1 HRFI) should match, the TRAIL should be ignored - assert len(files) == 5 + # only 4 FDHSI should match, the TRAIL should be ignored + assert len(files) == 4 _chans = {"solar": ["vis_04", "vis_05", "vis_06", "vis_08", "vis_09", "nir_13", "nir_16", "nir_22"], @@ -614,7 +617,7 @@ def test_area_definition_computation(self, reader_configs): class TestFCIL1cNCReaderBadData(TestFCIL1cNCReader): """Test the FCI L1c NetCDF Reader for bad data input.""" - _alt_handler = FakeNetCDF4FileHandler3 + _alt_handler = FakeFCIFileHandlerWithBadData def test_handling_bad_data_ir(self, reader_configs, caplog): """Test handling of bad IR data.""" @@ -654,7 +657,7 @@ def test_handling_bad_data_vis(self, reader_configs, caplog): class TestFCIL1cNCReaderBadDataFromIDPF(TestFCIL1cNCReader): """Test the FCI L1c NetCDF Reader for bad data input.""" - _alt_handler = FakeNetCDF4FileHandler4 + _alt_handler = FakeFCIFileHandlerWithBadIDPFData def test_handling_bad_earthsun_distance(self, reader_configs, caplog): """Test handling of bad earth-sun distance data.""" From 4477b5881bc09b86c1d73e0bb844fe065fb786cd Mon Sep 17 00:00:00 2001 From: andream Date: Thu, 17 Nov 2022 15:58:00 +0100 Subject: [PATCH 04/26] first working example of parametrized test for FDHSI and HRFI --- satpy/tests/reader_tests/test_fci_l1c_nc.py | 179 ++++++++++++++------ 1 file changed, 127 insertions(+), 52 deletions(-) diff --git a/satpy/tests/reader_tests/test_fci_l1c_nc.py b/satpy/tests/reader_tests/test_fci_l1c_nc.py index a95481a681..c4435205d3 100644 --- a/satpy/tests/reader_tests/test_fci_l1c_nc.py +++ b/satpy/tests/reader_tests/test_fci_l1c_nc.py @@ -273,6 +273,22 @@ def _get_test_content_all_channels(self): return data +class FakeFCIFileHandlerHRFI(FakeFCIFileHandlerBase): + """Mock HRFI data.""" + + def _get_test_content_all_channels(self): + chan_patterns = { + "vis_{:>02d}_hr": [6], + "nir_{:>02d}_hr": [22], + "ir_{:>02d}_hr": [38, 105], + } + data = {} + for pat in chan_patterns: + for ch_num in chan_patterns[pat]: + data.update(self._get_test_content_for_channel(pat, ch_num)) + return data + + @pytest.fixture def reader_configs(): """Return reader configs for FCI.""" @@ -291,29 +307,80 @@ def _get_reader_with_filehandlers(filenames, reader_configs): class TestFCIL1cNCReader: """Initialize the unittest TestCase for the FCI L1c NetCDF Reader.""" - yaml_file = "fci_l1c_nc.yaml" - _alt_handler = FakeFCIFileHandlerFDHSI - @pytest.fixture(autouse=True, scope="class") - def fake_handler(self): - """Wrap NetCDF4 FileHandler with our own fake handler.""" - # implementation strongly inspired by test_viirs_l1b.py - from satpy.readers.fci_l1c_nc import FCIL1cNCFileHandler - p = mock.patch.object( - FCIL1cNCFileHandler, - "__bases__", - (self._alt_handler,)) - with p: - p.is_local = True - yield p +class TestFCIL1cNCReaderGoodData(TestFCIL1cNCReader): + """Test FCI L1c NetCDF reader with good data, on the FDHSI example.""" + + _alt_handler_FDHSI = FakeFCIFileHandlerFDHSI + _alt_handler_HRFI = FakeFCIFileHandlerHRFI + # + # @pytest.fixture(autouse=True, scope="function") + # def fake_handler(self): + # """Wrap NetCDF4 FileHandler with our own fake handler.""" + # # implementation strongly inspired by test_viirs_l1b.py + # from satpy.readers.fci_l1c_nc import FCIL1cNCFileHandler + # p = mock.patch.object( + # FCIL1cNCFileHandler, + # "__bases__", + # (self._alt_handler,)) + # with p: + # p.is_local = True + # yield p + + _chans_fdhsi = {"solar": ["vis_04", "vis_05", "vis_06", "vis_08", "vis_09", + "nir_13", "nir_16", "nir_22"], + "terran": ["ir_38", "wv_63", "wv_73", "ir_87", "ir_97", "ir_105", + "ir_123", "ir_133"]} + _chans_hrfi = {"solar": ["vis_06", "nir_22"], + "terran": ["ir_38", "ir_105"]} -class TestFCIL1cNCReaderGoodData(TestFCIL1cNCReader): - """Test FCI L1c NetCDF reader.""" + @pytest.mark.parametrize('filehandler,channels,filenames,expected_res_n', [ + (FakeFCIFileHandlerFDHSI, _chans_fdhsi, [ + "W_XX-EUMETSAT-Darmstadt,IMG+SAT,MTI1+FCI-1C-RRAD-FDHSI-FD--" + "CHK-BODY--L2P-NC4E_C_EUMT_20170410114434_GTT_DEV_" + "20170410113925_20170410113934_N__C_0070_0067.nc", + "W_XX-EUMETSAT-Darmstadt,IMG+SAT,MTI1+FCI-1C-RRAD-FDHSI-FD--" + "CHK-BODY--L2P-NC4E_C_EUMT_20170410114442_GTT_DEV_" + "20170410113934_20170410113942_N__C_0070_0068.nc", + ], 16), + (FakeFCIFileHandlerHRFI, _chans_hrfi, [ + "W_XX-EUMETSAT-Darmstadt,IMG+SAT,MTI1+FCI-1C-RRAD-HRFI-FD--" + "CHK-BODY--L2P-NC4E_C_EUMT_20170410114434_GTT_DEV_" + "20170410113925_20170410113934_N__C_0070_0067.nc", + "W_XX-EUMETSAT-Darmstadt,IMG+SAT,MTI1+FCI-1C-RRAD-HRFI-FD--" + "CHK-BODY--L2P-NC4E_C_EUMT_20170410114442_GTT_DEV_" + "20170410113934_20170410113942_N__C_0070_0068.nc", + ], 4), - _alt_handler = FakeFCIFileHandlerFDHSI + ]) + def test_load_counts(self, reader_configs, filehandler, channels, filenames, + expected_res_n): + """Test loading with counts.""" + from satpy.tests.utils import make_dataid + self._alt_handler = filehandler + from satpy.readers.fci_l1c_nc import FCIL1cNCFileHandler + p = mock.patch.object(FCIL1cNCFileHandler, "__bases__", (self._alt_handler,)) + with p: + p.is_local = True + self._chans = channels + reader = _get_reader_with_filehandlers(filenames, reader_configs) + res = reader.load( + [make_dataid(name=name, calibration="counts") for name in + self._chans["solar"] + self._chans["terran"]], pad_data=False) + assert expected_res_n == len(res) + for ch in self._chans["solar"] + self._chans["terran"]: + assert res[ch].shape == (200 * 2, 11136) + assert res[ch].dtype == np.uint16 + assert res[ch].attrs["calibration"] == "counts" + assert res[ch].attrs["units"] == "count" + if ch == 'ir_38': + numpy.testing.assert_array_equal(res[ch][~0], 1) + numpy.testing.assert_array_equal(res[ch][0], 5000) + else: + numpy.testing.assert_array_equal(res[ch], 1) def test_file_pattern(self, reader_configs): """Test file pattern matching.""" @@ -343,41 +410,6 @@ def test_file_pattern(self, reader_configs): # only 4 FDHSI should match, the TRAIL should be ignored assert len(files) == 4 - _chans = {"solar": ["vis_04", "vis_05", "vis_06", "vis_08", "vis_09", - "nir_13", "nir_16", "nir_22"], - "terran": ["ir_38", "wv_63", "wv_73", "ir_87", "ir_97", "ir_105", - "ir_123", "ir_133"]} - - def test_load_counts(self, reader_configs): - """Test loading with counts.""" - from satpy.tests.utils import make_dataid - - # testing two filenames to test correctly combined - filenames = [ - "W_XX-EUMETSAT-Darmstadt,IMG+SAT,MTI1+FCI-1C-RRAD-FDHSI-FD--" - "CHK-BODY--L2P-NC4E_C_EUMT_20170410114434_GTT_DEV_" - "20170410113925_20170410113934_N__C_0070_0067.nc", - "W_XX-EUMETSAT-Darmstadt,IMG+SAT,MTI1+FCI-1C-RRAD-FDHSI-FD--" - "CHK-BODY--L2P-NC4E_C_EUMT_20170410114442_GTT_DEV_" - "20170410113934_20170410113942_N__C_0070_0068.nc", - ] - - reader = _get_reader_with_filehandlers(filenames, reader_configs) - res = reader.load( - [make_dataid(name=name, calibration="counts") for name in - self._chans["solar"] + self._chans["terran"]], pad_data=False) - assert 16 == len(res) - for ch in self._chans["solar"] + self._chans["terran"]: - assert res[ch].shape == (200 * 2, 11136) - assert res[ch].dtype == np.uint16 - assert res[ch].attrs["calibration"] == "counts" - assert res[ch].attrs["units"] == "count" - if ch == 'ir_38': - numpy.testing.assert_array_equal(res[ch][~0], 1) - numpy.testing.assert_array_equal(res[ch][0], 5000) - else: - numpy.testing.assert_array_equal(res[ch], 1) - def test_load_radiance(self, reader_configs): """Test loading with radiance.""" from satpy.tests.utils import make_dataid @@ -690,3 +722,46 @@ def test_bad_xy_coords(self, reader_configs): np.testing.assert_array_almost_equal(np.array(area_def.area_extent), np.array([-5568062.270889, 5168057.806632, 16704186.298937, 5568062.270889])) + + +# class TestFCIL1cNCReaderHRFI(TestFCIL1cNCReaderGoodData): +# """Test FCI L1c NetCDF reader for the HRFI case.""" +# +# _alt_handler = FakeFCIFileHandlerHRFI +# +# def test_file_pattern(self, reader_configs): +# """Test file pattern matching.""" +# from satpy.readers import load_reader +# +# filenames = [ +# "W_XX-EUMETSAT-Darmstadt,IMG+SAT,MTI1+FCI-1C-RRAD-HRFI-FD--" +# "CHK-BODY--L2P-NC4E_C_EUMT_20170410114434_GTT_DEV_" +# "20170410113925_20170410113934_N__C_0070_0038.nc", +# "W_XX-EUMETSAT-Darmstadt,IMG+SAT,MTI1+FCI-1C-RRAD-HRFI-FD--" +# "CHK-BODY--L2P-NC4E_C_EUMT_20170410114442_GTT_DEV_" +# "20170410113934_20170410113942_N__C_0070_0039.nc", +# # this is a TRAIL file, which we don't use/read +# "W_XX-EUMETSAT-Darmstadt,IMG+SAT,MTI1+FCI-1C-RRAD-HRFI-FD--" +# "CHK-TRAIL--L2P-NC4E_C_EUMT_20170410114600_GTT_DEV_" +# "20170410113000_20170410114000_N__C_0070_0071.nc", +# ] +# +# reader = load_reader(reader_configs) +# files = reader.select_files_from_pathnames(filenames) +# # only 2 HRFI should match, the TRAIL should be ignored +# assert len(files) == 2 +# +# _chans = {"solar": ["vis_06", "nir_22"], +# "terran": ["ir_38", "ir_105"]} +# +# def test_load_counts(self, reader_configs): +# # testing two filenames to test correctly combined +# filenames = [ +# "W_XX-EUMETSAT-Darmstadt,IMG+SAT,MTI1+FCI-1C-RRAD-HRFI-FD--" +# "CHK-BODY--L2P-NC4E_C_EUMT_20170410114434_GTT_DEV_" +# "20170410113925_20170410113934_N__C_0070_0038.nc", +# "W_XX-EUMETSAT-Darmstadt,IMG+SAT,MTI1+FCI-1C-RRAD-HRFI-FD--" +# "CHK-BODY--L2P-NC4E_C_EUMT_20170410114442_GTT_DEV_" +# "20170410113934_20170410113942_N__C_0070_0039.nc", +# ] +# self._check_load_counts(reader_configs, filenames) From aabd0b4d60cd1325de82e4e5550f03a1a297bd12 Mon Sep 17 00:00:00 2001 From: andream Date: Fri, 18 Nov 2022 09:13:07 +0100 Subject: [PATCH 05/26] first parametrisation of tests working --- satpy/tests/reader_tests/test_fci_l1c_nc.py | 508 +++++++++++--------- 1 file changed, 279 insertions(+), 229 deletions(-) diff --git a/satpy/tests/reader_tests/test_fci_l1c_nc.py b/satpy/tests/reader_tests/test_fci_l1c_nc.py index c4435205d3..739abe6043 100644 --- a/satpy/tests/reader_tests/test_fci_l1c_nc.py +++ b/satpy/tests/reader_tests/test_fci_l1c_nc.py @@ -16,7 +16,7 @@ # You should have received a copy of the GNU General Public License along with # satpy. If not, see . """Tests for the 'fci_l1c_nc' reader.""" - +import contextlib import logging import os from unittest import mock @@ -27,7 +27,9 @@ import pytest import xarray as xr +from satpy.readers.fci_l1c_nc import FCIL1cNCFileHandler from satpy.tests.reader_tests.test_netcdf_utils import FakeNetCDF4FileHandler +from satpy.tests.utils import make_dataid class FakeFCIFileHandlerBase(FakeNetCDF4FileHandler): @@ -183,7 +185,7 @@ def _get_test_content_aux_data(self): # compute the last data entry to simulate the FCI caching data[list(AUX_DATA.values())[-1]] = data[list(AUX_DATA.values())[-1]].compute() - data['index'] = xrda(da.arange(indices_dim, dtype="uint16")+100, dims=("index")) + data['index'] = xrda(da.arange(indices_dim, dtype="uint16") + 100, dims=("index")) return data def _get_global_attributes(self): @@ -305,37 +307,55 @@ def _get_reader_with_filehandlers(filenames, reader_configs): return reader +@contextlib.contextmanager +def mocked_basefilehandler(filehandler): + p = mock.patch.object(FCIL1cNCFileHandler, "__bases__", (filehandler,)) + with p: + p.is_local = True + yield + class TestFCIL1cNCReader: """Initialize the unittest TestCase for the FCI L1c NetCDF Reader.""" - yaml_file = "fci_l1c_nc.yaml" class TestFCIL1cNCReaderGoodData(TestFCIL1cNCReader): """Test FCI L1c NetCDF reader with good data, on the FDHSI example.""" - _alt_handler_FDHSI = FakeFCIFileHandlerFDHSI - _alt_handler_HRFI = FakeFCIFileHandlerHRFI - # - # @pytest.fixture(autouse=True, scope="function") - # def fake_handler(self): - # """Wrap NetCDF4 FileHandler with our own fake handler.""" - # # implementation strongly inspired by test_viirs_l1b.py - # from satpy.readers.fci_l1c_nc import FCIL1cNCFileHandler - # p = mock.patch.object( - # FCIL1cNCFileHandler, - # "__bases__", - # (self._alt_handler,)) - # with p: - # p.is_local = True - # yield p - _chans_fdhsi = {"solar": ["vis_04", "vis_05", "vis_06", "vis_08", "vis_09", - "nir_13", "nir_16", "nir_22"], - "terran": ["ir_38", "wv_63", "wv_73", "ir_87", "ir_97", "ir_105", - "ir_123", "ir_133"]} + "nir_13", "nir_16", "nir_22"], + "terran": ["ir_38", "wv_63", "wv_73", "ir_87", "ir_97", "ir_105", + "ir_123", "ir_133"]} _chans_hrfi = {"solar": ["vis_06", "nir_22"], - "terran": ["ir_38", "ir_105"]} + "terran": ["ir_38", "ir_105"]} + + @pytest.mark.parametrize('filenames', [ + [ + "W_XX-EUMETSAT-Darmstadt,IMG+SAT,MTI1+FCI-1C-RRAD-FDHSI-FD--" + "CHK-BODY--L2P-NC4E_C_EUMT_20170410114434_GTT_DEV_" + "20170410113925_20170410113934_N__C_0070_0067.nc", + "W_XX-EUMETSAT-Darmstadt,IMG+SAT,MTI1+FCI-1C-RRAD-FDHSI-FD--" + "CHK-BODY--L2P-NC4E_C_EUMT_20170410114442_GTT_DEV_" + "20170410113934_20170410113942_N__C_0070_0068.nc", + ], + [ + "W_XX-EUMETSAT-Darmstadt,IMG+SAT,MTI1+FCI-1C-RRAD-HRFI-FD--" + "CHK-BODY--L2P-NC4E_C_EUMT_20170410114434_GTT_DEV_" + "20170410113925_20170410113934_N__C_0070_0067.nc", + "W_XX-EUMETSAT-Darmstadt,IMG+SAT,MTI1+FCI-1C-RRAD-HRFI-FD--" + "CHK-BODY--L2P-NC4E_C_EUMT_20170410114442_GTT_DEV_" + "20170410113934_20170410113942_N__C_0070_0068.nc", + ] + + ]) + def test_file_pattern(self, reader_configs, filenames): + """Test file pattern matching.""" + from satpy.readers import load_reader + + reader = load_reader(reader_configs) + files = reader.select_files_from_pathnames(filenames) + # only 2 should match, the TRAIL should be ignored + assert len(files) == 2 @pytest.mark.parametrize('filehandler,channels,filenames,expected_res_n', [ (FakeFCIFileHandlerFDHSI, _chans_fdhsi, [ @@ -359,19 +379,13 @@ class TestFCIL1cNCReaderGoodData(TestFCIL1cNCReader): def test_load_counts(self, reader_configs, filehandler, channels, filenames, expected_res_n): """Test loading with counts.""" - from satpy.tests.utils import make_dataid - self._alt_handler = filehandler - from satpy.readers.fci_l1c_nc import FCIL1cNCFileHandler - p = mock.patch.object(FCIL1cNCFileHandler, "__bases__", (self._alt_handler,)) - with p: - p.is_local = True - self._chans = channels + with mocked_basefilehandler(filehandler): reader = _get_reader_with_filehandlers(filenames, reader_configs) res = reader.load( [make_dataid(name=name, calibration="counts") for name in - self._chans["solar"] + self._chans["terran"]], pad_data=False) + channels["solar"] + channels["terran"]], pad_data=False) assert expected_res_n == len(res) - for ch in self._chans["solar"] + self._chans["terran"]: + for ch in channels["solar"] + channels["terran"]: assert res[ch].shape == (200 * 2, 11136) assert res[ch].dtype == np.uint16 assert res[ch].attrs["calibration"] == "counts" @@ -382,176 +396,192 @@ def test_load_counts(self, reader_configs, filehandler, channels, filenames, else: numpy.testing.assert_array_equal(res[ch], 1) - def test_file_pattern(self, reader_configs): - """Test file pattern matching.""" - from satpy.readers import load_reader - - filenames = [ + @pytest.mark.parametrize('filehandler,channels,filenames,expected_res_n', [ + (FakeFCIFileHandlerFDHSI, _chans_fdhsi, [ "W_XX-EUMETSAT-Darmstadt,IMG+SAT,MTI1+FCI-1C-RRAD-FDHSI-FD--" "CHK-BODY--L2P-NC4E_C_EUMT_20170410114434_GTT_DEV_" - "20170410113925_20170410113934_N__C_0070_0067.nc", - "W_XX-EUMETSAT-Darmstadt,IMG+SAT,MTI1+FCI-1C-RRAD-FDHSI-FD--" - "CHK-BODY--L2P-NC4E_C_EUMT_20170410114442_GTT_DEV_" - "20170410113934_20170410113942_N__C_0070_0068.nc", - "W_XX-EUMETSAT-Darmstadt,IMG+SAT,MTI1+FCI-1C-RRAD-FDHSI-FD--" - "CHK-BODY--L2P-NC4E_C_EUMT_20170410114451_GTT_DEV_" - "20170410113942_20170410113951_N__C_0070_0069.nc", - "W_XX-EUMETSAT-Darmstadt,IMG+SAT,MTI1+FCI-1C-RRAD-FDHSI-FD--" - "CHK-BODY--L2P-NC4E_C_EUMT_20170410114500_GTT_DEV_" - "20170410113951_20170410114000_N__C_0070_0070.nc", - # this is a TRAIL file, which we don't use/read - "W_XX-EUMETSAT-Darmstadt,IMG+SAT,MTI1+FCI-1C-RRAD-FDHSI-FD--" - "CHK-TRAIL--L2P-NC4E_C_EUMT_20170410114600_GTT_DEV_" - "20170410113000_20170410114000_N__C_0070_0071.nc", - ] - - reader = load_reader(reader_configs) - files = reader.select_files_from_pathnames(filenames) - # only 4 FDHSI should match, the TRAIL should be ignored - assert len(files) == 4 + "20170410113925_20170410113934_N__C_0070_0067.nc" + ], 16), + (FakeFCIFileHandlerHRFI, _chans_hrfi, [ + "W_XX-EUMETSAT-Darmstadt,IMG+SAT,MTI1+FCI-1C-RRAD-HRFI-FD--" + "CHK-BODY--L2P-NC4E_C_EUMT_20170410114434_GTT_DEV_" + "20170410113925_20170410113934_N__C_0070_0067.nc" + ], 4), - def test_load_radiance(self, reader_configs): + ]) + def test_load_radiance(self, reader_configs, filehandler, channels, filenames, + expected_res_n): """Test loading with radiance.""" - from satpy.tests.utils import make_dataid - filenames = [ + with mocked_basefilehandler(filehandler): + reader = _get_reader_with_filehandlers(filenames, reader_configs) + res = reader.load( + [make_dataid(name=name, calibration="radiance") for name in + channels["solar"] + channels["terran"]], pad_data=False) + assert expected_res_n == len(res) + for ch in channels["solar"] + channels["terran"]: + assert res[ch].shape == (200, 11136) + assert res[ch].dtype == np.float64 + assert res[ch].attrs["calibration"] == "radiance" + assert res[ch].attrs["units"] == 'mW m-2 sr-1 (cm-1)-1' + assert res[ch].attrs["radiance_unit_conversion_coefficient"] == 1234.56 + if ch == 'ir_38': + numpy.testing.assert_array_equal(res[ch][~0], 15) + numpy.testing.assert_array_equal(res[ch][0], 9700) + else: + numpy.testing.assert_array_equal(res[ch], 15) + + @pytest.mark.parametrize('filehandler,channels,filenames,expected_res_n', [ + (FakeFCIFileHandlerFDHSI, _chans_fdhsi, [ "W_XX-EUMETSAT-Darmstadt,IMG+SAT,MTI1+FCI-1C-RRAD-FDHSI-FD--" "CHK-BODY--L2P-NC4E_C_EUMT_20170410114434_GTT_DEV_" - "20170410113925_20170410113934_N__C_0070_0067.nc", - ] + "20170410113925_20170410113934_N__C_0070_0067.nc" + ], 8), + (FakeFCIFileHandlerHRFI, _chans_hrfi, [ + "W_XX-EUMETSAT-Darmstadt,IMG+SAT,MTI1+FCI-1C-RRAD-HRFI-FD--" + "CHK-BODY--L2P-NC4E_C_EUMT_20170410114434_GTT_DEV_" + "20170410113925_20170410113934_N__C_0070_0067.nc" + ], 2), - reader = _get_reader_with_filehandlers(filenames, reader_configs) - res = reader.load( - [make_dataid(name=name, calibration="radiance") for name in - self._chans["solar"] + self._chans["terran"]], pad_data=False) - assert 16 == len(res) - for ch in self._chans["solar"] + self._chans["terran"]: - assert res[ch].shape == (200, 11136) - assert res[ch].dtype == np.float64 - assert res[ch].attrs["calibration"] == "radiance" - assert res[ch].attrs["units"] == 'mW m-2 sr-1 (cm-1)-1' - assert res[ch].attrs["radiance_unit_conversion_coefficient"] == 1234.56 - if ch == 'ir_38': - numpy.testing.assert_array_equal(res[ch][~0], 15) - numpy.testing.assert_array_equal(res[ch][0], 9700) - else: - numpy.testing.assert_array_equal(res[ch], 15) - - def test_load_reflectance(self, reader_configs): + ]) + def test_load_reflectance(self, reader_configs, filehandler, channels, filenames, + expected_res_n): """Test loading with reflectance.""" - from satpy.tests.utils import make_dataid + with mocked_basefilehandler(filehandler): + reader = _get_reader_with_filehandlers(filenames, reader_configs) + res = reader.load( + [make_dataid(name=name, calibration="reflectance") for name in + channels["solar"]], pad_data=False) + assert expected_res_n == len(res) + for ch in channels["solar"]: + assert res[ch].shape == (200, 11136) + assert res[ch].dtype == np.float64 + assert res[ch].attrs["calibration"] == "reflectance" + assert res[ch].attrs["units"] == "%" + numpy.testing.assert_array_almost_equal(res[ch], 100 * 15 * 1 * np.pi / 50) - filenames = [ + @pytest.mark.parametrize('filehandler,channels,filenames,expected_res_n', [ + (FakeFCIFileHandlerFDHSI, _chans_fdhsi, [ "W_XX-EUMETSAT-Darmstadt,IMG+SAT,MTI1+FCI-1C-RRAD-FDHSI-FD--" "CHK-BODY--L2P-NC4E_C_EUMT_20170410114434_GTT_DEV_" - "20170410113925_20170410113934_N__C_0070_0067.nc", - ] + "20170410113925_20170410113934_N__C_0070_0067.nc" + ], 8), + (FakeFCIFileHandlerHRFI, _chans_hrfi, [ + "W_XX-EUMETSAT-Darmstadt,IMG+SAT,MTI1+FCI-1C-RRAD-HRFI-FD--" + "CHK-BODY--L2P-NC4E_C_EUMT_20170410114434_GTT_DEV_" + "20170410113925_20170410113934_N__C_0070_0067.nc" + ], 2), - reader = _get_reader_with_filehandlers(filenames, reader_configs) - res = reader.load( - [make_dataid(name=name, calibration="reflectance") for name in - self._chans["solar"]], pad_data=False) - assert 8 == len(res) - for ch in self._chans["solar"]: - assert res[ch].shape == (200, 11136) - assert res[ch].dtype == np.float64 - assert res[ch].attrs["calibration"] == "reflectance" - assert res[ch].attrs["units"] == "%" - numpy.testing.assert_array_almost_equal(res[ch], 100 * 15 * 1 * np.pi / 50) - - def test_load_bt(self, reader_configs, caplog): + ]) + def test_load_bt(self, reader_configs, caplog, filehandler, channels, filenames, + expected_res_n): """Test loading with bt.""" - from satpy.tests.utils import make_dataid + with mocked_basefilehandler(filehandler): + reader = _get_reader_with_filehandlers(filenames, reader_configs) + with caplog.at_level(logging.WARNING): + res = reader.load( + [make_dataid(name=name, calibration="brightness_temperature") for + name in channels["terran"]], pad_data=False) + assert caplog.text == "" + assert expected_res_n == len(res) + for ch in channels["terran"]: + assert res[ch].shape == (200, 11136) + assert res[ch].dtype == np.float64 + assert res[ch].attrs["calibration"] == "brightness_temperature" + assert res[ch].attrs["units"] == "K" - filenames = [ + if ch == 'ir_38': + numpy.testing.assert_array_almost_equal(res[ch][~0], 209.68274099) + numpy.testing.assert_array_almost_equal(res[ch][0], 1888.851296) + else: + numpy.testing.assert_array_almost_equal(res[ch], 209.68274099) + + @pytest.mark.parametrize('filehandler,channels,filenames', [ + (FakeFCIFileHandlerFDHSI, _chans_fdhsi, [ "W_XX-EUMETSAT-Darmstadt,IMG+SAT,MTI1+FCI-1C-RRAD-FDHSI-FD--" "CHK-BODY--L2P-NC4E_C_EUMT_20170410114434_GTT_DEV_" - "20170410113925_20170410113934_N__C_0070_0067.nc", - ] + "20170410113925_20170410113934_N__C_0070_0067.nc" + ]), + (FakeFCIFileHandlerHRFI, _chans_hrfi, [ + "W_XX-EUMETSAT-Darmstadt,IMG+SAT,MTI1+FCI-1C-RRAD-HRFI-FD--" + "CHK-BODY--L2P-NC4E_C_EUMT_20170410114434_GTT_DEV_" + "20170410113925_20170410113934_N__C_0070_0067.nc" + ]), - reader = _get_reader_with_filehandlers(filenames, reader_configs) - with caplog.at_level(logging.WARNING): - res = reader.load( - [make_dataid(name=name, calibration="brightness_temperature") for - name in self._chans["terran"]], pad_data=False) - assert caplog.text == "" - for ch in self._chans["terran"]: - assert res[ch].shape == (200, 11136) - assert res[ch].dtype == np.float64 - assert res[ch].attrs["calibration"] == "brightness_temperature" - assert res[ch].attrs["units"] == "K" - - if ch == 'ir_38': - numpy.testing.assert_array_almost_equal(res[ch][~0], 209.68274099) - numpy.testing.assert_array_almost_equal(res[ch][0], 1888.851296) - else: - numpy.testing.assert_array_almost_equal(res[ch], 209.68274099) - - def test_orbital_parameters_attr(self, reader_configs): + ]) + def test_orbital_parameters_attr(self, reader_configs, filehandler, channels, filenames): """Test the orbital parameter attribute.""" - from satpy.tests.utils import make_dataid + with mocked_basefilehandler(filehandler): + reader = _get_reader_with_filehandlers(filenames, reader_configs) + res = reader.load( + [make_dataid(name=name) for name in + channels["solar"] + channels["terran"]], pad_data=False) + + for ch in channels["solar"] + channels["terran"]: + assert res[ch].attrs["orbital_parameters"] == { + 'satellite_actual_longitude': np.mean(np.arange(6000)), + 'satellite_actual_latitude': np.mean(np.arange(6000)), + 'satellite_actual_altitude': np.mean(np.arange(6000)), + 'satellite_nominal_longitude': 0.0, + 'satellite_nominal_latitude': 0, + 'satellite_nominal_altitude': 35786400.0, + 'projection_longitude': 0.0, + 'projection_latitude': 0, + 'projection_altitude': 35786400.0, + } - filenames = [ + @pytest.mark.parametrize('filehandler,channels,filenames,expected_res_n', [ + (FakeFCIFileHandlerFDHSI, _chans_fdhsi, [ "W_XX-EUMETSAT-Darmstadt,IMG+SAT,MTI1+FCI-1C-RRAD-FDHSI-FD--" "CHK-BODY--L2P-NC4E_C_EUMT_20170410114434_GTT_DEV_" - "20170410113925_20170410113934_N__C_0070_0067.nc", - ] - - reader = _get_reader_with_filehandlers(filenames, reader_configs) - res = reader.load( - [make_dataid(name=name) for name in - self._chans["solar"] + self._chans["terran"]], pad_data=False) - - for ch in self._chans["solar"] + self._chans["terran"]: - assert res[ch].attrs["orbital_parameters"] == { - 'satellite_actual_longitude': np.mean(np.arange(6000)), - 'satellite_actual_latitude': np.mean(np.arange(6000)), - 'satellite_actual_altitude': np.mean(np.arange(6000)), - 'satellite_nominal_longitude': 0.0, - 'satellite_nominal_latitude': 0, - 'satellite_nominal_altitude': 35786400.0, - 'projection_longitude': 0.0, - 'projection_latitude': 0, - 'projection_altitude': 35786400.0, - } - - def test_load_index_map(self, reader_configs): - """Test loading of index_map.""" - filenames = [ - "W_XX-EUMETSAT-Darmstadt,IMG+SAT,MTI1+FCI-1C-RRAD-FDHSI-FD--" + "20170410113925_20170410113934_N__C_0070_0067.nc" + ], 16), + (FakeFCIFileHandlerHRFI, _chans_hrfi, [ + "W_XX-EUMETSAT-Darmstadt,IMG+SAT,MTI1+FCI-1C-RRAD-HRFI-FD--" "CHK-BODY--L2P-NC4E_C_EUMT_20170410114434_GTT_DEV_" "20170410113925_20170410113934_N__C_0070_0067.nc" - ] + ], 4), - reader = _get_reader_with_filehandlers(filenames, reader_configs) - res = reader.load( - [name + '_index_map' for name in - self._chans["solar"] + self._chans["terran"]], pad_data=False) - assert 16 == len(res) - for ch in self._chans["solar"] + self._chans["terran"]: - assert res[ch + '_index_map'].shape == (200, 11136) - numpy.testing.assert_array_equal(res[ch + '_index_map'][1, 1], 5237) - - def test_load_aux_data(self, reader_configs): - """Test loading of auxiliary data.""" - from satpy.readers.fci_l1c_nc import AUX_DATA + ]) + def test_load_index_map(self, reader_configs, filehandler, channels, filenames, expected_res_n): + """Test loading of index_map.""" + with mocked_basefilehandler(filehandler): + reader = _get_reader_with_filehandlers(filenames, reader_configs) + res = reader.load( + [name + '_index_map' for name in + channels["solar"] + channels["terran"]], pad_data=False) + assert expected_res_n == len(res) + for ch in channels["solar"] + channels["terran"]: + assert res[ch + '_index_map'].shape == (200, 11136) + numpy.testing.assert_array_equal(res[ch + '_index_map'][1, 1], 5237) - filenames = [ + @pytest.mark.parametrize('filehandler,filenames', [ + (FakeFCIFileHandlerFDHSI, [ "W_XX-EUMETSAT-Darmstadt,IMG+SAT,MTI1+FCI-1C-RRAD-FDHSI-FD--" "CHK-BODY--L2P-NC4E_C_EUMT_20170410114434_GTT_DEV_" "20170410113925_20170410113934_N__C_0070_0067.nc" - ] + ]), + (FakeFCIFileHandlerHRFI, [ + "W_XX-EUMETSAT-Darmstadt,IMG+SAT,MTI1+FCI-1C-RRAD-HRFI-FD--" + "CHK-BODY--L2P-NC4E_C_EUMT_20170410114434_GTT_DEV_" + "20170410113925_20170410113934_N__C_0070_0067.nc" + ]), - reader = _get_reader_with_filehandlers(filenames, reader_configs) - res = reader.load(['vis_04_' + key for key in AUX_DATA.keys()], - pad_data=False) - for aux in ['vis_04_' + key for key in AUX_DATA.keys()]: + ]) + def test_load_aux_data(self, reader_configs, filehandler, filenames): + """Test loading of auxiliary data.""" + from satpy.readers.fci_l1c_nc import AUX_DATA + with mocked_basefilehandler(filehandler): + reader = _get_reader_with_filehandlers(filenames, reader_configs) + res = reader.load(['vis_06_' + key for key in AUX_DATA.keys()], + pad_data=False) + for aux in ['vis_06_' + key for key in AUX_DATA.keys()]: - assert res[aux].shape == (200, 11136) - if aux == 'vis_04_earth_sun_distance': - numpy.testing.assert_array_equal(res[aux][1, 1], 149597870.7) - else: - numpy.testing.assert_array_equal(res[aux][1, 1], 5137) + assert res[aux].shape == (200, 11136) + if aux == 'vis_06_earth_sun_distance': + numpy.testing.assert_array_equal(res[aux][1, 1], 149597870.7) + else: + numpy.testing.assert_array_equal(res[aux][1, 1], 5137) def test_load_composite(self): """Test that composites are loadable.""" @@ -564,86 +594,107 @@ def test_load_composite(self): assert len(comps["fci"]) > 0 assert len(mods["fci"]) > 0 - def test_load_quality_only(self, reader_configs): - """Test that loading quality only works.""" - filenames = [ + @pytest.mark.parametrize('filehandler,channels,filenames,expected_res_n', [ + (FakeFCIFileHandlerFDHSI, _chans_fdhsi, [ "W_XX-EUMETSAT-Darmstadt,IMG+SAT,MTI1+FCI-1C-RRAD-FDHSI-FD--" "CHK-BODY--L2P-NC4E_C_EUMT_20170410114434_GTT_DEV_" - "20170410113925_20170410113934_N__C_0070_0067.nc", - ] + "20170410113925_20170410113934_N__C_0070_0067.nc" + ], 16), + (FakeFCIFileHandlerHRFI, _chans_hrfi, [ + "W_XX-EUMETSAT-Darmstadt,IMG+SAT,MTI1+FCI-1C-RRAD-HRFI-FD--" + "CHK-BODY--L2P-NC4E_C_EUMT_20170410114434_GTT_DEV_" + "20170410113925_20170410113934_N__C_0070_0067.nc" + ], 4), - reader = _get_reader_with_filehandlers(filenames, reader_configs) - res = reader.load( - [name + '_pixel_quality' for name in - self._chans["solar"] + self._chans["terran"]], pad_data=False) - assert 16 == len(res) - for ch in self._chans["solar"] + self._chans["terran"]: - assert res[ch + '_pixel_quality'].shape == (200, 11136) - numpy.testing.assert_array_equal(res[ch + '_pixel_quality'][1, 1], 1) - assert res[ch + '_pixel_quality'].attrs["name"] == ch + '_pixel_quality' - - def test_platform_name(self, reader_configs): + ]) + def test_load_quality_only(self, reader_configs, filehandler, channels, filenames, expected_res_n): + """Test that loading quality only works.""" + with mocked_basefilehandler(filehandler): + reader = _get_reader_with_filehandlers(filenames, reader_configs) + res = reader.load( + [name + '_pixel_quality' for name in + channels["solar"] + channels["terran"]], pad_data=False) + assert expected_res_n == len(res) + for ch in channels["solar"] + channels["terran"]: + assert res[ch + '_pixel_quality'].shape == (200, 11136) + numpy.testing.assert_array_equal(res[ch + '_pixel_quality'][1, 1], 1) + assert res[ch + '_pixel_quality'].attrs["name"] == ch + '_pixel_quality' + + @pytest.mark.parametrize('filehandler,filenames', [ + (FakeFCIFileHandlerFDHSI, [ + "W_XX-EUMETSAT-Darmstadt,IMG+SAT,MTI1+FCI-1C-RRAD-FDHSI-FD--" + "CHK-BODY--L2P-NC4E_C_EUMT_20170410114434_GTT_DEV_" + "20170410113925_20170410113934_N__C_0070_0067.nc" + ]), + (FakeFCIFileHandlerHRFI, [ + "W_XX-EUMETSAT-Darmstadt,IMG+SAT,MTI1+FCI-1C-RRAD-HRFI-FD--" + "CHK-BODY--L2P-NC4E_C_EUMT_20170410114434_GTT_DEV_" + "20170410113925_20170410113934_N__C_0070_0067.nc" + ]), + ]) + def test_platform_name(self, reader_configs, filehandler, filenames): """Test that platform name is exposed. Test that the FCI reader exposes the platform name. Corresponds to GH issue 1014. """ - filenames = [ - "W_XX-EUMETSAT-Darmstadt,IMG+SAT,MTI1+FCI-1C-RRAD-FDHSI-FD--" - "CHK-BODY--L2P-NC4E_C_EUMT_20170410114434_GTT_DEV_" - "20170410113925_20170410113934_N__C_0070_0067.nc", - ] - - reader = _get_reader_with_filehandlers(filenames, reader_configs) - res = reader.load(["ir_123"], pad_data=False) - assert res["ir_123"].attrs["platform_name"] == "MTG-I1" + with mocked_basefilehandler(filehandler): + reader = _get_reader_with_filehandlers(filenames, reader_configs) + res = reader.load(["vis_06"], pad_data=False) + assert res["vis_06"].attrs["platform_name"] == "MTG-I1" def test_excs(self, reader_configs): """Test that exceptions are raised where expected.""" - from satpy.tests.utils import make_dataid filenames = [ "W_XX-EUMETSAT-Darmstadt,IMG+SAT,MTI1+FCI-1C-RRAD-FDHSI-FD--" "CHK-BODY--L2P-NC4E_C_EUMT_20170410114434_GTT_DEV_" "20170410113925_20170410113934_N__C_0070_0067.nc", ] + with mocked_basefilehandler(FakeFCIFileHandlerFDHSI): + reader = _get_reader_with_filehandlers(filenames, reader_configs) - reader = _get_reader_with_filehandlers(filenames, reader_configs) - - with pytest.raises(ValueError): - reader.file_handlers["fci_l1c_fdhsi"][0].get_dataset(make_dataid(name="invalid"), {}) - with pytest.raises(ValueError): - reader.file_handlers["fci_l1c_fdhsi"][0].get_dataset( - make_dataid(name="ir_123", calibration="unknown"), - {"units": "unknown"}) + with pytest.raises(ValueError): + reader.file_handlers["fci_l1c_fdhsi"][0].get_dataset(make_dataid(name="invalid"), {}) + with pytest.raises(ValueError): + reader.file_handlers["fci_l1c_fdhsi"][0].get_dataset( + make_dataid(name="ir_123", calibration="unknown"), + {"units": "unknown"}) - def test_area_definition_computation(self, reader_configs): - """Test that the geolocation computation is correct.""" - filenames = [ + @pytest.mark.parametrize('filehandler,filenames, expected_area', [ + (FakeFCIFileHandlerFDHSI, [ "W_XX-EUMETSAT-Darmstadt,IMG+SAT,MTI1+FCI-1C-RRAD-FDHSI-FD--" "CHK-BODY--L2P-NC4E_C_EUMT_20170410114434_GTT_DEV_" - "20170410113925_20170410113934_N__C_0070_0067.nc", - ] - - reader = _get_reader_with_filehandlers(filenames, reader_configs) - res = reader.load(['ir_105', 'vis_06'], pad_data=False) + "20170410113925_20170410113934_N__C_0070_0067.nc" + ], ['mtg_fci_fdss_1km', 'mtg_fci_fdss_2km']), + (FakeFCIFileHandlerHRFI, [ + "W_XX-EUMETSAT-Darmstadt,IMG+SAT,MTI1+FCI-1C-RRAD-HRFI-FD--" + "CHK-BODY--L2P-NC4E_C_EUMT_20170410114434_GTT_DEV_" + "20170410113925_20170410113934_N__C_0070_0067.nc" + ], ['mtg_fci_fdss_500m', 'mtg_fci_fdss_1km']), + ]) + def test_area_definition_computation(self, reader_configs, filehandler, filenames, expected_area): + """Test that the geolocation computation is correct.""" + with mocked_basefilehandler(filehandler): + reader = _get_reader_with_filehandlers(filenames, reader_configs) + res = reader.load(['ir_105', 'vis_06'], pad_data=False) - # test that area_ids are harmonisation-conform ___ - assert res['vis_06'].attrs['area'].area_id == 'mtg_fci_fdss_1km' - assert res['ir_105'].attrs['area'].area_id == 'mtg_fci_fdss_2km' + # test that area_ids are harmonisation-conform ___ + assert res['vis_06'].attrs['area'].area_id == expected_area[0] + assert res['ir_105'].attrs['area'].area_id == expected_area[1] - area_def = res['ir_105'].attrs['area'] - # test area extents computation - np.testing.assert_array_almost_equal(np.array(area_def.area_extent), - np.array([-5568062.23065902, 5168057.7600648, - 16704186.692027, 5568062.23065902])) + area_def = res['ir_105'].attrs['area'] + # test area extents computation + np.testing.assert_array_almost_equal(np.array(area_def.area_extent), + np.array([-5568062.23065902, 5168057.7600648, + 16704186.692027, 5568062.23065902])) - # check that the projection is read in properly - assert area_def.crs.coordinate_operation.method_name == 'Geostationary Satellite (Sweep Y)' - assert area_def.crs.coordinate_operation.params[0].value == 0.0 # projection origin longitude - assert area_def.crs.coordinate_operation.params[1].value == 35786400.0 # projection height - assert area_def.crs.ellipsoid.semi_major_metre == 6378137.0 - assert area_def.crs.ellipsoid.inverse_flattening == 298.257223563 - assert area_def.crs.ellipsoid.is_semi_minor_computed + # check that the projection is read in properly + assert area_def.crs.coordinate_operation.method_name == 'Geostationary Satellite (Sweep Y)' + assert area_def.crs.coordinate_operation.params[0].value == 0.0 # projection origin longitude + assert area_def.crs.coordinate_operation.params[1].value == 35786400.0 # projection height + assert area_def.crs.ellipsoid.semi_major_metre == 6378137.0 + assert area_def.crs.ellipsoid.inverse_flattening == 298.257223563 + assert area_def.crs.ellipsoid.is_semi_minor_computed class TestFCIL1cNCReaderBadData(TestFCIL1cNCReader): @@ -723,7 +774,6 @@ def test_bad_xy_coords(self, reader_configs): np.array([-5568062.270889, 5168057.806632, 16704186.298937, 5568062.270889])) - # class TestFCIL1cNCReaderHRFI(TestFCIL1cNCReaderGoodData): # """Test FCI L1c NetCDF reader for the HRFI case.""" # From 8205b9beabcce16b8074346cb55f5552207425d5 Mon Sep 17 00:00:00 2001 From: andream Date: Fri, 18 Nov 2022 10:17:17 +0100 Subject: [PATCH 06/26] update the bad data tests --- satpy/tests/reader_tests/test_fci_l1c_nc.py | 377 ++++++-------------- 1 file changed, 107 insertions(+), 270 deletions(-) diff --git a/satpy/tests/reader_tests/test_fci_l1c_nc.py b/satpy/tests/reader_tests/test_fci_l1c_nc.py index 739abe6043..c0855af10f 100644 --- a/satpy/tests/reader_tests/test_fci_l1c_nc.py +++ b/satpy/tests/reader_tests/test_fci_l1c_nc.py @@ -35,6 +35,9 @@ class FakeFCIFileHandlerBase(FakeNetCDF4FileHandler): """Class for faking the NetCDF4 Filehandler.""" + # overwritten by FDHSI and HRFI FIle Handlers + chan_patterns = {} + def _get_test_calib_for_channel_ir(self, chroot, meas): from pyspectral.blackbody import C_SPEED as c from pyspectral.blackbody import H_PLANCK as h @@ -145,7 +148,11 @@ def _get_test_content_for_channel(self, pat, ch): return data def _get_test_content_all_channels(self): - raise NotImplementedError + data = {} + for pat in self.chan_patterns: + for ch_num in self.chan_patterns[pat]: + data.update(self._get_test_content_for_channel(pat, ch_num)) + return data def _get_test_content_areadef(self): data = {} @@ -216,18 +223,12 @@ def get_test_content(self, filename, filename_info, filetype_info): class FakeFCIFileHandlerFDHSI(FakeFCIFileHandlerBase): """Mock FDHSI data.""" - def _get_test_content_all_channels(self): - chan_patterns = { + chan_patterns = { "vis_{:>02d}": (4, 5, 6, 8, 9), "nir_{:>02d}": (13, 16, 22), "ir_{:>02d}": (38, 87, 97, 105, 123, 133), "wv_{:>02d}": (63, 73), } - data = {} - for pat in chan_patterns: - for ch_num in chan_patterns[pat]: - data.update(self._get_test_content_for_channel(pat, ch_num)) - return data class FakeFCIFileHandlerWithBadData(FakeFCIFileHandlerFDHSI): @@ -236,12 +237,11 @@ class FakeFCIFileHandlerWithBadData(FakeFCIFileHandlerFDHSI): def _get_test_calib_for_channel_ir(self, chroot, meas): from netCDF4 import default_fillvals v = xr.DataArray(default_fillvals["f4"]) - data = {} - data[meas + "/radiance_to_bt_conversion_coefficient_wavenumber"] = v - data[meas + "/radiance_to_bt_conversion_coefficient_a"] = v - data[meas + "/radiance_to_bt_conversion_coefficient_b"] = v - data[meas + "/radiance_to_bt_conversion_constant_c1"] = v - data[meas + "/radiance_to_bt_conversion_constant_c2"] = v + data = {meas + "/radiance_to_bt_conversion_coefficient_wavenumber": v, + meas + "/radiance_to_bt_conversion_coefficient_a": v, + meas + "/radiance_to_bt_conversion_coefficient_b": v, + meas + "/radiance_to_bt_conversion_constant_c1": v, + meas + "/radiance_to_bt_conversion_constant_c2": v} return data def _get_test_calib_for_channel_vis(self, chroot, meas): @@ -262,15 +262,15 @@ def _get_test_calib_for_channel_vis(self, chroot, meas): def _get_test_content_all_channels(self): data = super()._get_test_content_all_channels() - data['data/vis_04/measured/x'].attrs['scale_factor'] *= -1 - data['data/vis_04/measured/x'].attrs['scale_factor'] = \ - np.float32(data['data/vis_04/measured/x'].attrs['scale_factor']) - data['data/vis_04/measured/x'].attrs['add_offset'] = \ - np.float32(data['data/vis_04/measured/x'].attrs['add_offset']) - data['data/vis_04/measured/y'].attrs['scale_factor'] = \ - np.float32(data['data/vis_04/measured/y'].attrs['scale_factor']) - data['data/vis_04/measured/y'].attrs['add_offset'] = \ - np.float32(data['data/vis_04/measured/y'].attrs['add_offset']) + data['data/vis_06/measured/x'].attrs['scale_factor'] *= -1 + data['data/vis_06/measured/x'].attrs['scale_factor'] = \ + np.float32(data['data/vis_06/measured/x'].attrs['scale_factor']) + data['data/vis_06/measured/x'].attrs['add_offset'] = \ + np.float32(data['data/vis_06/measured/x'].attrs['add_offset']) + data['data/vis_06/measured/y'].attrs['scale_factor'] = \ + np.float32(data['data/vis_06/measured/y'].attrs['scale_factor']) + data['data/vis_06/measured/y'].attrs['add_offset'] = \ + np.float32(data['data/vis_06/measured/y'].attrs['add_offset']) return data @@ -278,18 +278,11 @@ def _get_test_content_all_channels(self): class FakeFCIFileHandlerHRFI(FakeFCIFileHandlerBase): """Mock HRFI data.""" - def _get_test_content_all_channels(self): - chan_patterns = { + chan_patterns = { "vis_{:>02d}_hr": [6], "nir_{:>02d}_hr": [22], "ir_{:>02d}_hr": [38, 105], } - data = {} - for pat in chan_patterns: - for ch_num in chan_patterns[pat]: - data.update(self._get_test_content_for_channel(pat, ch_num)) - return data - @pytest.fixture def reader_configs(): @@ -307,6 +300,26 @@ def _get_reader_with_filehandlers(filenames, reader_configs): return reader +_chans_fdhsi = {"solar": ["vis_04", "vis_05", "vis_06", "vis_08", "vis_09", + "nir_13", "nir_16", "nir_22"], + "terran": ["ir_38", "wv_63", "wv_73", "ir_87", "ir_97", "ir_105", + "ir_123", "ir_133"]} + +_chans_hrfi = {"solar": ["vis_06", "nir_22"], + "terran": ["ir_38", "ir_105"]} + +_test_filenames = {'fdhsi' : [ + "W_XX-EUMETSAT-Darmstadt,IMG+SAT,MTI1+FCI-1C-RRAD-FDHSI-FD--" + "CHK-BODY--L2P-NC4E_C_EUMT_20170410114434_GTT_DEV_" + "20170410113925_20170410113934_N__C_0070_0067.nc" + ], +'hrfi': [ + "W_XX-EUMETSAT-Darmstadt,IMG+SAT,MTI1+FCI-1C-RRAD-HRFI-FD--" + "CHK-BODY--L2P-NC4E_C_EUMT_20170410114434_GTT_DEV_" + "20170410113925_20170410113934_N__C_0070_0067.nc" + ] +} + @contextlib.contextmanager def mocked_basefilehandler(filehandler): p = mock.patch.object(FCIL1cNCFileHandler, "__bases__", (filehandler,)) @@ -321,59 +334,30 @@ class TestFCIL1cNCReader: class TestFCIL1cNCReaderGoodData(TestFCIL1cNCReader): """Test FCI L1c NetCDF reader with good data, on the FDHSI example.""" - _chans_fdhsi = {"solar": ["vis_04", "vis_05", "vis_06", "vis_08", "vis_09", - "nir_13", "nir_16", "nir_22"], - "terran": ["ir_38", "wv_63", "wv_73", "ir_87", "ir_97", "ir_105", - "ir_123", "ir_133"]} - - _chans_hrfi = {"solar": ["vis_06", "nir_22"], - "terran": ["ir_38", "ir_105"]} - - @pytest.mark.parametrize('filenames', [ - [ - "W_XX-EUMETSAT-Darmstadt,IMG+SAT,MTI1+FCI-1C-RRAD-FDHSI-FD--" - "CHK-BODY--L2P-NC4E_C_EUMT_20170410114434_GTT_DEV_" - "20170410113925_20170410113934_N__C_0070_0067.nc", - "W_XX-EUMETSAT-Darmstadt,IMG+SAT,MTI1+FCI-1C-RRAD-FDHSI-FD--" - "CHK-BODY--L2P-NC4E_C_EUMT_20170410114442_GTT_DEV_" - "20170410113934_20170410113942_N__C_0070_0068.nc", - ], - [ - "W_XX-EUMETSAT-Darmstadt,IMG+SAT,MTI1+FCI-1C-RRAD-HRFI-FD--" - "CHK-BODY--L2P-NC4E_C_EUMT_20170410114434_GTT_DEV_" - "20170410113925_20170410113934_N__C_0070_0067.nc", - "W_XX-EUMETSAT-Darmstadt,IMG+SAT,MTI1+FCI-1C-RRAD-HRFI-FD--" - "CHK-BODY--L2P-NC4E_C_EUMT_20170410114442_GTT_DEV_" - "20170410113934_20170410113942_N__C_0070_0068.nc", - ] - ]) + + @pytest.mark.parametrize('filenames', [_test_filenames['fdhsi'], _test_filenames['hrfi']]) def test_file_pattern(self, reader_configs, filenames): """Test file pattern matching.""" from satpy.readers import load_reader reader = load_reader(reader_configs) files = reader.select_files_from_pathnames(filenames) - # only 2 should match, the TRAIL should be ignored - assert len(files) == 2 + assert len(files) == 1 + + @pytest.mark.parametrize('filenames', [_test_filenames['fdhsi'][0].replace('BODY', 'TRAIL'), + _test_filenames['hrfi'][0].replace('BODY', 'TRAIL')]) + def test_file_pattern_for_TRAIL_file(self, reader_configs, filenames): + """Test file pattern matching for TRAIL files, which should not be picked up.""" + from satpy.readers import load_reader + + reader = load_reader(reader_configs) + files = reader.select_files_from_pathnames(filenames) + assert len(files) == 0 @pytest.mark.parametrize('filehandler,channels,filenames,expected_res_n', [ - (FakeFCIFileHandlerFDHSI, _chans_fdhsi, [ - "W_XX-EUMETSAT-Darmstadt,IMG+SAT,MTI1+FCI-1C-RRAD-FDHSI-FD--" - "CHK-BODY--L2P-NC4E_C_EUMT_20170410114434_GTT_DEV_" - "20170410113925_20170410113934_N__C_0070_0067.nc", - "W_XX-EUMETSAT-Darmstadt,IMG+SAT,MTI1+FCI-1C-RRAD-FDHSI-FD--" - "CHK-BODY--L2P-NC4E_C_EUMT_20170410114442_GTT_DEV_" - "20170410113934_20170410113942_N__C_0070_0068.nc", - ], 16), - (FakeFCIFileHandlerHRFI, _chans_hrfi, [ - "W_XX-EUMETSAT-Darmstadt,IMG+SAT,MTI1+FCI-1C-RRAD-HRFI-FD--" - "CHK-BODY--L2P-NC4E_C_EUMT_20170410114434_GTT_DEV_" - "20170410113925_20170410113934_N__C_0070_0067.nc", - "W_XX-EUMETSAT-Darmstadt,IMG+SAT,MTI1+FCI-1C-RRAD-HRFI-FD--" - "CHK-BODY--L2P-NC4E_C_EUMT_20170410114442_GTT_DEV_" - "20170410113934_20170410113942_N__C_0070_0068.nc", - ], 4), + (FakeFCIFileHandlerFDHSI, _chans_fdhsi, _test_filenames['fdhsi'], 16), + (FakeFCIFileHandlerHRFI, _chans_hrfi, _test_filenames['hrfi'], 4), ]) def test_load_counts(self, reader_configs, filehandler, channels, filenames, @@ -386,7 +370,7 @@ def test_load_counts(self, reader_configs, filehandler, channels, filenames, channels["solar"] + channels["terran"]], pad_data=False) assert expected_res_n == len(res) for ch in channels["solar"] + channels["terran"]: - assert res[ch].shape == (200 * 2, 11136) + assert res[ch].shape == (200, 11136) assert res[ch].dtype == np.uint16 assert res[ch].attrs["calibration"] == "counts" assert res[ch].attrs["units"] == "count" @@ -397,16 +381,8 @@ def test_load_counts(self, reader_configs, filehandler, channels, filenames, numpy.testing.assert_array_equal(res[ch], 1) @pytest.mark.parametrize('filehandler,channels,filenames,expected_res_n', [ - (FakeFCIFileHandlerFDHSI, _chans_fdhsi, [ - "W_XX-EUMETSAT-Darmstadt,IMG+SAT,MTI1+FCI-1C-RRAD-FDHSI-FD--" - "CHK-BODY--L2P-NC4E_C_EUMT_20170410114434_GTT_DEV_" - "20170410113925_20170410113934_N__C_0070_0067.nc" - ], 16), - (FakeFCIFileHandlerHRFI, _chans_hrfi, [ - "W_XX-EUMETSAT-Darmstadt,IMG+SAT,MTI1+FCI-1C-RRAD-HRFI-FD--" - "CHK-BODY--L2P-NC4E_C_EUMT_20170410114434_GTT_DEV_" - "20170410113925_20170410113934_N__C_0070_0067.nc" - ], 4), + (FakeFCIFileHandlerFDHSI, _chans_fdhsi, _test_filenames['fdhsi'] , 16), + (FakeFCIFileHandlerHRFI, _chans_hrfi, _test_filenames['hrfi'] , 4), ]) def test_load_radiance(self, reader_configs, filehandler, channels, filenames, @@ -432,16 +408,8 @@ def test_load_radiance(self, reader_configs, filehandler, channels, filenames, numpy.testing.assert_array_equal(res[ch], 15) @pytest.mark.parametrize('filehandler,channels,filenames,expected_res_n', [ - (FakeFCIFileHandlerFDHSI, _chans_fdhsi, [ - "W_XX-EUMETSAT-Darmstadt,IMG+SAT,MTI1+FCI-1C-RRAD-FDHSI-FD--" - "CHK-BODY--L2P-NC4E_C_EUMT_20170410114434_GTT_DEV_" - "20170410113925_20170410113934_N__C_0070_0067.nc" - ], 8), - (FakeFCIFileHandlerHRFI, _chans_hrfi, [ - "W_XX-EUMETSAT-Darmstadt,IMG+SAT,MTI1+FCI-1C-RRAD-HRFI-FD--" - "CHK-BODY--L2P-NC4E_C_EUMT_20170410114434_GTT_DEV_" - "20170410113925_20170410113934_N__C_0070_0067.nc" - ], 2), + (FakeFCIFileHandlerFDHSI, _chans_fdhsi, _test_filenames['fdhsi'], 8), + (FakeFCIFileHandlerHRFI, _chans_hrfi, _test_filenames['hrfi'], 2), ]) def test_load_reflectance(self, reader_configs, filehandler, channels, filenames, @@ -461,16 +429,8 @@ def test_load_reflectance(self, reader_configs, filehandler, channels, filenames numpy.testing.assert_array_almost_equal(res[ch], 100 * 15 * 1 * np.pi / 50) @pytest.mark.parametrize('filehandler,channels,filenames,expected_res_n', [ - (FakeFCIFileHandlerFDHSI, _chans_fdhsi, [ - "W_XX-EUMETSAT-Darmstadt,IMG+SAT,MTI1+FCI-1C-RRAD-FDHSI-FD--" - "CHK-BODY--L2P-NC4E_C_EUMT_20170410114434_GTT_DEV_" - "20170410113925_20170410113934_N__C_0070_0067.nc" - ], 8), - (FakeFCIFileHandlerHRFI, _chans_hrfi, [ - "W_XX-EUMETSAT-Darmstadt,IMG+SAT,MTI1+FCI-1C-RRAD-HRFI-FD--" - "CHK-BODY--L2P-NC4E_C_EUMT_20170410114434_GTT_DEV_" - "20170410113925_20170410113934_N__C_0070_0067.nc" - ], 2), + (FakeFCIFileHandlerFDHSI, _chans_fdhsi, _test_filenames['fdhsi'], 8), + (FakeFCIFileHandlerHRFI, _chans_hrfi, _test_filenames['hrfi'], 2), ]) def test_load_bt(self, reader_configs, caplog, filehandler, channels, filenames, @@ -497,16 +457,8 @@ def test_load_bt(self, reader_configs, caplog, filehandler, channels, filenames, numpy.testing.assert_array_almost_equal(res[ch], 209.68274099) @pytest.mark.parametrize('filehandler,channels,filenames', [ - (FakeFCIFileHandlerFDHSI, _chans_fdhsi, [ - "W_XX-EUMETSAT-Darmstadt,IMG+SAT,MTI1+FCI-1C-RRAD-FDHSI-FD--" - "CHK-BODY--L2P-NC4E_C_EUMT_20170410114434_GTT_DEV_" - "20170410113925_20170410113934_N__C_0070_0067.nc" - ]), - (FakeFCIFileHandlerHRFI, _chans_hrfi, [ - "W_XX-EUMETSAT-Darmstadt,IMG+SAT,MTI1+FCI-1C-RRAD-HRFI-FD--" - "CHK-BODY--L2P-NC4E_C_EUMT_20170410114434_GTT_DEV_" - "20170410113925_20170410113934_N__C_0070_0067.nc" - ]), + (FakeFCIFileHandlerFDHSI, _chans_fdhsi, _test_filenames['fdhsi']), + (FakeFCIFileHandlerHRFI, _chans_hrfi, _test_filenames['hrfi']), ]) def test_orbital_parameters_attr(self, reader_configs, filehandler, channels, filenames): @@ -531,16 +483,8 @@ def test_orbital_parameters_attr(self, reader_configs, filehandler, channels, fi } @pytest.mark.parametrize('filehandler,channels,filenames,expected_res_n', [ - (FakeFCIFileHandlerFDHSI, _chans_fdhsi, [ - "W_XX-EUMETSAT-Darmstadt,IMG+SAT,MTI1+FCI-1C-RRAD-FDHSI-FD--" - "CHK-BODY--L2P-NC4E_C_EUMT_20170410114434_GTT_DEV_" - "20170410113925_20170410113934_N__C_0070_0067.nc" - ], 16), - (FakeFCIFileHandlerHRFI, _chans_hrfi, [ - "W_XX-EUMETSAT-Darmstadt,IMG+SAT,MTI1+FCI-1C-RRAD-HRFI-FD--" - "CHK-BODY--L2P-NC4E_C_EUMT_20170410114434_GTT_DEV_" - "20170410113925_20170410113934_N__C_0070_0067.nc" - ], 4), + (FakeFCIFileHandlerFDHSI, _chans_fdhsi, _test_filenames['fdhsi'], 16), + (FakeFCIFileHandlerHRFI, _chans_hrfi, _test_filenames['hrfi'], 4), ]) def test_load_index_map(self, reader_configs, filehandler, channels, filenames, expected_res_n): @@ -556,16 +500,8 @@ def test_load_index_map(self, reader_configs, filehandler, channels, filenames, numpy.testing.assert_array_equal(res[ch + '_index_map'][1, 1], 5237) @pytest.mark.parametrize('filehandler,filenames', [ - (FakeFCIFileHandlerFDHSI, [ - "W_XX-EUMETSAT-Darmstadt,IMG+SAT,MTI1+FCI-1C-RRAD-FDHSI-FD--" - "CHK-BODY--L2P-NC4E_C_EUMT_20170410114434_GTT_DEV_" - "20170410113925_20170410113934_N__C_0070_0067.nc" - ]), - (FakeFCIFileHandlerHRFI, [ - "W_XX-EUMETSAT-Darmstadt,IMG+SAT,MTI1+FCI-1C-RRAD-HRFI-FD--" - "CHK-BODY--L2P-NC4E_C_EUMT_20170410114434_GTT_DEV_" - "20170410113925_20170410113934_N__C_0070_0067.nc" - ]), + (FakeFCIFileHandlerFDHSI, _test_filenames['fdhsi']), + (FakeFCIFileHandlerHRFI, _test_filenames['hrfi']), ]) def test_load_aux_data(self, reader_configs, filehandler, filenames): @@ -595,16 +531,8 @@ def test_load_composite(self): assert len(mods["fci"]) > 0 @pytest.mark.parametrize('filehandler,channels,filenames,expected_res_n', [ - (FakeFCIFileHandlerFDHSI, _chans_fdhsi, [ - "W_XX-EUMETSAT-Darmstadt,IMG+SAT,MTI1+FCI-1C-RRAD-FDHSI-FD--" - "CHK-BODY--L2P-NC4E_C_EUMT_20170410114434_GTT_DEV_" - "20170410113925_20170410113934_N__C_0070_0067.nc" - ], 16), - (FakeFCIFileHandlerHRFI, _chans_hrfi, [ - "W_XX-EUMETSAT-Darmstadt,IMG+SAT,MTI1+FCI-1C-RRAD-HRFI-FD--" - "CHK-BODY--L2P-NC4E_C_EUMT_20170410114434_GTT_DEV_" - "20170410113925_20170410113934_N__C_0070_0067.nc" - ], 4), + (FakeFCIFileHandlerFDHSI, _chans_fdhsi, _test_filenames['fdhsi'], 16), + (FakeFCIFileHandlerHRFI, _chans_hrfi, _test_filenames['hrfi'], 4), ]) def test_load_quality_only(self, reader_configs, filehandler, channels, filenames, expected_res_n): @@ -621,16 +549,8 @@ def test_load_quality_only(self, reader_configs, filehandler, channels, filename assert res[ch + '_pixel_quality'].attrs["name"] == ch + '_pixel_quality' @pytest.mark.parametrize('filehandler,filenames', [ - (FakeFCIFileHandlerFDHSI, [ - "W_XX-EUMETSAT-Darmstadt,IMG+SAT,MTI1+FCI-1C-RRAD-FDHSI-FD--" - "CHK-BODY--L2P-NC4E_C_EUMT_20170410114434_GTT_DEV_" - "20170410113925_20170410113934_N__C_0070_0067.nc" - ]), - (FakeFCIFileHandlerHRFI, [ - "W_XX-EUMETSAT-Darmstadt,IMG+SAT,MTI1+FCI-1C-RRAD-HRFI-FD--" - "CHK-BODY--L2P-NC4E_C_EUMT_20170410114434_GTT_DEV_" - "20170410113925_20170410113934_N__C_0070_0067.nc" - ]), + (FakeFCIFileHandlerFDHSI, _test_filenames['fdhsi']), + (FakeFCIFileHandlerHRFI, _test_filenames['hrfi']), ]) def test_platform_name(self, reader_configs, filehandler, filenames): """Test that platform name is exposed. @@ -645,13 +565,9 @@ def test_platform_name(self, reader_configs, filehandler, filenames): def test_excs(self, reader_configs): """Test that exceptions are raised where expected.""" - filenames = [ - "W_XX-EUMETSAT-Darmstadt,IMG+SAT,MTI1+FCI-1C-RRAD-FDHSI-FD--" - "CHK-BODY--L2P-NC4E_C_EUMT_20170410114434_GTT_DEV_" - "20170410113925_20170410113934_N__C_0070_0067.nc", - ] + with mocked_basefilehandler(FakeFCIFileHandlerFDHSI): - reader = _get_reader_with_filehandlers(filenames, reader_configs) + reader = _get_reader_with_filehandlers(_test_filenames['fdhsi'], reader_configs) with pytest.raises(ValueError): reader.file_handlers["fci_l1c_fdhsi"][0].get_dataset(make_dataid(name="invalid"), {}) @@ -661,16 +577,8 @@ def test_excs(self, reader_configs): {"units": "unknown"}) @pytest.mark.parametrize('filehandler,filenames, expected_area', [ - (FakeFCIFileHandlerFDHSI, [ - "W_XX-EUMETSAT-Darmstadt,IMG+SAT,MTI1+FCI-1C-RRAD-FDHSI-FD--" - "CHK-BODY--L2P-NC4E_C_EUMT_20170410114434_GTT_DEV_" - "20170410113925_20170410113934_N__C_0070_0067.nc" - ], ['mtg_fci_fdss_1km', 'mtg_fci_fdss_2km']), - (FakeFCIFileHandlerHRFI, [ - "W_XX-EUMETSAT-Darmstadt,IMG+SAT,MTI1+FCI-1C-RRAD-HRFI-FD--" - "CHK-BODY--L2P-NC4E_C_EUMT_20170410114434_GTT_DEV_" - "20170410113925_20170410113934_N__C_0070_0067.nc" - ], ['mtg_fci_fdss_500m', 'mtg_fci_fdss_1km']), + (FakeFCIFileHandlerFDHSI, _test_filenames['fdhsi'], ['mtg_fci_fdss_1km', 'mtg_fci_fdss_2km']), + (FakeFCIFileHandlerHRFI, _test_filenames['hrfi'], ['mtg_fci_fdss_500m', 'mtg_fci_fdss_1km']), ]) def test_area_definition_computation(self, reader_configs, filehandler, filenames, expected_area): """Test that the geolocation computation is correct.""" @@ -700,118 +608,47 @@ def test_area_definition_computation(self, reader_configs, filehandler, filename class TestFCIL1cNCReaderBadData(TestFCIL1cNCReader): """Test the FCI L1c NetCDF Reader for bad data input.""" - _alt_handler = FakeFCIFileHandlerWithBadData - def test_handling_bad_data_ir(self, reader_configs, caplog): """Test handling of bad IR data.""" - from satpy.tests.utils import make_dataid - filenames = [ - "W_XX-EUMETSAT-Darmstadt,IMG+SAT,MTI1+FCI-1C-RRAD-FDHSI-FD--" - "CHK-BODY--L2P-NC4E_C_EUMT_20170410114434_GTT_DEV_" - "20170410113925_20170410113934_N__C_0070_0067.nc", - ] - - reader = _get_reader_with_filehandlers(filenames, reader_configs) - with caplog.at_level("ERROR"): - reader.load([make_dataid( - name="ir_123", - calibration="brightness_temperature")], pad_data=False) - assert "cannot produce brightness temperature" in caplog.text + with mocked_basefilehandler(FakeFCIFileHandlerWithBadData): + reader = _get_reader_with_filehandlers(_test_filenames['fdhsi'], reader_configs) + with caplog.at_level("ERROR"): + reader.load([make_dataid( + name="ir_105", + calibration="brightness_temperature")], pad_data=False) + assert "cannot produce brightness temperature" in caplog.text def test_handling_bad_data_vis(self, reader_configs, caplog): """Test handling of bad VIS data.""" - from satpy.tests.utils import make_dataid - - filenames = [ - "W_XX-EUMETSAT-Darmstadt,IMG+SAT,MTI1+FCI-1C-RRAD-FDHSI-FD--" - "CHK-BODY--L2P-NC4E_C_EUMT_20170410114434_GTT_DEV_" - "20170410113925_20170410113934_N__C_0070_0067.nc", - ] - - reader = _get_reader_with_filehandlers(filenames, reader_configs) - with caplog.at_level("ERROR"): - reader.load([make_dataid( - name="vis_04", - calibration="reflectance")], pad_data=False) - assert "cannot produce reflectance" in caplog.text + with mocked_basefilehandler(FakeFCIFileHandlerWithBadData): + reader = _get_reader_with_filehandlers(_test_filenames['fdhsi'], reader_configs) + with caplog.at_level("ERROR"): + reader.load([make_dataid( + name="vis_06", + calibration="reflectance")], pad_data=False) + assert "cannot produce reflectance" in caplog.text class TestFCIL1cNCReaderBadDataFromIDPF(TestFCIL1cNCReader): """Test the FCI L1c NetCDF Reader for bad data input.""" - _alt_handler = FakeFCIFileHandlerWithBadIDPFData - def test_handling_bad_earthsun_distance(self, reader_configs, caplog): """Test handling of bad earth-sun distance data.""" - from satpy.tests.utils import make_dataid - - filenames = [ - "W_XX-EUMETSAT-Darmstadt,IMG+SAT,MTI1+FCI-1C-RRAD-FDHSI-FD--" - "CHK-BODY--L2P-NC4E_C_EUMT_20170410114434_GTT_DEV_" - "20170410113925_20170410113934_N__C_0070_0067.nc", - ] + with mocked_basefilehandler(FakeFCIFileHandlerWithBadIDPFData): + reader = _get_reader_with_filehandlers(_test_filenames['fdhsi'], reader_configs) - reader = _get_reader_with_filehandlers(filenames, reader_configs) - - res = reader.load([make_dataid(name=["vis_04"], calibration="reflectance")], pad_data=False) - numpy.testing.assert_array_almost_equal(res["vis_04"], 100 * 15 * 1 * np.pi / 50) + res = reader.load([make_dataid(name=["vis_06"], calibration="reflectance")], pad_data=False) + numpy.testing.assert_array_almost_equal(res["vis_06"], 100 * 15 * 1 * np.pi / 50) def test_bad_xy_coords(self, reader_configs): """Test that the geolocation computation is correct.""" - filenames = [ - "W_XX-EUMETSAT-Darmstadt,IMG+SAT,MTI1+FCI-1C-RRAD-FDHSI-FD--" - "CHK-BODY--L2P-NC4E_C_EUMT_20170410114434_GTT_DEV_" - "20170410113925_20170410113934_N__C_0070_0067.nc", - ] - - reader = _get_reader_with_filehandlers(filenames, reader_configs) - res = reader.load(['vis_04'], pad_data=False) - - area_def = res['vis_04'].attrs['area'] - # test area extents computation - np.testing.assert_array_almost_equal(np.array(area_def.area_extent), - np.array([-5568062.270889, 5168057.806632, - 16704186.298937, 5568062.270889])) - -# class TestFCIL1cNCReaderHRFI(TestFCIL1cNCReaderGoodData): -# """Test FCI L1c NetCDF reader for the HRFI case.""" -# -# _alt_handler = FakeFCIFileHandlerHRFI -# -# def test_file_pattern(self, reader_configs): -# """Test file pattern matching.""" -# from satpy.readers import load_reader -# -# filenames = [ -# "W_XX-EUMETSAT-Darmstadt,IMG+SAT,MTI1+FCI-1C-RRAD-HRFI-FD--" -# "CHK-BODY--L2P-NC4E_C_EUMT_20170410114434_GTT_DEV_" -# "20170410113925_20170410113934_N__C_0070_0038.nc", -# "W_XX-EUMETSAT-Darmstadt,IMG+SAT,MTI1+FCI-1C-RRAD-HRFI-FD--" -# "CHK-BODY--L2P-NC4E_C_EUMT_20170410114442_GTT_DEV_" -# "20170410113934_20170410113942_N__C_0070_0039.nc", -# # this is a TRAIL file, which we don't use/read -# "W_XX-EUMETSAT-Darmstadt,IMG+SAT,MTI1+FCI-1C-RRAD-HRFI-FD--" -# "CHK-TRAIL--L2P-NC4E_C_EUMT_20170410114600_GTT_DEV_" -# "20170410113000_20170410114000_N__C_0070_0071.nc", -# ] -# -# reader = load_reader(reader_configs) -# files = reader.select_files_from_pathnames(filenames) -# # only 2 HRFI should match, the TRAIL should be ignored -# assert len(files) == 2 -# -# _chans = {"solar": ["vis_06", "nir_22"], -# "terran": ["ir_38", "ir_105"]} -# -# def test_load_counts(self, reader_configs): -# # testing two filenames to test correctly combined -# filenames = [ -# "W_XX-EUMETSAT-Darmstadt,IMG+SAT,MTI1+FCI-1C-RRAD-HRFI-FD--" -# "CHK-BODY--L2P-NC4E_C_EUMT_20170410114434_GTT_DEV_" -# "20170410113925_20170410113934_N__C_0070_0038.nc", -# "W_XX-EUMETSAT-Darmstadt,IMG+SAT,MTI1+FCI-1C-RRAD-HRFI-FD--" -# "CHK-BODY--L2P-NC4E_C_EUMT_20170410114442_GTT_DEV_" -# "20170410113934_20170410113942_N__C_0070_0039.nc", -# ] -# self._check_load_counts(reader_configs, filenames) + with mocked_basefilehandler(FakeFCIFileHandlerWithBadIDPFData): + reader = _get_reader_with_filehandlers(_test_filenames['fdhsi'], reader_configs) + res = reader.load(['vis_06'], pad_data=False) + + area_def = res['vis_06'].attrs['area'] + # test area extents computation + np.testing.assert_array_almost_equal(np.array(area_def.area_extent), + np.array([-5568062.270889, 5168057.806632, + 16704186.298937, 5568062.270889])) From 04d13a5dad477e8e5002e5a08ba36c32cd0af8ec Mon Sep 17 00:00:00 2001 From: andream Date: Fri, 18 Nov 2022 11:03:28 +0100 Subject: [PATCH 07/26] fix pre-commit --- satpy/tests/reader_tests/test_fci_l1c_nc.py | 82 +++++++++------------ 1 file changed, 34 insertions(+), 48 deletions(-) diff --git a/satpy/tests/reader_tests/test_fci_l1c_nc.py b/satpy/tests/reader_tests/test_fci_l1c_nc.py index c0855af10f..033394b65d 100644 --- a/satpy/tests/reader_tests/test_fci_l1c_nc.py +++ b/satpy/tests/reader_tests/test_fci_l1c_nc.py @@ -19,6 +19,7 @@ import contextlib import logging import os +from typing import Dict, List from unittest import mock import dask.array as da @@ -36,26 +37,24 @@ class FakeFCIFileHandlerBase(FakeNetCDF4FileHandler): """Class for faking the NetCDF4 Filehandler.""" # overwritten by FDHSI and HRFI FIle Handlers - chan_patterns = {} + chan_patterns: Dict[str, List[int]] = {} def _get_test_calib_for_channel_ir(self, chroot, meas): from pyspectral.blackbody import C_SPEED as c from pyspectral.blackbody import H_PLANCK as h from pyspectral.blackbody import K_BOLTZMANN as k xrda = xr.DataArray - data = {} - data[meas + "/radiance_to_bt_conversion_coefficient_wavenumber"] = xrda(955) - data[meas + "/radiance_to_bt_conversion_coefficient_a"] = xrda(1) - data[meas + "/radiance_to_bt_conversion_coefficient_b"] = xrda(0.4) - data[meas + "/radiance_to_bt_conversion_constant_c1"] = xrda(1e11 * 2 * h * c ** 2) - data[meas + "/radiance_to_bt_conversion_constant_c2"] = xrda(1e2 * h * c / k) + data = {meas + "/radiance_to_bt_conversion_coefficient_wavenumber": xrda(955), + meas + "/radiance_to_bt_conversion_coefficient_a": xrda(1), + meas + "/radiance_to_bt_conversion_coefficient_b": xrda(0.4), + meas + "/radiance_to_bt_conversion_constant_c1": xrda(1e11 * 2 * h * c ** 2), + meas + "/radiance_to_bt_conversion_constant_c2": xrda(1e2 * h * c / k)} return data def _get_test_calib_for_channel_vis(self, chroot, meas): xrda = xr.DataArray - data = {} - data["state/celestial/earth_sun_distance"] = xrda(da.repeat(da.array([149597870.7]), 6000)) - data[meas + "/channel_effective_solar_irradiance"] = xrda(50) + data = {"state/celestial/earth_sun_distance": xrda(da.repeat(da.array([149597870.7]), 6000)), + meas + "/channel_effective_solar_irradiance": xrda(50)} return data def _get_test_content_for_channel(self, pat, ch): @@ -204,14 +203,6 @@ def _get_global_attributes(self): def get_test_content(self, filename, filename_info, filetype_info): """Get the content of the test data.""" - # mock global attributes - # - root groups global - # - other groups global - # mock data variables - # mock dimensions - # - # ... but only what satpy is using ... - D = {} D.update(self._get_test_content_all_channels()) D.update(self._get_test_content_areadef()) @@ -224,11 +215,11 @@ class FakeFCIFileHandlerFDHSI(FakeFCIFileHandlerBase): """Mock FDHSI data.""" chan_patterns = { - "vis_{:>02d}": (4, 5, 6, 8, 9), - "nir_{:>02d}": (13, 16, 22), - "ir_{:>02d}": (38, 87, 97, 105, 123, 133), - "wv_{:>02d}": (63, 73), - } + "vis_{:>02d}": [4, 5, 6, 8, ], + "nir_{:>02d}": [13, 16, 22], + "ir_{:>02d}": [38, 87, 97, 105, 123, 133], + "wv_{:>02d}": [63, 73], + } class FakeFCIFileHandlerWithBadData(FakeFCIFileHandlerFDHSI): @@ -279,10 +270,11 @@ class FakeFCIFileHandlerHRFI(FakeFCIFileHandlerBase): """Mock HRFI data.""" chan_patterns = { - "vis_{:>02d}_hr": [6], - "nir_{:>02d}_hr": [22], - "ir_{:>02d}_hr": [38, 105], - } + "vis_{:>02d}_hr": [6], + "nir_{:>02d}_hr": [22], + "ir_{:>02d}_hr": [38, 105], + } + @pytest.fixture def reader_configs(): @@ -308,33 +300,30 @@ def _get_reader_with_filehandlers(filenames, reader_configs): _chans_hrfi = {"solar": ["vis_06", "nir_22"], "terran": ["ir_38", "ir_105"]} -_test_filenames = {'fdhsi' : [ - "W_XX-EUMETSAT-Darmstadt,IMG+SAT,MTI1+FCI-1C-RRAD-FDHSI-FD--" - "CHK-BODY--L2P-NC4E_C_EUMT_20170410114434_GTT_DEV_" - "20170410113925_20170410113934_N__C_0070_0067.nc" - ], -'hrfi': [ +_test_filenames = {'fdhsi': [ + "W_XX-EUMETSAT-Darmstadt,IMG+SAT,MTI1+FCI-1C-RRAD-FDHSI-FD--" + "CHK-BODY--L2P-NC4E_C_EUMT_20170410114434_GTT_DEV_" + "20170410113925_20170410113934_N__C_0070_0067.nc" +], + 'hrfi': [ "W_XX-EUMETSAT-Darmstadt,IMG+SAT,MTI1+FCI-1C-RRAD-HRFI-FD--" "CHK-BODY--L2P-NC4E_C_EUMT_20170410114434_GTT_DEV_" "20170410113925_20170410113934_N__C_0070_0067.nc" ] } + @contextlib.contextmanager def mocked_basefilehandler(filehandler): + """Mock patch the base class of the FCIL1cNCFileHandler with the content of our fake files (filehandler).""" p = mock.patch.object(FCIL1cNCFileHandler, "__bases__", (filehandler,)) with p: p.is_local = True yield -class TestFCIL1cNCReader: - """Initialize the unittest TestCase for the FCI L1c NetCDF Reader.""" - - -class TestFCIL1cNCReaderGoodData(TestFCIL1cNCReader): - """Test FCI L1c NetCDF reader with good data, on the FDHSI example.""" - +class TestFCIL1cNCReader: + """Test FCI L1c NetCDF reader with nominal data.""" @pytest.mark.parametrize('filenames', [_test_filenames['fdhsi'], _test_filenames['hrfi']]) def test_file_pattern(self, reader_configs, filenames): @@ -381,14 +370,13 @@ def test_load_counts(self, reader_configs, filehandler, channels, filenames, numpy.testing.assert_array_equal(res[ch], 1) @pytest.mark.parametrize('filehandler,channels,filenames,expected_res_n', [ - (FakeFCIFileHandlerFDHSI, _chans_fdhsi, _test_filenames['fdhsi'] , 16), - (FakeFCIFileHandlerHRFI, _chans_hrfi, _test_filenames['hrfi'] , 4), + (FakeFCIFileHandlerFDHSI, _chans_fdhsi, _test_filenames['fdhsi'], 16), + (FakeFCIFileHandlerHRFI, _chans_hrfi, _test_filenames['hrfi'], 4), ]) def test_load_radiance(self, reader_configs, filehandler, channels, filenames, expected_res_n): """Test loading with radiance.""" - with mocked_basefilehandler(filehandler): reader = _get_reader_with_filehandlers(filenames, reader_configs) res = reader.load( @@ -413,7 +401,7 @@ def test_load_radiance(self, reader_configs, filehandler, channels, filenames, ]) def test_load_reflectance(self, reader_configs, filehandler, channels, filenames, - expected_res_n): + expected_res_n): """Test loading with reflectance.""" with mocked_basefilehandler(filehandler): reader = _get_reader_with_filehandlers(filenames, reader_configs) @@ -434,7 +422,7 @@ def test_load_reflectance(self, reader_configs, filehandler, channels, filenames ]) def test_load_bt(self, reader_configs, caplog, filehandler, channels, filenames, - expected_res_n): + expected_res_n): """Test loading with bt.""" with mocked_basefilehandler(filehandler): reader = _get_reader_with_filehandlers(filenames, reader_configs) @@ -565,7 +553,6 @@ def test_platform_name(self, reader_configs, filehandler, filenames): def test_excs(self, reader_configs): """Test that exceptions are raised where expected.""" - with mocked_basefilehandler(FakeFCIFileHandlerFDHSI): reader = _get_reader_with_filehandlers(_test_filenames['fdhsi'], reader_configs) @@ -610,7 +597,6 @@ class TestFCIL1cNCReaderBadData(TestFCIL1cNCReader): def test_handling_bad_data_ir(self, reader_configs, caplog): """Test handling of bad IR data.""" - with mocked_basefilehandler(FakeFCIFileHandlerWithBadData): reader = _get_reader_with_filehandlers(_test_filenames['fdhsi'], reader_configs) with caplog.at_level("ERROR"): @@ -637,8 +623,8 @@ def test_handling_bad_earthsun_distance(self, reader_configs, caplog): """Test handling of bad earth-sun distance data.""" with mocked_basefilehandler(FakeFCIFileHandlerWithBadIDPFData): reader = _get_reader_with_filehandlers(_test_filenames['fdhsi'], reader_configs) - res = reader.load([make_dataid(name=["vis_06"], calibration="reflectance")], pad_data=False) + numpy.testing.assert_array_almost_equal(res["vis_06"], 100 * 15 * 1 * np.pi / 50) def test_bad_xy_coords(self, reader_configs): From 3a2a355f1d336931a11537dc9250ffb7c130789f Mon Sep 17 00:00:00 2001 From: andream Date: Fri, 18 Nov 2022 11:17:22 +0100 Subject: [PATCH 08/26] update docstring --- satpy/readers/fci_l1c_nc.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/satpy/readers/fci_l1c_nc.py b/satpy/readers/fci_l1c_nc.py index f503a117eb..c78fff78f9 100644 --- a/satpy/readers/fci_l1c_nc.py +++ b/satpy/readers/fci_l1c_nc.py @@ -28,8 +28,8 @@ .. note:: This reader currently supports Full Disk High Spectral Resolution Imagery - (FDHSI) files. Support for High Spatial Resolution Fast Imagery (HRFI) files - will be implemented when corresponding test datasets will be available. + (FDHSI) and High Spatial Resolution Fast Imagery (HRFI) data. RSS data is + not supported yet. Geolocation is based on information from the data files. It uses: From 1d3aa41450735ca9969d86a0d19f544c322a4957 Mon Sep 17 00:00:00 2001 From: andream Date: Fri, 18 Nov 2022 12:22:43 +0100 Subject: [PATCH 09/26] replace ~0 by -1 and fix tests after typo --- satpy/tests/reader_tests/test_fci_l1c_nc.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/satpy/tests/reader_tests/test_fci_l1c_nc.py b/satpy/tests/reader_tests/test_fci_l1c_nc.py index 033394b65d..71856c318c 100644 --- a/satpy/tests/reader_tests/test_fci_l1c_nc.py +++ b/satpy/tests/reader_tests/test_fci_l1c_nc.py @@ -215,7 +215,7 @@ class FakeFCIFileHandlerFDHSI(FakeFCIFileHandlerBase): """Mock FDHSI data.""" chan_patterns = { - "vis_{:>02d}": [4, 5, 6, 8, ], + "vis_{:>02d}": [4, 5, 6, 8, 9], "nir_{:>02d}": [13, 16, 22], "ir_{:>02d}": [38, 87, 97, 105, 123, 133], "wv_{:>02d}": [63, 73], @@ -364,7 +364,7 @@ def test_load_counts(self, reader_configs, filehandler, channels, filenames, assert res[ch].attrs["calibration"] == "counts" assert res[ch].attrs["units"] == "count" if ch == 'ir_38': - numpy.testing.assert_array_equal(res[ch][~0], 1) + numpy.testing.assert_array_equal(res[ch][-1], 1) numpy.testing.assert_array_equal(res[ch][0], 5000) else: numpy.testing.assert_array_equal(res[ch], 1) @@ -390,7 +390,7 @@ def test_load_radiance(self, reader_configs, filehandler, channels, filenames, assert res[ch].attrs["units"] == 'mW m-2 sr-1 (cm-1)-1' assert res[ch].attrs["radiance_unit_conversion_coefficient"] == 1234.56 if ch == 'ir_38': - numpy.testing.assert_array_equal(res[ch][~0], 15) + numpy.testing.assert_array_equal(res[ch][-1], 15) numpy.testing.assert_array_equal(res[ch][0], 9700) else: numpy.testing.assert_array_equal(res[ch], 15) @@ -439,7 +439,7 @@ def test_load_bt(self, reader_configs, caplog, filehandler, channels, filenames, assert res[ch].attrs["units"] == "K" if ch == 'ir_38': - numpy.testing.assert_array_almost_equal(res[ch][~0], 209.68274099) + numpy.testing.assert_array_almost_equal(res[ch][-1], 209.68274099) numpy.testing.assert_array_almost_equal(res[ch][0], 1888.851296) else: numpy.testing.assert_array_almost_equal(res[ch], 209.68274099) From 41aaa1a92f08d945626f99d0ce8e57b2ba005db7 Mon Sep 17 00:00:00 2001 From: andream Date: Thu, 24 Nov 2022 18:40:30 +0100 Subject: [PATCH 10/26] make get_segment_position_info population dynamic depending on file type --- satpy/readers/fci_l1c_nc.py | 42 ++++++++++++++++++++++++++----------- 1 file changed, 30 insertions(+), 12 deletions(-) diff --git a/satpy/readers/fci_l1c_nc.py b/satpy/readers/fci_l1c_nc.py index c78fff78f9..fc48d1a64c 100644 --- a/satpy/readers/fci_l1c_nc.py +++ b/satpy/readers/fci_l1c_nc.py @@ -134,6 +134,15 @@ 'swath_direction': 'data/swath_direction', } +HIGH_RES_GRID_INFO = {'fci_l1c_hrfi': {'grid_type': '500m', + 'width': 22272}, + 'fci_l1c_fdhsi': {'grid_type': '1km', + 'width': 11136}} +LOW_RES_GRID_INFO = {'fci_l1c_hrfi': {'grid_type': '1km', + 'width': 11136}, + 'fci_l1c_fdhsi': {'grid_type': '2km', + 'width': 5568}} + def _get_aux_data_name_from_dsname(dsname): aux_data_name = [key for key in AUX_DATA.keys() if key in dsname] @@ -213,23 +222,32 @@ def get_channel_measured_group_path(self, channel): return measured_group_path def get_segment_position_info(self): - """Get the vertical position and size information of the chunk (aka segment) for both 1km and 2km grids. + """Get information about the size and the position of the chunk. - This is used in the GEOVariableSegmentYAMLReader to compute optimal chunk sizes for missing chunks. + As the final array is composed by stacking chunks (aka segments) vertically, the position of a chunk + inside the array is defined by the numbers of the start (lowest) and end (highest) row of the chunk. + This info is used in the GEOVariableSegmentYAMLReader to compute optimal chunk sizes for missing chunks. """ vis_06_measured_path = self.get_channel_measured_group_path('vis_06') ir_105_measured_path = self.get_channel_measured_group_path('ir_105') + + file_type = self.filetype_info['file_type'] + segment_position_info = { - '1km': {'start_position_row': self[vis_06_measured_path+'/start_position_row'].item(), - 'end_position_row': self[vis_06_measured_path+'/end_position_row'].item(), - 'segment_height': self[vis_06_measured_path+'/end_position_row'].item() - - self[vis_06_measured_path+'/start_position_row'].item() + 1, - 'segment_width': 11136}, - '2km': {'start_position_row': self[ir_105_measured_path+'/start_position_row'].item(), - 'end_position_row': self[ir_105_measured_path+'/end_position_row'].item(), - 'segment_height': self[ir_105_measured_path+'/end_position_row'].item() - - self[ir_105_measured_path+'/start_position_row'].item() + 1, - 'segment_width': 5568} + HIGH_RES_GRID_INFO[file_type]['grid_type']: { + 'start_position_row': self[vis_06_measured_path + '/start_position_row'].item(), + 'end_position_row': self[vis_06_measured_path + '/end_position_row'].item(), + 'segment_height': self[vis_06_measured_path + '/end_position_row'].item() - + self[vis_06_measured_path + '/start_position_row'].item() + 1, + 'segment_width': HIGH_RES_GRID_INFO[file_type]['width'] + }, + LOW_RES_GRID_INFO[file_type]['grid_type']: { + 'start_position_row': self[ir_105_measured_path + '/start_position_row'].item(), + 'end_position_row': self[ir_105_measured_path + '/end_position_row'].item(), + 'segment_height': self[ir_105_measured_path + '/end_position_row'].item() - + self[ir_105_measured_path + '/start_position_row'].item() + 1, + 'segment_width': LOW_RES_GRID_INFO[file_type]['width'] + } } return segment_position_info From b2d4d8ec0fa789fc37759ccf8a4e685096ec2c2d Mon Sep 17 00:00:00 2001 From: andream Date: Thu, 24 Nov 2022 19:00:01 +0100 Subject: [PATCH 11/26] restructure call to segment_heights to be called only on grids that actually need it --- satpy/readers/yaml_reader.py | 29 +++++++++++++++++------------ 1 file changed, 17 insertions(+), 12 deletions(-) diff --git a/satpy/readers/yaml_reader.py b/satpy/readers/yaml_reader.py index d856f56c19..1fa3284405 100644 --- a/satpy/readers/yaml_reader.py +++ b/satpy/readers/yaml_reader.py @@ -26,7 +26,7 @@ from collections import OrderedDict, deque from contextlib import suppress from fnmatch import fnmatch -from functools import cached_property +from functools import lru_cache from weakref import WeakValueDictionary import numpy as np @@ -1373,6 +1373,15 @@ class GEOVariableSegmentYAMLReader(GEOSegmentYAMLReader): For more information on please see the documentation of :func:`satpy.readers.yaml_reader.GEOSegmentYAMLReader`. """ + def __init__(self, + config_dict, + filter_parameters=None, + filter_filenames=True, + **kwargs): + """Initialise the GEOVariableSegmentYAMLReader object.""" + super().__init__(config_dict, filter_parameters, filter_filenames, **kwargs) + self.segment_heights = lru_cache(self._segment_heights) + def create_filehandlers(self, filenames, fh_kwargs=None): """Create file handler objects and collect the location information.""" created_fhs = super().create_filehandlers(filenames, fh_kwargs=fh_kwargs) @@ -1402,24 +1411,20 @@ def _initialise_segment_infos(self, filetype, filetype_fhs): 'width_to_grid_type': width_to_grid_type}}) def _get_empty_segment(self, dim=None, idx=None, filetype=None): - grid_type = self.segment_infos[filetype]['width_to_grid_type'][self.empty_segment.shape[1]] - segment_height = self.segment_heights[filetype][grid_type][idx] + width = self.empty_segment.shape[1] + segment_height = self.segment_heights(filetype, width)[idx] return _get_empty_segment_with_height(self.empty_segment, segment_height, dim=dim) - @cached_property - def segment_heights(self): + def _segment_heights(self, filetype, width): """Compute optimal padded segment heights (in number of pixels) based on the location of available segments.""" - segment_heights = dict() - for filetype, filetype_seginfos in self.segment_infos.items(): - filetype_seg_heights = {'1km': _compute_optimal_missing_segment_heights(filetype_seginfos, '1km', 11136), - '2km': _compute_optimal_missing_segment_heights(filetype_seginfos, '2km', 5568)} - segment_heights.update({filetype: filetype_seg_heights}) + grid_type = self.segment_infos[filetype]['width_to_grid_type'][width] + segment_heights = _compute_optimal_missing_segment_heights(self.segment_infos[filetype], grid_type, width) return segment_heights def _get_new_areadef_heights(self, previous_area, previous_seg_size, segment_n=None, filetype=None): # retrieve the segment height in number of pixels - grid_type = self.segment_infos[filetype]['width_to_grid_type'][previous_seg_size[1]] - new_height_px = self.segment_heights[filetype][grid_type][segment_n - 1] + width = previous_seg_size[1] + new_height_px = self.segment_heights(filetype, width)[segment_n - 1] # scale the previous vertical area extent using the new pixel height prev_area_extent = previous_area.area_extent[1] - previous_area.area_extent[3] new_height_proj_coord = prev_area_extent * new_height_px / previous_seg_size[0] From 132f9a32e21323b4dcef8c399c25cdbfdd6428c3 Mon Sep 17 00:00:00 2001 From: andream Date: Fri, 25 Nov 2022 16:07:23 +0100 Subject: [PATCH 12/26] move collection of segment info out of the init, to be called only later during get_dataset uses the caching trick as discussed in https://github.com/pytroll/satpy/pull/2237#discussion_r1030708191 --- satpy/_compat.py | 8 +- satpy/readers/fci_l1c_nc.py | 12 +-- satpy/readers/yaml_reader.py | 52 +++++------ satpy/tests/test_yaml_reader.py | 157 ++++++++++++++++++-------------- 4 files changed, 125 insertions(+), 104 deletions(-) diff --git a/satpy/_compat.py b/satpy/_compat.py index 6a2a4fd528..b49b5a961b 100644 --- a/satpy/_compat.py +++ b/satpy/_compat.py @@ -50,7 +50,7 @@ def __get__(self, instance, owner=None): # noqa raise TypeError( "Cannot use cached_property instance without calling __set_name__ on it.") try: - cache = instance.__dict__ + cache = instance.__dict__ # noqa except AttributeError: # not all objects have __dict__ (e.g. class defines slots) msg = ( f"No '__dict__' attribute on {type(instance).__name__!r} " @@ -88,3 +88,9 @@ def __get__(self, instance, owner=None): # noqa # numpy <1.20 from numpy import dtype as DTypeLike # noqa from numpy import ndarray as ArrayLike # noqa + + +try: + from functools import cache # type: ignore +except ImportError: + from functools import lru_cache as cache # noqa diff --git a/satpy/readers/fci_l1c_nc.py b/satpy/readers/fci_l1c_nc.py index fc48d1a64c..20818445c4 100644 --- a/satpy/readers/fci_l1c_nc.py +++ b/satpy/readers/fci_l1c_nc.py @@ -135,13 +135,13 @@ } HIGH_RES_GRID_INFO = {'fci_l1c_hrfi': {'grid_type': '500m', - 'width': 22272}, + 'grid_width': 22272}, 'fci_l1c_fdhsi': {'grid_type': '1km', - 'width': 11136}} + 'grid_width': 11136}} LOW_RES_GRID_INFO = {'fci_l1c_hrfi': {'grid_type': '1km', - 'width': 11136}, + 'grid_width': 11136}, 'fci_l1c_fdhsi': {'grid_type': '2km', - 'width': 5568}} + 'grid_width': 5568}} def _get_aux_data_name_from_dsname(dsname): @@ -239,14 +239,14 @@ def get_segment_position_info(self): 'end_position_row': self[vis_06_measured_path + '/end_position_row'].item(), 'segment_height': self[vis_06_measured_path + '/end_position_row'].item() - self[vis_06_measured_path + '/start_position_row'].item() + 1, - 'segment_width': HIGH_RES_GRID_INFO[file_type]['width'] + 'grid_width': HIGH_RES_GRID_INFO[file_type]['grid_width'] }, LOW_RES_GRID_INFO[file_type]['grid_type']: { 'start_position_row': self[ir_105_measured_path + '/start_position_row'].item(), 'end_position_row': self[ir_105_measured_path + '/end_position_row'].item(), 'segment_height': self[ir_105_measured_path + '/end_position_row'].item() - self[ir_105_measured_path + '/start_position_row'].item() + 1, - 'segment_width': LOW_RES_GRID_INFO[file_type]['width'] + 'grid_width': LOW_RES_GRID_INFO[file_type]['grid_width'] } } diff --git a/satpy/readers/yaml_reader.py b/satpy/readers/yaml_reader.py index 1fa3284405..658371fd42 100644 --- a/satpy/readers/yaml_reader.py +++ b/satpy/readers/yaml_reader.py @@ -26,7 +26,6 @@ from collections import OrderedDict, deque from contextlib import suppress from fnmatch import fnmatch -from functools import lru_cache from weakref import WeakValueDictionary import numpy as np @@ -38,6 +37,7 @@ from yaml import UnsafeLoader from satpy import DatasetDict +from satpy._compat import cache from satpy.aux_download import DataDownloadMixin from satpy.dataset import DataID, DataQuery, get_key from satpy.dataset.dataid import default_co_keys_config, default_id_keys_config, get_keys_from_config @@ -1380,51 +1380,47 @@ def __init__(self, **kwargs): """Initialise the GEOVariableSegmentYAMLReader object.""" super().__init__(config_dict, filter_parameters, filter_filenames, **kwargs) - self.segment_heights = lru_cache(self._segment_heights) - - def create_filehandlers(self, filenames, fh_kwargs=None): - """Create file handler objects and collect the location information.""" - created_fhs = super().create_filehandlers(filenames, fh_kwargs=fh_kwargs) - self._extract_segment_location_dicts(created_fhs) - return created_fhs - - def _extract_segment_location_dicts(self, created_fhs): + self.segment_heights = cache(self._segment_heights) self.segment_infos = dict() - for filetype, filetype_fhs in created_fhs.items(): - self._initialise_segment_infos(filetype, filetype_fhs) - self._collect_segment_position_infos(filetype, filetype_fhs) + + def _extract_segment_location_dicts(self, filetype): + self._initialise_segment_infos(filetype) + self._collect_segment_position_infos(filetype) return - def _collect_segment_position_infos(self, filetype, filetype_fhs): + def _collect_segment_position_infos(self, filetype): # collect the segment positioning infos for all available segments + filetype_fhs = self.file_handlers[filetype] for fh in filetype_fhs: chk_infos = fh.get_segment_position_info() chk_infos.update({'segment_nr': fh.filename_info['segment'] - 1}) self.segment_infos[filetype]['available_segment_infos'].append(chk_infos) - def _initialise_segment_infos(self, filetype, filetype_fhs): + def _initialise_segment_infos(self, filetype): # initialise the segment info for this filetype + filetype_fhs = self.file_handlers[filetype] exp_segment_nr = filetype_fhs[0].filetype_info['expected_segments'] - width_to_grid_type = _get_width_to_grid_type(filetype_fhs[0].get_segment_position_info()) + grid_width_to_grid_type = _get_grid_width_to_grid_type(filetype_fhs[0].get_segment_position_info()) self.segment_infos.update({filetype: {'available_segment_infos': [], 'expected_segments': exp_segment_nr, - 'width_to_grid_type': width_to_grid_type}}) + 'grid_width_to_grid_type': grid_width_to_grid_type}}) def _get_empty_segment(self, dim=None, idx=None, filetype=None): - width = self.empty_segment.shape[1] - segment_height = self.segment_heights(filetype, width)[idx] + grid_width = self.empty_segment.shape[1] + segment_height = self.segment_heights(filetype, grid_width)[idx] return _get_empty_segment_with_height(self.empty_segment, segment_height, dim=dim) - def _segment_heights(self, filetype, width): + def _segment_heights(self, filetype, grid_width): """Compute optimal padded segment heights (in number of pixels) based on the location of available segments.""" - grid_type = self.segment_infos[filetype]['width_to_grid_type'][width] - segment_heights = _compute_optimal_missing_segment_heights(self.segment_infos[filetype], grid_type, width) + self._extract_segment_location_dicts(filetype) + grid_type = self.segment_infos[filetype]['grid_width_to_grid_type'][grid_width] + segment_heights = _compute_optimal_missing_segment_heights(self.segment_infos[filetype], grid_type, grid_width) return segment_heights def _get_new_areadef_heights(self, previous_area, previous_seg_size, segment_n=None, filetype=None): # retrieve the segment height in number of pixels - width = previous_seg_size[1] - new_height_px = self.segment_heights(filetype, width)[segment_n - 1] + grid_width = previous_seg_size[1] + new_height_px = self.segment_heights(filetype, grid_width)[segment_n - 1] # scale the previous vertical area extent using the new pixel height prev_area_extent = previous_area.area_extent[1] - previous_area.area_extent[3] new_height_proj_coord = prev_area_extent * new_height_px / previous_seg_size[0] @@ -1432,11 +1428,11 @@ def _get_new_areadef_heights(self, previous_area, previous_seg_size, segment_n=N return new_height_proj_coord, new_height_px -def _get_width_to_grid_type(seg_info): - width_to_grid_type = dict() +def _get_grid_width_to_grid_type(seg_info): + grid_width_to_grid_type = dict() for grid_type, grid_type_seg_info in seg_info.items(): - width_to_grid_type.update({grid_type_seg_info['segment_width']: grid_type}) - return width_to_grid_type + grid_width_to_grid_type.update({grid_type_seg_info['grid_width']: grid_type}) + return grid_width_to_grid_type def _compute_optimal_missing_segment_heights(seg_infos, grid_type, expected_vertical_size): diff --git a/satpy/tests/test_yaml_reader.py b/satpy/tests/test_yaml_reader.py index 89416ab76d..ef048e3eb6 100644 --- a/satpy/tests/test_yaml_reader.py +++ b/satpy/tests/test_yaml_reader.py @@ -25,9 +25,11 @@ from unittest.mock import MagicMock, call, patch import numpy as np +import pytest import xarray as xr import satpy.readers.yaml_reader as yr +from satpy._compat import cache from satpy.dataset import DataQuery from satpy.dataset.dataid import ModifierTuple from satpy.readers.file_handlers import BaseFileHandler @@ -41,7 +43,7 @@ 'default_channels': [1, 2, 3, 4, 5], 'data_identification_keys': {'name': {'required': True}, 'frequency_double_sideband': - {'type': FrequencyDoubleSideBand}, + {'type': FrequencyDoubleSideBand}, 'frequency_range': {'type': FrequencyRange}, 'resolution': None, 'polarization': {'enum': ['H', 'V']}, @@ -689,7 +691,7 @@ def test_update_ds_ids_from_file_handlers(self): # need to copy this because the dataset infos will be modified _orig_ids = {key: val.copy() for key, val in orig_ids.items()} with patch.dict(self.reader.all_ids, _orig_ids, clear=True), \ - patch.dict(self.reader.available_ids, {}, clear=True): + patch.dict(self.reader.available_ids, {}, clear=True): # Add a file handler with resolution property fh = MagicMock(filetype_info={'file_type': ftype}, resolution=resol) @@ -1245,27 +1247,62 @@ def test_find_missing_segments(self): self.assertTrue(proj is projectable) -class TestGEOVariableSegmentYAMLReader(unittest.TestCase): +@pytest.fixture +@patch.object(yr.GEOVariableSegmentYAMLReader, "__init__", lambda x: None) +def fake_GVSYReader(): + """Get a fixture of the GEOVariableSegmentYAMLReader.""" + from satpy.readers.yaml_reader import GEOVariableSegmentYAMLReader + p = patch.object(GEOVariableSegmentYAMLReader, "__init__", lambda x: None) + with p: + reader = GEOVariableSegmentYAMLReader() + reader.segment_infos = dict() + reader.segment_heights = cache(reader._segment_heights) + return reader + + +@pytest.fixture +def fake_geswh(): + """Get a fixture of the patched _get_empty_segment_with_height.""" + with patch('satpy.readers.yaml_reader._get_empty_segment_with_height') as geswh: + yield geswh + + +@pytest.fixture +def fake_xr(): + """Get a fixture of the patched xarray.""" + with patch('satpy.readers.yaml_reader.xr') as xr: + yield xr + + +@pytest.fixture +def fake_mss(): + """Get a fixture of the patched _find_missing_segments.""" + with patch('satpy.readers.yaml_reader._find_missing_segments') as mss: + yield mss + + +@pytest.fixture +def fake_adef(): + """Get a fixture of the patched AreaDefinition.""" + with patch('satpy.readers.yaml_reader.AreaDefinition') as adef: + yield adef + + +class TestGEOVariableSegmentYAMLReader: """Test GEOVariableSegmentYAMLReader.""" - @patch.object(yr.FileYAMLReader, "__init__", lambda x: None) - @patch('satpy.readers.yaml_reader._get_empty_segment_with_height') - @patch('satpy.readers.yaml_reader.xr') - @patch('satpy.readers.yaml_reader._find_missing_segments') - def test_get_empty_segment(self, mss, xr, geswh): + def test_get_empty_segment(self, fake_GVSYReader, fake_mss, fake_xr, fake_geswh): """Test execution of (overridden) get_empty_segment inside _load_dataset.""" - from satpy.readers.yaml_reader import GEOVariableSegmentYAMLReader - reader = GEOVariableSegmentYAMLReader() # Setup input, and output of mocked functions for first segment missing chk_pos_info = { '1km': {'start_position_row': 0, 'end_position_row': 0, 'segment_height': 0, - 'segment_width': 11136}, + 'grid_width': 11136}, '2km': {'start_position_row': 140, 'end_position_row': None, 'segment_height': 278, - 'segment_width': 5568} + 'grid_width': 5568} } expected_segments = 2 segment = 2 @@ -1273,8 +1310,7 @@ def test_get_empty_segment(self, mss, xr, geswh): ashape = [278, 5568] fh_2, seg2_area = _create_mocked_fh_and_areadef(aex, ashape, expected_segments, segment, chk_pos_info) - file_handlers = {'filetype1': [fh_2]} - reader._extract_segment_location_dicts(file_handlers) + fake_GVSYReader.file_handlers = {'filetype1': [fh_2]} counter = 2 seg = MagicMock(dims=['y', 'x']) @@ -1283,32 +1319,28 @@ def test_get_empty_segment(self, mss, xr, geswh): projectable = MagicMock() empty_segment = MagicMock() empty_segment.shape = [278, 5568] - xr.full_like.return_value = empty_segment + fake_xr.full_like.return_value = empty_segment dataid = MagicMock() ds_info = MagicMock() - mss.return_value = (counter, expected_segments, slice_list, - failure, projectable) - reader._load_dataset(dataid, ds_info, [fh_2]) + fake_mss.return_value = (counter, expected_segments, slice_list, + failure, projectable) + fake_GVSYReader._load_dataset(dataid, ds_info, [fh_2]) # the return of get_empty_segment - geswh.assert_called_once_with(empty_segment, 139, dim='y') + fake_geswh.assert_called_once_with(empty_segment, 139, dim='y') - @patch.object(yr.FileYAMLReader, "__init__", lambda x: None) - @patch('satpy.readers.yaml_reader.AreaDefinition') - def test_pad_earlier_segments_area(self, AreaDefinition): + def test_pad_earlier_segments_area(self, fake_GVSYReader, fake_adef): """Test _pad_earlier_segments_area() for the variable segment case.""" - from satpy.readers.yaml_reader import GEOVariableSegmentYAMLReader - reader = GEOVariableSegmentYAMLReader() # setting to 0 or None values that shouldn't be relevant chk_pos_info = { '1km': {'start_position_row': 0, 'end_position_row': 0, 'segment_height': 0, - 'segment_width': 11136}, + 'grid_width': 11136}, '2km': {'start_position_row': 140, 'end_position_row': None, 'segment_height': 278, - 'segment_width': 5568} + 'grid_width': 5568} } expected_segments = 2 segment = 2 @@ -1316,12 +1348,11 @@ def test_pad_earlier_segments_area(self, AreaDefinition): ashape = [278, 5568] fh_2, seg2_area = _create_mocked_fh_and_areadef(aex, ashape, expected_segments, segment, chk_pos_info) - file_handlers = {'filetype1': [fh_2]} - reader._extract_segment_location_dicts(file_handlers) + fake_GVSYReader.file_handlers = {'filetype1': [fh_2]} dataid = 'dataid' area_defs = {2: seg2_area} - res = reader._pad_earlier_segments_area([fh_2], dataid, area_defs) - self.assertEqual(len(res), 2) + res = fake_GVSYReader._pad_earlier_segments_area([fh_2], dataid, area_defs) + assert len(res) == 2 # The later vertical chunk (nr. 2) size is 278, which is exactly double the size # of the gap left by the missing first chunk (139, as the second chunk starts at line 140). @@ -1331,35 +1362,29 @@ def test_pad_earlier_segments_area(self, AreaDefinition): seg1_extent = (0, 500, 200, 250) expected_call = ('fill', 'fill', 'fill', 'some_crs', 5568, 139, seg1_extent) - AreaDefinition.assert_called_once_with(*expected_call) + fake_adef.assert_called_once_with(*expected_call) - @patch.object(yr.FileYAMLReader, "__init__", lambda x: None) - @patch('satpy.readers.yaml_reader.AreaDefinition') - def test_pad_later_segments_area(self, AreaDefinition): + def test_pad_later_segments_area(self, fake_GVSYReader, fake_adef): """Test _pad_later_segments_area() in the variable padding case.""" - from satpy.readers.yaml_reader import GEOVariableSegmentYAMLReader - reader = GEOVariableSegmentYAMLReader() - chk_pos_info = { '1km': {'start_position_row': None, 'end_position_row': 11136 - 278, 'segment_height': 556, - 'segment_width': 11136}, + 'grid_width': 11136}, '2km': {'start_position_row': 0, 'end_position_row': 0, 'segment_height': 0, - 'segment_width': 5568}} + 'grid_width': 5568}} expected_segments = 2 segment = 1 aex = [0, 1000, 200, 500] ashape = [556, 11136] fh_1, _ = _create_mocked_fh_and_areadef(aex, ashape, expected_segments, segment, chk_pos_info) - file_handlers = {'filetype1': [fh_1]} - reader._extract_segment_location_dicts(file_handlers) + fake_GVSYReader.file_handlers = {'filetype1': [fh_1]} dataid = 'dataid' - res = reader._pad_later_segments_area([fh_1], dataid) - self.assertEqual(len(res), 2) + res = fake_GVSYReader._pad_later_segments_area([fh_1], dataid) + assert len(res) == 2 # The previous chunk size is 556, which is exactly double the size of the gap left # by the missing last chunk (278, as the second-to-last chunk ends at line 11136 - 278 ) @@ -1368,14 +1393,10 @@ def test_pad_later_segments_area(self, AreaDefinition): seg2_extent = (0, 1250, 200, 1000) expected_call = ('fill', 'fill', 'fill', 'some_crs', 11136, 278, seg2_extent) - AreaDefinition.assert_called_once_with(*expected_call) + fake_adef.assert_called_once_with(*expected_call) - @patch.object(yr.FileYAMLReader, "__init__", lambda x: None) - @patch('satpy.readers.yaml_reader.AreaDefinition') - def test_pad_later_segments_area_for_multiple_chunks_gap(self, AreaDefinition): + def test_pad_later_segments_area_for_multiple_chunks_gap(self, fake_GVSYReader, fake_adef): """Test _pad_later_segments_area() in the variable padding case for multiple gaps with multiple chunks.""" - from satpy.readers.yaml_reader import GEOVariableSegmentYAMLReader - reader = GEOVariableSegmentYAMLReader() def side_effect_areadef(a, b, c, crs, width, height, aex): m = MagicMock() @@ -1384,17 +1405,17 @@ def side_effect_areadef(a, b, c, crs, width, height, aex): m.crs = crs return m - AreaDefinition.side_effect = side_effect_areadef + fake_adef.side_effect = side_effect_areadef chk_pos_info = { '1km': {'start_position_row': 11136 - 600 - 100 + 1, 'end_position_row': 11136 - 600, 'segment_height': 100, - 'segment_width': 11136}, + 'grid_width': 11136}, '2km': {'start_position_row': 0, 'end_position_row': 0, 'segment_height': 0, - 'segment_width': 5568}} + 'grid_width': 5568}} expected_segments = 8 segment = 1 aex = [0, 1000, 200, 500] @@ -1404,11 +1425,11 @@ def side_effect_areadef(a, b, c, crs, width, height, aex): '1km': {'start_position_row': 11136 - 300 - 100 + 1, 'end_position_row': 11136 - 300, 'segment_height': 100, - 'segment_width': 11136}, + 'grid_width': 11136}, '2km': {'start_position_row': 0, 'end_position_row': 0, 'segment_height': 0, - 'segment_width': 5568}} + 'grid_width': 5568}} segment = 4 fh_4, _ = _create_mocked_fh_and_areadef(aex, ashape, expected_segments, segment, chk_pos_info) @@ -1416,20 +1437,18 @@ def side_effect_areadef(a, b, c, crs, width, height, aex): '1km': {'start_position_row': 11136 - 100 + 1, 'end_position_row': None, 'segment_height': 100, - 'segment_width': 11136}, + 'grid_width': 11136}, '2km': {'start_position_row': 0, 'end_position_row': 0, 'segment_height': 0, - 'segment_width': 5568}} + 'grid_width': 5568}} segment = 8 fh_8, _ = _create_mocked_fh_and_areadef(aex, ashape, expected_segments, segment, chk_pos_info) - file_handlers = {'filetype1': [fh_1, fh_4, fh_8]} - - reader._extract_segment_location_dicts(file_handlers) + fake_GVSYReader.file_handlers = {'filetype1': [fh_1, fh_4, fh_8]} dataid = 'dataid' - res = reader._pad_later_segments_area([fh_1, fh_4, fh_8], dataid) - self.assertEqual(len(res), 8) + res = fake_GVSYReader._pad_later_segments_area([fh_1, fh_4, fh_8], dataid) + assert len(res) == 8 # Regarding the chunk sizes: # First group of missing chunks: @@ -1455,7 +1474,7 @@ def side_effect_areadef(a, b, c, crs, width, height, aex): # The first padded chunk has 66px height -> 500*66/100=330 area extent height ->1000+330=1330 # The second padded chunk has 67px height -> 500*67/100=335 area extent height ->1330+335=1665 # The first padded chunk has 67px height -> 500*67/100=335 area extent height ->1665+335=2000 - self.assertEqual(AreaDefinition.call_count, 5) + assert fake_adef.call_count == 5 expected_call1 = ('fill', 'fill', 'fill', 'some_crs', 11136, 100, (0, 1500.0, 200, 1000)) expected_call2 = ('fill', 'fill', 'fill', 'some_crs', 11136, 100, @@ -1467,13 +1486,13 @@ def side_effect_areadef(a, b, c, crs, width, height, aex): expected_call5 = ('fill', 'fill', 'fill', 'some_crs', 11136, 67, (0, 2000.0, 200, 1665.0)) - AreaDefinition.side_effect = None - AreaDefinition.assert_has_calls([call(*expected_call1), - call(*expected_call2), - call(*expected_call3), - call(*expected_call4), - call(*expected_call5) - ]) + fake_adef.side_effect = None + fake_adef.assert_has_calls([call(*expected_call1), + call(*expected_call2), + call(*expected_call3), + call(*expected_call4), + call(*expected_call5) + ]) def test_get_empty_segment_with_height(self): """Test _get_empty_segment_with_height().""" From 93af5f5d230d716bbd5c1e8873acd05ee8705340 Mon Sep 17 00:00:00 2001 From: andream Date: Fri, 25 Nov 2022 16:47:40 +0100 Subject: [PATCH 13/26] remove unnecessary patch context and rename reader fixture --- satpy/tests/test_yaml_reader.py | 38 ++++++++++++++++----------------- 1 file changed, 18 insertions(+), 20 deletions(-) diff --git a/satpy/tests/test_yaml_reader.py b/satpy/tests/test_yaml_reader.py index ef048e3eb6..479d91ffb4 100644 --- a/satpy/tests/test_yaml_reader.py +++ b/satpy/tests/test_yaml_reader.py @@ -1249,15 +1249,13 @@ def test_find_missing_segments(self): @pytest.fixture @patch.object(yr.GEOVariableSegmentYAMLReader, "__init__", lambda x: None) -def fake_GVSYReader(): +def GVSYReader(): """Get a fixture of the GEOVariableSegmentYAMLReader.""" from satpy.readers.yaml_reader import GEOVariableSegmentYAMLReader - p = patch.object(GEOVariableSegmentYAMLReader, "__init__", lambda x: None) - with p: - reader = GEOVariableSegmentYAMLReader() - reader.segment_infos = dict() - reader.segment_heights = cache(reader._segment_heights) - return reader + reader = GEOVariableSegmentYAMLReader() + reader.segment_infos = dict() + reader.segment_heights = cache(reader._segment_heights) + return reader @pytest.fixture @@ -1291,7 +1289,7 @@ def fake_adef(): class TestGEOVariableSegmentYAMLReader: """Test GEOVariableSegmentYAMLReader.""" - def test_get_empty_segment(self, fake_GVSYReader, fake_mss, fake_xr, fake_geswh): + def test_get_empty_segment(self, GVSYReader, fake_mss, fake_xr, fake_geswh): """Test execution of (overridden) get_empty_segment inside _load_dataset.""" # Setup input, and output of mocked functions for first segment missing chk_pos_info = { @@ -1308,9 +1306,9 @@ def test_get_empty_segment(self, fake_GVSYReader, fake_mss, fake_xr, fake_geswh) segment = 2 aex = [0, 1000, 200, 500] ashape = [278, 5568] - fh_2, seg2_area = _create_mocked_fh_and_areadef(aex, ashape, expected_segments, segment, chk_pos_info) + fh_2, _ = _create_mocked_fh_and_areadef(aex, ashape, expected_segments, segment, chk_pos_info) - fake_GVSYReader.file_handlers = {'filetype1': [fh_2]} + GVSYReader.file_handlers = {'filetype1': [fh_2]} counter = 2 seg = MagicMock(dims=['y', 'x']) @@ -1325,11 +1323,11 @@ def test_get_empty_segment(self, fake_GVSYReader, fake_mss, fake_xr, fake_geswh) fake_mss.return_value = (counter, expected_segments, slice_list, failure, projectable) - fake_GVSYReader._load_dataset(dataid, ds_info, [fh_2]) + GVSYReader._load_dataset(dataid, ds_info, [fh_2]) # the return of get_empty_segment fake_geswh.assert_called_once_with(empty_segment, 139, dim='y') - def test_pad_earlier_segments_area(self, fake_GVSYReader, fake_adef): + def test_pad_earlier_segments_area(self, GVSYReader, fake_adef): """Test _pad_earlier_segments_area() for the variable segment case.""" # setting to 0 or None values that shouldn't be relevant chk_pos_info = { @@ -1348,10 +1346,10 @@ def test_pad_earlier_segments_area(self, fake_GVSYReader, fake_adef): ashape = [278, 5568] fh_2, seg2_area = _create_mocked_fh_and_areadef(aex, ashape, expected_segments, segment, chk_pos_info) - fake_GVSYReader.file_handlers = {'filetype1': [fh_2]} + GVSYReader.file_handlers = {'filetype1': [fh_2]} dataid = 'dataid' area_defs = {2: seg2_area} - res = fake_GVSYReader._pad_earlier_segments_area([fh_2], dataid, area_defs) + res = GVSYReader._pad_earlier_segments_area([fh_2], dataid, area_defs) assert len(res) == 2 # The later vertical chunk (nr. 2) size is 278, which is exactly double the size @@ -1364,7 +1362,7 @@ def test_pad_earlier_segments_area(self, fake_GVSYReader, fake_adef): seg1_extent) fake_adef.assert_called_once_with(*expected_call) - def test_pad_later_segments_area(self, fake_GVSYReader, fake_adef): + def test_pad_later_segments_area(self, GVSYReader, fake_adef): """Test _pad_later_segments_area() in the variable padding case.""" chk_pos_info = { '1km': {'start_position_row': None, @@ -1381,9 +1379,9 @@ def test_pad_later_segments_area(self, fake_GVSYReader, fake_adef): aex = [0, 1000, 200, 500] ashape = [556, 11136] fh_1, _ = _create_mocked_fh_and_areadef(aex, ashape, expected_segments, segment, chk_pos_info) - fake_GVSYReader.file_handlers = {'filetype1': [fh_1]} + GVSYReader.file_handlers = {'filetype1': [fh_1]} dataid = 'dataid' - res = fake_GVSYReader._pad_later_segments_area([fh_1], dataid) + res = GVSYReader._pad_later_segments_area([fh_1], dataid) assert len(res) == 2 # The previous chunk size is 556, which is exactly double the size of the gap left @@ -1395,7 +1393,7 @@ def test_pad_later_segments_area(self, fake_GVSYReader, fake_adef): seg2_extent) fake_adef.assert_called_once_with(*expected_call) - def test_pad_later_segments_area_for_multiple_chunks_gap(self, fake_GVSYReader, fake_adef): + def test_pad_later_segments_area_for_multiple_chunks_gap(self, GVSYReader, fake_adef): """Test _pad_later_segments_area() in the variable padding case for multiple gaps with multiple chunks.""" def side_effect_areadef(a, b, c, crs, width, height, aex): @@ -1445,9 +1443,9 @@ def side_effect_areadef(a, b, c, crs, width, height, aex): segment = 8 fh_8, _ = _create_mocked_fh_and_areadef(aex, ashape, expected_segments, segment, chk_pos_info) - fake_GVSYReader.file_handlers = {'filetype1': [fh_1, fh_4, fh_8]} + GVSYReader.file_handlers = {'filetype1': [fh_1, fh_4, fh_8]} dataid = 'dataid' - res = fake_GVSYReader._pad_later_segments_area([fh_1, fh_4, fh_8], dataid) + res = GVSYReader._pad_later_segments_area([fh_1, fh_4, fh_8], dataid) assert len(res) == 8 # Regarding the chunk sizes: From 24c8e5d6b5a600a2b589fb24c59dadaff604d924 Mon Sep 17 00:00:00 2001 From: andream Date: Fri, 25 Nov 2022 20:42:03 +0100 Subject: [PATCH 14/26] make the fci tests use different resolutions according to the channel and data type input, remove inheritance from main test to avoid test repetition, fix some dummy data input to make tests clearer --- satpy/tests/reader_tests/test_fci_l1c_nc.py | 181 ++++++++++++-------- 1 file changed, 107 insertions(+), 74 deletions(-) diff --git a/satpy/tests/reader_tests/test_fci_l1c_nc.py b/satpy/tests/reader_tests/test_fci_l1c_nc.py index 71856c318c..c4556f5107 100644 --- a/satpy/tests/reader_tests/test_fci_l1c_nc.py +++ b/satpy/tests/reader_tests/test_fci_l1c_nc.py @@ -19,7 +19,7 @@ import contextlib import logging import os -from typing import Dict, List +from typing import Dict, List, Union from unittest import mock import dask.array as da @@ -32,12 +32,33 @@ from satpy.tests.reader_tests.test_netcdf_utils import FakeNetCDF4FileHandler from satpy.tests.utils import make_dataid +GRID_TYPE_INFO_FOR_TEST_CONTENT = { + '500m': { + 'nrows': 400, + 'ncols': 22272, + 'scale_factor': 1.39717881644274e-05, + 'add_offset': 1.55596818893146e-01, + }, + '1km': { + 'nrows': 200, + 'ncols': 11136, + 'scale_factor': 2.79435763233999e-05, + 'add_offset': 1.55603804756852e-01, + }, + '2km': { + 'nrows': 100, + 'ncols': 5568, + 'scale_factor': 5.58871526031607e-05, + 'add_offset': 1.55617776423501e-01, + }, +} + class FakeFCIFileHandlerBase(FakeNetCDF4FileHandler): """Class for faking the NetCDF4 Filehandler.""" # overwritten by FDHSI and HRFI FIle Handlers - chan_patterns: Dict[str, List[int]] = {} + chan_patterns: Dict[str, Dict[str, Union[List[int], str]]] = {} def _get_test_calib_for_channel_ir(self, chroot, meas): from pyspectral.blackbody import C_SPEED as c @@ -57,10 +78,10 @@ def _get_test_calib_for_channel_vis(self, chroot, meas): meas + "/channel_effective_solar_irradiance": xrda(50)} return data - def _get_test_content_for_channel(self, pat, ch): + def _get_test_content_for_channel(self, pat, ch, grid_type): xrda = xr.DataArray - nrows = 200 - ncols = 11136 + nrows = GRID_TYPE_INFO_FOR_TEST_CONTENT[grid_type]['nrows'] + ncols = GRID_TYPE_INFO_FOR_TEST_CONTENT[grid_type]['ncols'] chroot = "data/{:s}" meas = chroot + "/measured" rad = meas + "/effective_radiance" @@ -112,25 +133,21 @@ def _get_test_content_for_channel(self, pat, ch): da.arange(1, ncols + 1, dtype="uint16"), dims=("x",), attrs={ - "scale_factor": -5.58877772833e-05, - "add_offset": 0.155619515845, + "scale_factor": -GRID_TYPE_INFO_FOR_TEST_CONTENT[grid_type]['scale_factor'], + "add_offset": GRID_TYPE_INFO_FOR_TEST_CONTENT[grid_type]['add_offset'], } ) data[y.format(ch_str)] = xrda( da.arange(1, nrows + 1, dtype="uint16"), dims=("y",), attrs={ - "scale_factor": -5.58877772833e-05, - "add_offset": 0.155619515845, + "scale_factor": GRID_TYPE_INFO_FOR_TEST_CONTENT[grid_type]['scale_factor'], + "add_offset": -GRID_TYPE_INFO_FOR_TEST_CONTENT[grid_type]['add_offset'], } ) - data[qual.format(ch_str)] = xrda( - da.arange(nrows * ncols, dtype="uint8").reshape(nrows, ncols) % 128, - dims=("y", "x")) - # add dummy data for index map starting from 100 - data[index_map.format(ch_str)] = xrda( - (da.arange(nrows * ncols, dtype="uint16").reshape(nrows, ncols) % 6000) + 100, - dims=("y", "x")) + data[qual.format(ch_str)] = xrda((da.ones((nrows, ncols))) * 3, dims=("y", "x")) + # add dummy data for index map + data[index_map.format(ch_str)] = xrda((da.ones((nrows, ncols))) * 110, dims=("y", "x")) data[rad_conv_coeff.format(ch_str)] = xrda(1234.56) data[pos.format(ch_str, "start", "row")] = xrda(0) @@ -149,8 +166,8 @@ def _get_test_content_for_channel(self, pat, ch): def _get_test_content_all_channels(self): data = {} for pat in self.chan_patterns: - for ch_num in self.chan_patterns[pat]: - data.update(self._get_test_content_for_channel(pat, ch_num)) + for ch_num in self.chan_patterns[pat]['channels']: + data.update(self._get_test_content_for_channel(pat, ch_num, self.chan_patterns[pat]['grid_type'])) return data def _get_test_content_areadef(self): @@ -191,7 +208,7 @@ def _get_test_content_aux_data(self): # compute the last data entry to simulate the FCI caching data[list(AUX_DATA.values())[-1]] = data[list(AUX_DATA.values())[-1]].compute() - data['index'] = xrda(da.arange(indices_dim, dtype="uint16") + 100, dims=("index")) + data['index'] = xrda(da.ones(indices_dim, dtype="uint16") * 100, dims=("index")) return data def _get_global_attributes(self): @@ -215,10 +232,14 @@ class FakeFCIFileHandlerFDHSI(FakeFCIFileHandlerBase): """Mock FDHSI data.""" chan_patterns = { - "vis_{:>02d}": [4, 5, 6, 8, 9], - "nir_{:>02d}": [13, 16, 22], - "ir_{:>02d}": [38, 87, 97, 105, 123, 133], - "wv_{:>02d}": [63, 73], + "vis_{:>02d}": {'channels': [4, 5, 6, 8, 9], + 'grid_type': '1km'}, + "nir_{:>02d}": {'channels': [13, 16, 22], + 'grid_type': '1km'}, + "ir_{:>02d}": {'channels': [38, 87, 97, 105, 123, 133], + 'grid_type': '2km'}, + "wv_{:>02d}": {'channels': [63, 73], + 'grid_type': '2km'}, } @@ -270,9 +291,12 @@ class FakeFCIFileHandlerHRFI(FakeFCIFileHandlerBase): """Mock HRFI data.""" chan_patterns = { - "vis_{:>02d}_hr": [6], - "nir_{:>02d}_hr": [22], - "ir_{:>02d}_hr": [38, 105], + "vis_{:>02d}_hr": {'channels': [6], + 'grid_type': '500m'}, + "nir_{:>02d}_hr": {'channels': [22], + 'grid_type': '500m'}, + "ir_{:>02d}_hr": {'channels': [38, 105], + 'grid_type': '1km'}, } @@ -294,11 +318,15 @@ def _get_reader_with_filehandlers(filenames, reader_configs): _chans_fdhsi = {"solar": ["vis_04", "vis_05", "vis_06", "vis_08", "vis_09", "nir_13", "nir_16", "nir_22"], + "solar_grid_type": ["1km"] * 8, "terran": ["ir_38", "wv_63", "wv_73", "ir_87", "ir_97", "ir_105", - "ir_123", "ir_133"]} + "ir_123", "ir_133"], + "terran_grid_type": ["2km"] * 8} _chans_hrfi = {"solar": ["vis_06", "nir_22"], - "terran": ["ir_38", "ir_105"]} + "solar_grid_type": ["500m"] * 2, + "terran": ["ir_38", "ir_105"], + "terran_grid_type": ["1km"] * 2} _test_filenames = {'fdhsi': [ "W_XX-EUMETSAT-Darmstadt,IMG+SAT,MTI1+FCI-1C-RRAD-FDHSI-FD--" @@ -346,8 +374,7 @@ def test_file_pattern_for_TRAIL_file(self, reader_configs, filenames): @pytest.mark.parametrize('filehandler,channels,filenames,expected_res_n', [ (FakeFCIFileHandlerFDHSI, _chans_fdhsi, _test_filenames['fdhsi'], 16), - (FakeFCIFileHandlerHRFI, _chans_hrfi, _test_filenames['hrfi'], 4), - + (FakeFCIFileHandlerHRFI, _chans_hrfi, _test_filenames['hrfi'], 4) ]) def test_load_counts(self, reader_configs, filehandler, channels, filenames, expected_res_n): @@ -358,8 +385,10 @@ def test_load_counts(self, reader_configs, filehandler, channels, filenames, [make_dataid(name=name, calibration="counts") for name in channels["solar"] + channels["terran"]], pad_data=False) assert expected_res_n == len(res) - for ch in channels["solar"] + channels["terran"]: - assert res[ch].shape == (200, 11136) + for ch, grid_type in zip(channels["solar"] + channels["terran"], + channels["solar_grid_type"] + channels["terran_grid_type"]): + assert res[ch].shape == (GRID_TYPE_INFO_FOR_TEST_CONTENT[grid_type]['nrows'], + GRID_TYPE_INFO_FOR_TEST_CONTENT[grid_type]['ncols']) assert res[ch].dtype == np.uint16 assert res[ch].attrs["calibration"] == "counts" assert res[ch].attrs["units"] == "count" @@ -371,8 +400,7 @@ def test_load_counts(self, reader_configs, filehandler, channels, filenames, @pytest.mark.parametrize('filehandler,channels,filenames,expected_res_n', [ (FakeFCIFileHandlerFDHSI, _chans_fdhsi, _test_filenames['fdhsi'], 16), - (FakeFCIFileHandlerHRFI, _chans_hrfi, _test_filenames['hrfi'], 4), - + (FakeFCIFileHandlerHRFI, _chans_hrfi, _test_filenames['hrfi'], 4) ]) def test_load_radiance(self, reader_configs, filehandler, channels, filenames, expected_res_n): @@ -383,8 +411,10 @@ def test_load_radiance(self, reader_configs, filehandler, channels, filenames, [make_dataid(name=name, calibration="radiance") for name in channels["solar"] + channels["terran"]], pad_data=False) assert expected_res_n == len(res) - for ch in channels["solar"] + channels["terran"]: - assert res[ch].shape == (200, 11136) + for ch, grid_type in zip(channels["solar"] + channels["terran"], + channels["solar_grid_type"] + channels["terran_grid_type"]): + assert res[ch].shape == (GRID_TYPE_INFO_FOR_TEST_CONTENT[grid_type]['nrows'], + GRID_TYPE_INFO_FOR_TEST_CONTENT[grid_type]['ncols']) assert res[ch].dtype == np.float64 assert res[ch].attrs["calibration"] == "radiance" assert res[ch].attrs["units"] == 'mW m-2 sr-1 (cm-1)-1' @@ -397,8 +427,7 @@ def test_load_radiance(self, reader_configs, filehandler, channels, filenames, @pytest.mark.parametrize('filehandler,channels,filenames,expected_res_n', [ (FakeFCIFileHandlerFDHSI, _chans_fdhsi, _test_filenames['fdhsi'], 8), - (FakeFCIFileHandlerHRFI, _chans_hrfi, _test_filenames['hrfi'], 2), - + (FakeFCIFileHandlerHRFI, _chans_hrfi, _test_filenames['hrfi'], 2) ]) def test_load_reflectance(self, reader_configs, filehandler, channels, filenames, expected_res_n): @@ -409,8 +438,9 @@ def test_load_reflectance(self, reader_configs, filehandler, channels, filenames [make_dataid(name=name, calibration="reflectance") for name in channels["solar"]], pad_data=False) assert expected_res_n == len(res) - for ch in channels["solar"]: - assert res[ch].shape == (200, 11136) + for ch, grid_type in zip(channels["solar"], channels["solar_grid_type"]): + assert res[ch].shape == (GRID_TYPE_INFO_FOR_TEST_CONTENT[grid_type]['nrows'], + GRID_TYPE_INFO_FOR_TEST_CONTENT[grid_type]['ncols']) assert res[ch].dtype == np.float64 assert res[ch].attrs["calibration"] == "reflectance" assert res[ch].attrs["units"] == "%" @@ -418,8 +448,7 @@ def test_load_reflectance(self, reader_configs, filehandler, channels, filenames @pytest.mark.parametrize('filehandler,channels,filenames,expected_res_n', [ (FakeFCIFileHandlerFDHSI, _chans_fdhsi, _test_filenames['fdhsi'], 8), - (FakeFCIFileHandlerHRFI, _chans_hrfi, _test_filenames['hrfi'], 2), - + (FakeFCIFileHandlerHRFI, _chans_hrfi, _test_filenames['hrfi'], 2) ]) def test_load_bt(self, reader_configs, caplog, filehandler, channels, filenames, expected_res_n): @@ -432,8 +461,9 @@ def test_load_bt(self, reader_configs, caplog, filehandler, channels, filenames, name in channels["terran"]], pad_data=False) assert caplog.text == "" assert expected_res_n == len(res) - for ch in channels["terran"]: - assert res[ch].shape == (200, 11136) + for ch, grid_type in zip(channels["terran"], channels["terran_grid_type"]): + assert res[ch].shape == (GRID_TYPE_INFO_FOR_TEST_CONTENT[grid_type]['nrows'], + GRID_TYPE_INFO_FOR_TEST_CONTENT[grid_type]['ncols']) assert res[ch].dtype == np.float64 assert res[ch].attrs["calibration"] == "brightness_temperature" assert res[ch].attrs["units"] == "K" @@ -446,8 +476,7 @@ def test_load_bt(self, reader_configs, caplog, filehandler, channels, filenames, @pytest.mark.parametrize('filehandler,channels,filenames', [ (FakeFCIFileHandlerFDHSI, _chans_fdhsi, _test_filenames['fdhsi']), - (FakeFCIFileHandlerHRFI, _chans_hrfi, _test_filenames['hrfi']), - + (FakeFCIFileHandlerHRFI, _chans_hrfi, _test_filenames['hrfi']) ]) def test_orbital_parameters_attr(self, reader_configs, filehandler, channels, filenames): """Test the orbital parameter attribute.""" @@ -472,8 +501,7 @@ def test_orbital_parameters_attr(self, reader_configs, filehandler, channels, fi @pytest.mark.parametrize('filehandler,channels,filenames,expected_res_n', [ (FakeFCIFileHandlerFDHSI, _chans_fdhsi, _test_filenames['fdhsi'], 16), - (FakeFCIFileHandlerHRFI, _chans_hrfi, _test_filenames['hrfi'], 4), - + (FakeFCIFileHandlerHRFI, _chans_hrfi, _test_filenames['hrfi'], 4) ]) def test_load_index_map(self, reader_configs, filehandler, channels, filenames, expected_res_n): """Test loading of index_map.""" @@ -483,29 +511,31 @@ def test_load_index_map(self, reader_configs, filehandler, channels, filenames, [name + '_index_map' for name in channels["solar"] + channels["terran"]], pad_data=False) assert expected_res_n == len(res) - for ch in channels["solar"] + channels["terran"]: - assert res[ch + '_index_map'].shape == (200, 11136) - numpy.testing.assert_array_equal(res[ch + '_index_map'][1, 1], 5237) - - @pytest.mark.parametrize('filehandler,filenames', [ - (FakeFCIFileHandlerFDHSI, _test_filenames['fdhsi']), - (FakeFCIFileHandlerHRFI, _test_filenames['hrfi']), + for ch, grid_type in zip(channels["solar"] + channels["terran"], + channels["solar_grid_type"] + channels["terran_grid_type"]): + assert res[ch + '_index_map'].shape == (GRID_TYPE_INFO_FOR_TEST_CONTENT[grid_type]['nrows'], + GRID_TYPE_INFO_FOR_TEST_CONTENT[grid_type]['ncols']) + numpy.testing.assert_array_equal(res[ch + '_index_map'][1, 1], 110) + @pytest.mark.parametrize('filehandler,channels,filenames', [ + (FakeFCIFileHandlerFDHSI, _chans_fdhsi, _test_filenames['fdhsi']), + (FakeFCIFileHandlerHRFI, _chans_hrfi, _test_filenames['hrfi']) ]) - def test_load_aux_data(self, reader_configs, filehandler, filenames): + def test_load_aux_data(self, reader_configs, filehandler, channels, filenames): """Test loading of auxiliary data.""" from satpy.readers.fci_l1c_nc import AUX_DATA with mocked_basefilehandler(filehandler): reader = _get_reader_with_filehandlers(filenames, reader_configs) - res = reader.load(['vis_06_' + key for key in AUX_DATA.keys()], + res = reader.load([channels['solar'][0] + '_' + key for key in AUX_DATA.keys()], pad_data=False) - for aux in ['vis_06_' + key for key in AUX_DATA.keys()]: - - assert res[aux].shape == (200, 11136) - if aux == 'vis_06_earth_sun_distance': + grid_type = channels['solar_grid_type'][0] + for aux in [channels['solar'][0] + '_' + key for key in AUX_DATA.keys()]: + assert res[aux].shape == (GRID_TYPE_INFO_FOR_TEST_CONTENT[grid_type]['nrows'], + GRID_TYPE_INFO_FOR_TEST_CONTENT[grid_type]['ncols']) + if aux == channels['solar'][0] + '_earth_sun_distance': numpy.testing.assert_array_equal(res[aux][1, 1], 149597870.7) else: - numpy.testing.assert_array_equal(res[aux][1, 1], 5137) + numpy.testing.assert_array_equal(res[aux][1, 1], 10) def test_load_composite(self): """Test that composites are loadable.""" @@ -520,8 +550,7 @@ def test_load_composite(self): @pytest.mark.parametrize('filehandler,channels,filenames,expected_res_n', [ (FakeFCIFileHandlerFDHSI, _chans_fdhsi, _test_filenames['fdhsi'], 16), - (FakeFCIFileHandlerHRFI, _chans_hrfi, _test_filenames['hrfi'], 4), - + (FakeFCIFileHandlerHRFI, _chans_hrfi, _test_filenames['hrfi'], 4) ]) def test_load_quality_only(self, reader_configs, filehandler, channels, filenames, expected_res_n): """Test that loading quality only works.""" @@ -531,9 +560,11 @@ def test_load_quality_only(self, reader_configs, filehandler, channels, filename [name + '_pixel_quality' for name in channels["solar"] + channels["terran"]], pad_data=False) assert expected_res_n == len(res) - for ch in channels["solar"] + channels["terran"]: - assert res[ch + '_pixel_quality'].shape == (200, 11136) - numpy.testing.assert_array_equal(res[ch + '_pixel_quality'][1, 1], 1) + for ch, grid_type in zip(channels["solar"] + channels["terran"], + channels["solar_grid_type"] + channels["terran_grid_type"]): + assert res[ch + '_pixel_quality'].shape == (GRID_TYPE_INFO_FOR_TEST_CONTENT[grid_type]['nrows'], + GRID_TYPE_INFO_FOR_TEST_CONTENT[grid_type]['ncols']) + numpy.testing.assert_array_equal(res[ch + '_pixel_quality'][1, 1], 3) assert res[ch + '_pixel_quality'].attrs["name"] == ch + '_pixel_quality' @pytest.mark.parametrize('filehandler,filenames', [ @@ -580,8 +611,9 @@ def test_area_definition_computation(self, reader_configs, filehandler, filename area_def = res['ir_105'].attrs['area'] # test area extents computation np.testing.assert_array_almost_equal(np.array(area_def.area_extent), - np.array([-5568062.23065902, 5168057.7600648, - 16704186.692027, 5568062.23065902])) + np.array([-5567999.994203, -5367999.994411, + 5567999.994203, -5567999.994203]), + decimal=2) # check that the projection is read in properly assert area_def.crs.coordinate_operation.method_name == 'Geostationary Satellite (Sweep Y)' @@ -592,7 +624,7 @@ def test_area_definition_computation(self, reader_configs, filehandler, filename assert area_def.crs.ellipsoid.is_semi_minor_computed -class TestFCIL1cNCReaderBadData(TestFCIL1cNCReader): +class TestFCIL1cNCReaderBadData: """Test the FCI L1c NetCDF Reader for bad data input.""" def test_handling_bad_data_ir(self, reader_configs, caplog): @@ -616,8 +648,8 @@ def test_handling_bad_data_vis(self, reader_configs, caplog): assert "cannot produce reflectance" in caplog.text -class TestFCIL1cNCReaderBadDataFromIDPF(TestFCIL1cNCReader): - """Test the FCI L1c NetCDF Reader for bad data input.""" +class TestFCIL1cNCReaderBadDataFromIDPF: + """Test the FCI L1c NetCDF Reader for bad data input, specifically the IDPF issues.""" def test_handling_bad_earthsun_distance(self, reader_configs, caplog): """Test handling of bad earth-sun distance data.""" @@ -636,5 +668,6 @@ def test_bad_xy_coords(self, reader_configs): area_def = res['vis_06'].attrs['area'] # test area extents computation np.testing.assert_array_almost_equal(np.array(area_def.area_extent), - np.array([-5568062.270889, 5168057.806632, - 16704186.298937, 5568062.270889])) + np.array([-5568000.227139, -5368000.221262, + 5568000.100073, -5568000.227139]), + decimal=2) From af99f4b5f102e712bbe3b9b42e9973832798e8c1 Mon Sep 17 00:00:00 2001 From: andream Date: Fri, 25 Nov 2022 20:49:44 +0100 Subject: [PATCH 15/26] update position info docstring a bit more --- satpy/readers/fci_l1c_nc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/satpy/readers/fci_l1c_nc.py b/satpy/readers/fci_l1c_nc.py index 20818445c4..b9e7f55b21 100644 --- a/satpy/readers/fci_l1c_nc.py +++ b/satpy/readers/fci_l1c_nc.py @@ -222,7 +222,7 @@ def get_channel_measured_group_path(self, channel): return measured_group_path def get_segment_position_info(self): - """Get information about the size and the position of the chunk. + """Get information about the size and the position of the chunk inside the final image array. As the final array is composed by stacking chunks (aka segments) vertically, the position of a chunk inside the array is defined by the numbers of the start (lowest) and end (highest) row of the chunk. From e080fd7e0c01a6223416bf9694e70f8537b7f96a Mon Sep 17 00:00:00 2001 From: andream Date: Fri, 25 Nov 2022 21:11:38 +0100 Subject: [PATCH 16/26] add new areadef for the 500m grid in the areas.yaml and update the other areas extents using latest grid definition --- satpy/etc/areas.yaml | 28 ++++++++++++++++++++++++---- 1 file changed, 24 insertions(+), 4 deletions(-) diff --git a/satpy/etc/areas.yaml b/satpy/etc/areas.yaml index 04233d059d..9738691723 100644 --- a/satpy/etc/areas.yaml +++ b/satpy/etc/areas.yaml @@ -335,6 +335,26 @@ SouthAmerica: # ---------- Meteosat Third Generation (MTG) / FCI Instrument ----------------- # Full disk +mtg_fci_fdss_500m: + description: + MTG FCI Full Disk Scanning Service area definition + with 1 km resolution + projection: + proj: geos + lon_0: 0 + h: 35786400 + x_0: 0 + y_0: 0 + ellps: WGS84 + no_defs: null + shape: + height: 22272 + width: 22272 + area_extent: + lower_left_xy: [-5567999.999637696, -5567999.999637696] + upper_right_xy: [5567999.999637678, 5567999.999637678] + units: m + mtg_fci_fdss_1km: description: MTG FCI Full Disk Scanning Service area definition @@ -351,8 +371,8 @@ mtg_fci_fdss_1km: height: 11136 width: 11136 area_extent: - lower_left_xy: [-5567999.998577303, -5567999.998577303] - upper_right_xy: [5567999.998527619, 5567999.998527619] + lower_left_xy: [-5567999.998550739, -5567999.998550739] + upper_right_xy: [5567999.998550762, 5567999.998550762] units: m mtg_fci_fdss_2km: @@ -371,8 +391,8 @@ mtg_fci_fdss_2km: height: 5568 width: 5568 area_extent: - lower_left_xy: [-5567999.994200589, -5567999.994200589] - upper_right_xy: [5567999.994206558, 5567999.994206558] + lower_left_xy: [-5567999.994203018, -5567999.994203018] + upper_right_xy: [5567999.994203017, 5567999.994203017] units: m # Full disk - segmented products From 218183f52e9e38a1fcabc0412fc2515504fbcf1d Mon Sep 17 00:00:00 2001 From: andream Date: Fri, 25 Nov 2022 21:14:20 +0100 Subject: [PATCH 17/26] update also the segmented product areas --- satpy/etc/areas.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/satpy/etc/areas.yaml b/satpy/etc/areas.yaml index 9738691723..9b1cc738ba 100644 --- a/satpy/etc/areas.yaml +++ b/satpy/etc/areas.yaml @@ -412,8 +412,8 @@ mtg_fci_fdss_6km: height: 1856 width: 1856 area_extent: - lower_left_xy: [-5567999.994200589, -5567999.994200589] - upper_right_xy: [5567999.994206558, 5567999.994206558] + lower_left_xy: [-5567999.994203018, -5567999.994203018] + upper_right_xy: [5567999.994203017, 5567999.994203017] units: m mtg_fci_fdss_32km: @@ -432,8 +432,8 @@ mtg_fci_fdss_32km: height: 348 width: 348 area_extent: - lower_left_xy: [-5567999.994200589, -5567999.994200589] - upper_right_xy: [5567999.994206558, 5567999.994206558] + lower_left_xy: [-5567999.994203018, -5567999.994203018] + upper_right_xy: [5567999.994203017, 5567999.994203017] units: m # Geostationary Operational Environmental Satellite (GOES) / ABI Instrument From 01b41396aafd8c571c55083d46bbf1a54b27c810 Mon Sep 17 00:00:00 2001 From: andream Date: Mon, 28 Nov 2022 11:35:26 +0100 Subject: [PATCH 18/26] refactor _get_test_content_for_channel in sub-methods --- satpy/tests/reader_tests/test_fci_l1c_nc.py | 135 +++++++++++--------- 1 file changed, 73 insertions(+), 62 deletions(-) diff --git a/satpy/tests/reader_tests/test_fci_l1c_nc.py b/satpy/tests/reader_tests/test_fci_l1c_nc.py index c4556f5107..a5f8afd30c 100644 --- a/satpy/tests/reader_tests/test_fci_l1c_nc.py +++ b/satpy/tests/reader_tests/test_fci_l1c_nc.py @@ -60,41 +60,49 @@ class FakeFCIFileHandlerBase(FakeNetCDF4FileHandler): # overwritten by FDHSI and HRFI FIle Handlers chan_patterns: Dict[str, Dict[str, Union[List[int], str]]] = {} - def _get_test_calib_for_channel_ir(self, chroot, meas): + def _get_test_calib_for_channel_ir(self, meas_path): from pyspectral.blackbody import C_SPEED as c from pyspectral.blackbody import H_PLANCK as h from pyspectral.blackbody import K_BOLTZMANN as k - xrda = xr.DataArray - data = {meas + "/radiance_to_bt_conversion_coefficient_wavenumber": xrda(955), - meas + "/radiance_to_bt_conversion_coefficient_a": xrda(1), - meas + "/radiance_to_bt_conversion_coefficient_b": xrda(0.4), - meas + "/radiance_to_bt_conversion_constant_c1": xrda(1e11 * 2 * h * c ** 2), - meas + "/radiance_to_bt_conversion_constant_c2": xrda(1e2 * h * c / k)} + data = {meas_path + "/radiance_to_bt_conversion_coefficient_wavenumber": xr.DataArray(955), + meas_path + "/radiance_to_bt_conversion_coefficient_a": xr.DataArray(1), + meas_path + "/radiance_to_bt_conversion_coefficient_b": xr.DataArray(0.4), + meas_path + "/radiance_to_bt_conversion_constant_c1": xr.DataArray(1e11 * 2 * h * c ** 2), + meas_path + "/radiance_to_bt_conversion_constant_c2": xr.DataArray(1e2 * h * c / k)} return data - def _get_test_calib_for_channel_vis(self, chroot, meas): - xrda = xr.DataArray - data = {"state/celestial/earth_sun_distance": xrda(da.repeat(da.array([149597870.7]), 6000)), - meas + "/channel_effective_solar_irradiance": xrda(50)} + def _get_test_calib_for_channel_vis(self, meas): + data = {"state/celestial/earth_sun_distance": xr.DataArray(da.repeat(da.array([149597870.7]), 6000)), + meas + "/channel_effective_solar_irradiance": xr.DataArray(50)} return data - def _get_test_content_for_channel(self, pat, ch, grid_type): - xrda = xr.DataArray + def _get_test_content_for_channel(self, ch_str, grid_type): + nrows = GRID_TYPE_INFO_FOR_TEST_CONTENT[grid_type]['nrows'] ncols = GRID_TYPE_INFO_FOR_TEST_CONTENT[grid_type]['ncols'] - chroot = "data/{:s}" - meas = chroot + "/measured" - rad = meas + "/effective_radiance" - qual = meas + "/pixel_quality" - index_map = meas + "/index_map" - rad_conv_coeff = meas + "/radiance_unit_conversion_coefficient" - pos = meas + "/{:s}_position_{:s}" - shp = rad + "/shape" - x = meas + "/x" - y = meas + "/y" + n_rows_cols = (nrows, ncols) + data = {} - ch_str = pat.format(ch) - ch_path = rad.format(ch_str) + + self._get_test_image_data_for_channel(data, ch_str, n_rows_cols) + self._get_test_calib_data_for_channel(data, ch_str) + self._get_test_geolocation_for_channel(data, ch_str, grid_type, n_rows_cols) + self._get_test_pixel_quality_for_channel(data, ch_str, n_rows_cols) + self._get_test_index_map_for_channel(data, ch_str, n_rows_cols) + self._get_test_chunk_position_for_channel(data, ch_str, n_rows_cols) + + return data + + def _get_test_calib_data_for_channel(self, data, ch_str): + meas_path = "data/{:s}/measured".format(ch_str) + if ch_str.startswith("ir") or ch_str.startswith("wv"): + data.update(self._get_test_calib_for_channel_ir(meas_path)) + elif ch_str.startswith("vis") or ch_str.startswith("nir"): + data.update(self._get_test_calib_for_channel_vis(meas_path)) + data[meas_path + "/radiance_unit_conversion_coefficient"] = xr.DataArray(1234.56) + + def _get_test_image_data_for_channel(self, data, ch_str, n_rows_cols): + ch_path = "data/{:s}/measured/effective_radiance".format(ch_str) common_attrs = { "scale_factor": 5, @@ -103,10 +111,10 @@ def _get_test_content_for_channel(self, pat, ch, grid_type): "units": "mW.m-2.sr-1.(cm-1)-1", "ancillary_variables": "pixel_quality" } - if ch == 38: - fire_line = da.ones((1, ncols), dtype="uint16", chunks=1024) * 5000 - data_without_fires = da.ones((nrows - 1, ncols), dtype="uint16", chunks=1024) - d = xrda( + if "38" in ch_path: + fire_line = da.ones((1, n_rows_cols[1]), dtype="uint16", chunks=1024) * 5000 + data_without_fires = da.ones((n_rows_cols[0] - 1, n_rows_cols[1]), dtype="uint16", chunks=1024) + d = xr.DataArray( da.concatenate([fire_line, data_without_fires], axis=0), dims=("y", "x"), attrs={ @@ -117,8 +125,8 @@ def _get_test_content_for_channel(self, pat, ch, grid_type): } ) else: - d = xrda( - da.ones((nrows, ncols), dtype="uint16", chunks=1024), + d = xr.DataArray( + da.ones(n_rows_cols, dtype="uint16", chunks=1024), dims=("y", "x"), attrs={ "valid_range": [0, 4095], @@ -129,45 +137,49 @@ def _get_test_content_for_channel(self, pat, ch, grid_type): ) data[ch_path] = d - data[x.format(ch_str)] = xrda( - da.arange(1, ncols + 1, dtype="uint16"), + data[ch_path+'/shape'] = n_rows_cols + + def _get_test_chunk_position_for_channel(self, data, ch_str, n_rows_cols): + pos = "data/{:s}/measured/{:s}_position_{:s}" + data[pos.format(ch_str, "start", "row")] = xr.DataArray(0) + data[pos.format(ch_str, "start", "column")] = xr.DataArray(0) + data[pos.format(ch_str, "end", "row")] = xr.DataArray(n_rows_cols[0]) + data[pos.format(ch_str, "end", "column")] = xr.DataArray(n_rows_cols[1]) + + def _get_test_index_map_for_channel(self, data, ch_str, n_rows_cols): + index_map_path = "data/{:s}/measured/index_map".format(ch_str) + data[index_map_path] = xr.DataArray((da.ones(n_rows_cols)) * 110, dims=("y", "x")) + + def _get_test_pixel_quality_for_channel(self, data, ch_str, n_rows_cols): + qual_path = "data/{:s}/measured/pixel_quality".format(ch_str) + data[qual_path] = xr.DataArray((da.ones(n_rows_cols)) * 3, dims=("y", "x")) + + def _get_test_geolocation_for_channel(self, data, ch_str, grid_type, n_rows_cols): + x_path = "data/{:s}/measured/x".format(ch_str) + data[x_path] = xr.DataArray( + da.arange(1, n_rows_cols[1] + 1, dtype=np.dtype("uint16")), dims=("x",), attrs={ "scale_factor": -GRID_TYPE_INFO_FOR_TEST_CONTENT[grid_type]['scale_factor'], "add_offset": GRID_TYPE_INFO_FOR_TEST_CONTENT[grid_type]['add_offset'], } ) - data[y.format(ch_str)] = xrda( - da.arange(1, nrows + 1, dtype="uint16"), + + y_path = "data/{:s}/measured/y".format(ch_str) + data[y_path] = xr.DataArray( + da.arange(1, n_rows_cols[0] + 1, dtype=np.dtype("uint16")), dims=("y",), attrs={ "scale_factor": GRID_TYPE_INFO_FOR_TEST_CONTENT[grid_type]['scale_factor'], "add_offset": -GRID_TYPE_INFO_FOR_TEST_CONTENT[grid_type]['add_offset'], } ) - data[qual.format(ch_str)] = xrda((da.ones((nrows, ncols))) * 3, dims=("y", "x")) - # add dummy data for index map - data[index_map.format(ch_str)] = xrda((da.ones((nrows, ncols))) * 110, dims=("y", "x")) - - data[rad_conv_coeff.format(ch_str)] = xrda(1234.56) - data[pos.format(ch_str, "start", "row")] = xrda(0) - data[pos.format(ch_str, "start", "column")] = xrda(0) - data[pos.format(ch_str, "end", "row")] = xrda(nrows) - data[pos.format(ch_str, "end", "column")] = xrda(ncols) - if pat.startswith("ir") or pat.startswith("wv"): - data.update(self._get_test_calib_for_channel_ir(chroot.format(ch_str), - meas.format(ch_str))) - elif pat.startswith("vis") or pat.startswith("nir"): - data.update(self._get_test_calib_for_channel_vis(chroot.format(ch_str), - meas.format(ch_str))) - data[shp.format(ch_str)] = (nrows, ncols) - return data def _get_test_content_all_channels(self): data = {} for pat in self.chan_patterns: - for ch_num in self.chan_patterns[pat]['channels']: - data.update(self._get_test_content_for_channel(pat, ch_num, self.chan_patterns[pat]['grid_type'])) + for ch in self.chan_patterns[pat]['channels']: + data.update(self._get_test_content_for_channel(pat.format(ch), self.chan_patterns[pat]['grid_type'])) return data def _get_test_content_areadef(self): @@ -196,19 +208,18 @@ def _get_test_content_areadef(self): def _get_test_content_aux_data(self): from satpy.readers.fci_l1c_nc import AUX_DATA - xrda = xr.DataArray data = {} indices_dim = 6000 for key, value in AUX_DATA.items(): # skip population of earth_sun_distance as this is already defined for reflectance calculation if key == 'earth_sun_distance': continue - data[value] = xrda(da.arange(indices_dim, dtype="float32"), dims=("index")) + data[value] = xr.DataArray(da.arange(indices_dim, dtype="float32"), dims=("index")) # compute the last data entry to simulate the FCI caching data[list(AUX_DATA.values())[-1]] = data[list(AUX_DATA.values())[-1]].compute() - data['index'] = xrda(da.ones(indices_dim, dtype="uint16") * 100, dims=("index")) + data['index'] = xr.DataArray(da.ones(indices_dim, dtype="uint16") * 100, dims=("index")) return data def _get_global_attributes(self): @@ -246,7 +257,7 @@ class FakeFCIFileHandlerFDHSI(FakeFCIFileHandlerBase): class FakeFCIFileHandlerWithBadData(FakeFCIFileHandlerFDHSI): """Mock bad data.""" - def _get_test_calib_for_channel_ir(self, chroot, meas): + def _get_test_calib_for_channel_ir(self, meas): from netCDF4 import default_fillvals v = xr.DataArray(default_fillvals["f4"]) data = {meas + "/radiance_to_bt_conversion_coefficient_wavenumber": v, @@ -256,8 +267,8 @@ def _get_test_calib_for_channel_ir(self, chroot, meas): meas + "/radiance_to_bt_conversion_constant_c2": v} return data - def _get_test_calib_for_channel_vis(self, chroot, meas): - data = super()._get_test_calib_for_channel_vis(chroot, meas) + def _get_test_calib_for_channel_vis(self, meas): + data = super()._get_test_calib_for_channel_vis(meas) from netCDF4 import default_fillvals v = xr.DataArray(default_fillvals["f4"]) data[meas + "/channel_effective_solar_irradiance"] = v @@ -267,8 +278,8 @@ def _get_test_calib_for_channel_vis(self, chroot, meas): class FakeFCIFileHandlerWithBadIDPFData(FakeFCIFileHandlerFDHSI): """Mock bad data for IDPF TO-DO's.""" - def _get_test_calib_for_channel_vis(self, chroot, meas): - data = super()._get_test_calib_for_channel_vis(chroot, meas) + def _get_test_calib_for_channel_vis(self, meas): + data = super()._get_test_calib_for_channel_vis(meas) data["state/celestial/earth_sun_distance"] = xr.DataArray(da.repeat(da.array([30000000]), 6000)) return data From c1c9cbd93feb662466dd902b94ef54c6f1e47f1a Mon Sep 17 00:00:00 2001 From: andream Date: Mon, 28 Nov 2022 11:58:57 +0100 Subject: [PATCH 19/26] refactor _get_test_content_for_channel in functions --- satpy/tests/reader_tests/test_fci_l1c_nc.py | 349 ++++++++++---------- 1 file changed, 177 insertions(+), 172 deletions(-) diff --git a/satpy/tests/reader_tests/test_fci_l1c_nc.py b/satpy/tests/reader_tests/test_fci_l1c_nc.py index a5f8afd30c..569ca293e0 100644 --- a/satpy/tests/reader_tests/test_fci_l1c_nc.py +++ b/satpy/tests/reader_tests/test_fci_l1c_nc.py @@ -27,6 +27,7 @@ import numpy.testing import pytest import xarray as xr +from netCDF4 import default_fillvals from satpy.readers.fci_l1c_nc import FCIL1cNCFileHandler from satpy.tests.reader_tests.test_netcdf_utils import FakeNetCDF4FileHandler @@ -54,188 +55,200 @@ } -class FakeFCIFileHandlerBase(FakeNetCDF4FileHandler): - """Class for faking the NetCDF4 Filehandler.""" +def _get_test_calib_for_channel_ir(data, meas_path): + from pyspectral.blackbody import C_SPEED as c + from pyspectral.blackbody import H_PLANCK as h + from pyspectral.blackbody import K_BOLTZMANN as k + data[meas_path + "/radiance_to_bt_conversion_coefficient_wavenumber"] = xr.DataArray(955) + data[meas_path + "/radiance_to_bt_conversion_coefficient_a"] = xr.DataArray(1) + data[meas_path + "/radiance_to_bt_conversion_coefficient_b"] = xr.DataArray(0.4) + data[meas_path + "/radiance_to_bt_conversion_constant_c1"] = xr.DataArray(1e11 * 2 * h * c ** 2) + data[meas_path + "/radiance_to_bt_conversion_constant_c2"] = xr.DataArray(1e2 * h * c / k) + return data - # overwritten by FDHSI and HRFI FIle Handlers - chan_patterns: Dict[str, Dict[str, Union[List[int], str]]] = {} - def _get_test_calib_for_channel_ir(self, meas_path): - from pyspectral.blackbody import C_SPEED as c - from pyspectral.blackbody import H_PLANCK as h - from pyspectral.blackbody import K_BOLTZMANN as k - data = {meas_path + "/radiance_to_bt_conversion_coefficient_wavenumber": xr.DataArray(955), - meas_path + "/radiance_to_bt_conversion_coefficient_a": xr.DataArray(1), - meas_path + "/radiance_to_bt_conversion_coefficient_b": xr.DataArray(0.4), - meas_path + "/radiance_to_bt_conversion_constant_c1": xr.DataArray(1e11 * 2 * h * c ** 2), - meas_path + "/radiance_to_bt_conversion_constant_c2": xr.DataArray(1e2 * h * c / k)} - return data +def _get_test_calib_for_channel_vis(data, meas): + data["state/celestial/earth_sun_distance"] = xr.DataArray(da.repeat(da.array([149597870.7]), 6000)) + data[meas + "/channel_effective_solar_irradiance"] = xr.DataArray(50) + return data - def _get_test_calib_for_channel_vis(self, meas): - data = {"state/celestial/earth_sun_distance": xr.DataArray(da.repeat(da.array([149597870.7]), 6000)), - meas + "/channel_effective_solar_irradiance": xr.DataArray(50)} - return data - - def _get_test_content_for_channel(self, ch_str, grid_type): - nrows = GRID_TYPE_INFO_FOR_TEST_CONTENT[grid_type]['nrows'] - ncols = GRID_TYPE_INFO_FOR_TEST_CONTENT[grid_type]['ncols'] - n_rows_cols = (nrows, ncols) +def _get_test_calib_data_for_channel(data, ch_str): + meas_path = "data/{:s}/measured".format(ch_str) + if ch_str.startswith("ir") or ch_str.startswith("wv"): + _get_test_calib_for_channel_ir(data, meas_path) + elif ch_str.startswith("vis") or ch_str.startswith("nir"): + _get_test_calib_for_channel_vis(data, meas_path) + data[meas_path + "/radiance_unit_conversion_coefficient"] = xr.DataArray(1234.56) - data = {} - self._get_test_image_data_for_channel(data, ch_str, n_rows_cols) - self._get_test_calib_data_for_channel(data, ch_str) - self._get_test_geolocation_for_channel(data, ch_str, grid_type, n_rows_cols) - self._get_test_pixel_quality_for_channel(data, ch_str, n_rows_cols) - self._get_test_index_map_for_channel(data, ch_str, n_rows_cols) - self._get_test_chunk_position_for_channel(data, ch_str, n_rows_cols) +def _get_test_image_data_for_channel(data, ch_str, n_rows_cols): + ch_path = "data/{:s}/measured/effective_radiance".format(ch_str) - return data - - def _get_test_calib_data_for_channel(self, data, ch_str): - meas_path = "data/{:s}/measured".format(ch_str) - if ch_str.startswith("ir") or ch_str.startswith("wv"): - data.update(self._get_test_calib_for_channel_ir(meas_path)) - elif ch_str.startswith("vis") or ch_str.startswith("nir"): - data.update(self._get_test_calib_for_channel_vis(meas_path)) - data[meas_path + "/radiance_unit_conversion_coefficient"] = xr.DataArray(1234.56) - - def _get_test_image_data_for_channel(self, data, ch_str, n_rows_cols): - ch_path = "data/{:s}/measured/effective_radiance".format(ch_str) - - common_attrs = { - "scale_factor": 5, - "add_offset": 10, - "long_name": "Effective Radiance", - "units": "mW.m-2.sr-1.(cm-1)-1", - "ancillary_variables": "pixel_quality" - } - if "38" in ch_path: - fire_line = da.ones((1, n_rows_cols[1]), dtype="uint16", chunks=1024) * 5000 - data_without_fires = da.ones((n_rows_cols[0] - 1, n_rows_cols[1]), dtype="uint16", chunks=1024) - d = xr.DataArray( - da.concatenate([fire_line, data_without_fires], axis=0), - dims=("y", "x"), - attrs={ - "valid_range": [0, 8191], - "warm_scale_factor": 2, - "warm_add_offset": -300, - **common_attrs - } - ) - else: - d = xr.DataArray( - da.ones(n_rows_cols, dtype="uint16", chunks=1024), - dims=("y", "x"), - attrs={ - "valid_range": [0, 4095], - "warm_scale_factor": 1, - "warm_add_offset": 0, - **common_attrs - } - ) - - data[ch_path] = d - data[ch_path+'/shape'] = n_rows_cols - - def _get_test_chunk_position_for_channel(self, data, ch_str, n_rows_cols): - pos = "data/{:s}/measured/{:s}_position_{:s}" - data[pos.format(ch_str, "start", "row")] = xr.DataArray(0) - data[pos.format(ch_str, "start", "column")] = xr.DataArray(0) - data[pos.format(ch_str, "end", "row")] = xr.DataArray(n_rows_cols[0]) - data[pos.format(ch_str, "end", "column")] = xr.DataArray(n_rows_cols[1]) - - def _get_test_index_map_for_channel(self, data, ch_str, n_rows_cols): - index_map_path = "data/{:s}/measured/index_map".format(ch_str) - data[index_map_path] = xr.DataArray((da.ones(n_rows_cols)) * 110, dims=("y", "x")) - - def _get_test_pixel_quality_for_channel(self, data, ch_str, n_rows_cols): - qual_path = "data/{:s}/measured/pixel_quality".format(ch_str) - data[qual_path] = xr.DataArray((da.ones(n_rows_cols)) * 3, dims=("y", "x")) - - def _get_test_geolocation_for_channel(self, data, ch_str, grid_type, n_rows_cols): - x_path = "data/{:s}/measured/x".format(ch_str) - data[x_path] = xr.DataArray( - da.arange(1, n_rows_cols[1] + 1, dtype=np.dtype("uint16")), - dims=("x",), + common_attrs = { + "scale_factor": 5, + "add_offset": 10, + "long_name": "Effective Radiance", + "units": "mW.m-2.sr-1.(cm-1)-1", + "ancillary_variables": "pixel_quality" + } + if "38" in ch_path: + fire_line = da.ones((1, n_rows_cols[1]), dtype="uint16", chunks=1024) * 5000 + data_without_fires = da.ones((n_rows_cols[0] - 1, n_rows_cols[1]), dtype="uint16", chunks=1024) + d = xr.DataArray( + da.concatenate([fire_line, data_without_fires], axis=0), + dims=("y", "x"), attrs={ - "scale_factor": -GRID_TYPE_INFO_FOR_TEST_CONTENT[grid_type]['scale_factor'], - "add_offset": GRID_TYPE_INFO_FOR_TEST_CONTENT[grid_type]['add_offset'], + "valid_range": [0, 8191], + "warm_scale_factor": 2, + "warm_add_offset": -300, + **common_attrs } ) - - y_path = "data/{:s}/measured/y".format(ch_str) - data[y_path] = xr.DataArray( - da.arange(1, n_rows_cols[0] + 1, dtype=np.dtype("uint16")), - dims=("y",), + else: + d = xr.DataArray( + da.ones(n_rows_cols, dtype="uint16", chunks=1024), + dims=("y", "x"), attrs={ - "scale_factor": GRID_TYPE_INFO_FOR_TEST_CONTENT[grid_type]['scale_factor'], - "add_offset": -GRID_TYPE_INFO_FOR_TEST_CONTENT[grid_type]['add_offset'], + "valid_range": [0, 4095], + "warm_scale_factor": 1, + "warm_add_offset": 0, + **common_attrs } ) - def _get_test_content_all_channels(self): - data = {} - for pat in self.chan_patterns: - for ch in self.chan_patterns[pat]['channels']: - data.update(self._get_test_content_for_channel(pat.format(ch), self.chan_patterns[pat]['grid_type'])) - return data + data[ch_path] = d + data[ch_path + '/shape'] = n_rows_cols - def _get_test_content_areadef(self): - data = {} - proj = "data/mtg_geos_projection" - - attrs = { - "sweep_angle_axis": "y", - "perspective_point_height": "35786400.0", - "semi_major_axis": "6378137.0", - "longitude_of_projection_origin": "0.0", - "inverse_flattening": "298.257223563", - "units": "m"} - data[proj] = xr.DataArray( - 0, - dims=(), - attrs=attrs) - - # also set attributes cached, as this may be how they are accessed with - # the NetCDF4FileHandler - for (k, v) in attrs.items(): - data[proj + "/attr/" + k] = v +def _get_test_chunk_position_for_channel(data, ch_str, n_rows_cols): + pos = "data/{:s}/measured/{:s}_position_{:s}" + data[pos.format(ch_str, "start", "row")] = xr.DataArray(0) + data[pos.format(ch_str, "start", "column")] = xr.DataArray(0) + data[pos.format(ch_str, "end", "row")] = xr.DataArray(n_rows_cols[0]) + data[pos.format(ch_str, "end", "column")] = xr.DataArray(n_rows_cols[1]) - return data - def _get_test_content_aux_data(self): - from satpy.readers.fci_l1c_nc import AUX_DATA - data = {} - indices_dim = 6000 - for key, value in AUX_DATA.items(): - # skip population of earth_sun_distance as this is already defined for reflectance calculation - if key == 'earth_sun_distance': - continue - data[value] = xr.DataArray(da.arange(indices_dim, dtype="float32"), dims=("index")) +def _get_test_index_map_for_channel(data, ch_str, n_rows_cols): + index_map_path = "data/{:s}/measured/index_map".format(ch_str) + data[index_map_path] = xr.DataArray((da.ones(n_rows_cols)) * 110, dims=("y", "x")) - # compute the last data entry to simulate the FCI caching - data[list(AUX_DATA.values())[-1]] = data[list(AUX_DATA.values())[-1]].compute() - data['index'] = xr.DataArray(da.ones(indices_dim, dtype="uint16") * 100, dims=("index")) - return data +def _get_test_pixel_quality_for_channel(data, ch_str, n_rows_cols): + qual_path = "data/{:s}/measured/pixel_quality".format(ch_str) + data[qual_path] = xr.DataArray((da.ones(n_rows_cols)) * 3, dims=("y", "x")) + + +def _get_test_geolocation_for_channel(data, ch_str, grid_type, n_rows_cols): + x_path = "data/{:s}/measured/x".format(ch_str) + data[x_path] = xr.DataArray( + da.arange(1, n_rows_cols[1] + 1, dtype=np.dtype("uint16")), + dims=("x",), + attrs={ + "scale_factor": -GRID_TYPE_INFO_FOR_TEST_CONTENT[grid_type]['scale_factor'], + "add_offset": GRID_TYPE_INFO_FOR_TEST_CONTENT[grid_type]['add_offset'], + } + ) + + y_path = "data/{:s}/measured/y".format(ch_str) + data[y_path] = xr.DataArray( + da.arange(1, n_rows_cols[0] + 1, dtype=np.dtype("uint16")), + dims=("y",), + attrs={ + "scale_factor": GRID_TYPE_INFO_FOR_TEST_CONTENT[grid_type]['scale_factor'], + "add_offset": -GRID_TYPE_INFO_FOR_TEST_CONTENT[grid_type]['add_offset'], + } + ) + + +def _get_test_content_areadef(): + data = {} + + proj = "data/mtg_geos_projection" + + attrs = { + "sweep_angle_axis": "y", + "perspective_point_height": "35786400.0", + "semi_major_axis": "6378137.0", + "longitude_of_projection_origin": "0.0", + "inverse_flattening": "298.257223563", + "units": "m"} + data[proj] = xr.DataArray( + 0, + dims=(), + attrs=attrs) - def _get_global_attributes(self): + # also set attributes cached, as this may be how they are accessed with + # the NetCDF4FileHandler + for (k, v) in attrs.items(): + data[proj + "/attr/" + k] = v + + return data + + +def _get_test_content_aux_data(): + from satpy.readers.fci_l1c_nc import AUX_DATA + data = {} + indices_dim = 6000 + for key, value in AUX_DATA.items(): + # skip population of earth_sun_distance as this is already defined for reflectance calculation + if key == 'earth_sun_distance': + continue + data[value] = xr.DataArray(da.arange(indices_dim, dtype="float32"), dims=("index")) + + # compute the last data entry to simulate the FCI caching + data[list(AUX_DATA.values())[-1]] = data[list(AUX_DATA.values())[-1]].compute() + + data['index'] = xr.DataArray(da.ones(indices_dim, dtype="uint16") * 100, dims=("index")) + return data + + +def _get_global_attributes(): + data = {} + attrs = {"platform": "MTI1"} + for (k, v) in attrs.items(): + data["/attr/" + k] = v + return data + + +def _get_test_content_for_channel(ch_str, grid_type): + + nrows = GRID_TYPE_INFO_FOR_TEST_CONTENT[grid_type]['nrows'] + ncols = GRID_TYPE_INFO_FOR_TEST_CONTENT[grid_type]['ncols'] + n_rows_cols = (nrows, ncols) + + data = {} + + _get_test_image_data_for_channel(data, ch_str, n_rows_cols) + _get_test_calib_data_for_channel(data, ch_str) + _get_test_geolocation_for_channel(data, ch_str, grid_type, n_rows_cols) + _get_test_pixel_quality_for_channel(data, ch_str, n_rows_cols) + _get_test_index_map_for_channel(data, ch_str, n_rows_cols) + _get_test_chunk_position_for_channel(data, ch_str, n_rows_cols) + + return data + + +class FakeFCIFileHandlerBase(FakeNetCDF4FileHandler): + """Class for faking the NetCDF4 Filehandler.""" + + # overwritten by FDHSI and HRFI FIle Handlers + chan_patterns: Dict[str, Dict[str, Union[List[int], str]]] = {} + + def _get_test_content_all_channels(self): data = {} - attrs = {"platform": "MTI1"} - for (k, v) in attrs.items(): - data["/attr/" + k] = v + for pat in self.chan_patterns: + for ch in self.chan_patterns[pat]['channels']: + data.update(_get_test_content_for_channel(pat.format(ch), self.chan_patterns[pat]['grid_type'])) return data def get_test_content(self, filename, filename_info, filetype_info): """Get the content of the test data.""" D = {} D.update(self._get_test_content_all_channels()) - D.update(self._get_test_content_areadef()) - D.update(self._get_test_content_aux_data()) - D.update(self._get_global_attributes()) + D.update(_get_test_content_areadef()) + D.update(_get_test_content_aux_data()) + D.update(_get_global_attributes()) return D @@ -257,32 +270,23 @@ class FakeFCIFileHandlerFDHSI(FakeFCIFileHandlerBase): class FakeFCIFileHandlerWithBadData(FakeFCIFileHandlerFDHSI): """Mock bad data.""" - def _get_test_calib_for_channel_ir(self, meas): - from netCDF4 import default_fillvals + def _get_test_content_all_channels(self): + data = super()._get_test_content_all_channels() v = xr.DataArray(default_fillvals["f4"]) - data = {meas + "/radiance_to_bt_conversion_coefficient_wavenumber": v, - meas + "/radiance_to_bt_conversion_coefficient_a": v, - meas + "/radiance_to_bt_conversion_coefficient_b": v, - meas + "/radiance_to_bt_conversion_constant_c1": v, - meas + "/radiance_to_bt_conversion_constant_c2": v} - return data - def _get_test_calib_for_channel_vis(self, meas): - data = super()._get_test_calib_for_channel_vis(meas) - from netCDF4 import default_fillvals - v = xr.DataArray(default_fillvals["f4"]) - data[meas + "/channel_effective_solar_irradiance"] = v + data.update({"data/ir_105/measured/radiance_to_bt_conversion_coefficient_wavenumber": v, + "data/ir_105/measured/radiance_to_bt_conversion_coefficient_a": v, + "data/ir_105/measured/radiance_to_bt_conversion_coefficient_b": v, + "data/ir_105/measured/radiance_to_bt_conversion_constant_c1": v, + "data/ir_105/measured/radiance_to_bt_conversion_constant_c2": v, + "data/vis_06/measured/channel_effective_solar_irradiance": v}) + return data class FakeFCIFileHandlerWithBadIDPFData(FakeFCIFileHandlerFDHSI): """Mock bad data for IDPF TO-DO's.""" - def _get_test_calib_for_channel_vis(self, meas): - data = super()._get_test_calib_for_channel_vis(meas) - data["state/celestial/earth_sun_distance"] = xr.DataArray(da.repeat(da.array([30000000]), 6000)) - return data - def _get_test_content_all_channels(self): data = super()._get_test_content_all_channels() data['data/vis_06/measured/x'].attrs['scale_factor'] *= -1 @@ -295,6 +299,7 @@ def _get_test_content_all_channels(self): data['data/vis_06/measured/y'].attrs['add_offset'] = \ np.float32(data['data/vis_06/measured/y'].attrs['add_offset']) + data["state/celestial/earth_sun_distance"] = xr.DataArray(da.repeat(da.array([30000000]), 6000)) return data From 98bddc9e7cf87bafe0aa7de834b230f1999ee97c Mon Sep 17 00:00:00 2001 From: andream Date: Mon, 28 Nov 2022 12:16:25 +0100 Subject: [PATCH 20/26] refactor parametrisation to avoid too many arguments --- satpy/tests/reader_tests/test_fci_l1c_nc.py | 181 +++++++++++--------- 1 file changed, 99 insertions(+), 82 deletions(-) diff --git a/satpy/tests/reader_tests/test_fci_l1c_nc.py b/satpy/tests/reader_tests/test_fci_l1c_nc.py index 569ca293e0..b42bab60be 100644 --- a/satpy/tests/reader_tests/test_fci_l1c_nc.py +++ b/satpy/tests/reader_tests/test_fci_l1c_nc.py @@ -369,6 +369,19 @@ def mocked_basefilehandler(filehandler): class TestFCIL1cNCReader: """Test FCI L1c NetCDF reader with nominal data.""" + fh_param_for_filetype = { + 'fdhsi': { + 'filehandler': FakeFCIFileHandlerFDHSI, + 'channels': _chans_fdhsi, + 'filenames': _test_filenames['fdhsi'] + }, + 'hrfi': { + 'filehandler': FakeFCIFileHandlerHRFI, + 'channels': _chans_hrfi, + 'filenames': _test_filenames['hrfi'] + } + } + @pytest.mark.parametrize('filenames', [_test_filenames['fdhsi'], _test_filenames['hrfi']]) def test_file_pattern(self, reader_configs, filenames): """Test file pattern matching.""" @@ -388,21 +401,22 @@ def test_file_pattern_for_TRAIL_file(self, reader_configs, filenames): files = reader.select_files_from_pathnames(filenames) assert len(files) == 0 - @pytest.mark.parametrize('filehandler,channels,filenames,expected_res_n', [ - (FakeFCIFileHandlerFDHSI, _chans_fdhsi, _test_filenames['fdhsi'], 16), - (FakeFCIFileHandlerHRFI, _chans_hrfi, _test_filenames['hrfi'], 4) + @pytest.mark.parametrize('fh_param,expected_res_n', [ + (fh_param_for_filetype['fdhsi'], 16), + (fh_param_for_filetype['hrfi'], 4) ]) - def test_load_counts(self, reader_configs, filehandler, channels, filenames, + def test_load_counts(self, reader_configs, fh_param, expected_res_n): """Test loading with counts.""" - with mocked_basefilehandler(filehandler): - reader = _get_reader_with_filehandlers(filenames, reader_configs) + with mocked_basefilehandler(fh_param['filehandler']): + reader = _get_reader_with_filehandlers(fh_param['filenames'], reader_configs) res = reader.load( [make_dataid(name=name, calibration="counts") for name in - channels["solar"] + channels["terran"]], pad_data=False) + fh_param['channels']["solar"] + fh_param['channels']["terran"]], pad_data=False) assert expected_res_n == len(res) - for ch, grid_type in zip(channels["solar"] + channels["terran"], - channels["solar_grid_type"] + channels["terran_grid_type"]): + for ch, grid_type in zip(fh_param['channels']["solar"] + fh_param['channels']["terran"], + fh_param['channels']["solar_grid_type"] + + fh_param['channels']["terran_grid_type"]): assert res[ch].shape == (GRID_TYPE_INFO_FOR_TEST_CONTENT[grid_type]['nrows'], GRID_TYPE_INFO_FOR_TEST_CONTENT[grid_type]['ncols']) assert res[ch].dtype == np.uint16 @@ -414,21 +428,22 @@ def test_load_counts(self, reader_configs, filehandler, channels, filenames, else: numpy.testing.assert_array_equal(res[ch], 1) - @pytest.mark.parametrize('filehandler,channels,filenames,expected_res_n', [ - (FakeFCIFileHandlerFDHSI, _chans_fdhsi, _test_filenames['fdhsi'], 16), - (FakeFCIFileHandlerHRFI, _chans_hrfi, _test_filenames['hrfi'], 4) + @pytest.mark.parametrize('fh_param,expected_res_n', [ + (fh_param_for_filetype['fdhsi'], 16), + (fh_param_for_filetype['hrfi'], 4) ]) - def test_load_radiance(self, reader_configs, filehandler, channels, filenames, + def test_load_radiance(self, reader_configs, fh_param, expected_res_n): """Test loading with radiance.""" - with mocked_basefilehandler(filehandler): - reader = _get_reader_with_filehandlers(filenames, reader_configs) + with mocked_basefilehandler(fh_param['filehandler']): + reader = _get_reader_with_filehandlers(fh_param['filenames'], reader_configs) res = reader.load( [make_dataid(name=name, calibration="radiance") for name in - channels["solar"] + channels["terran"]], pad_data=False) + fh_param['channels']["solar"] + fh_param['channels']["terran"]], pad_data=False) assert expected_res_n == len(res) - for ch, grid_type in zip(channels["solar"] + channels["terran"], - channels["solar_grid_type"] + channels["terran_grid_type"]): + for ch, grid_type in zip(fh_param['channels']["solar"] + fh_param['channels']["terran"], + fh_param['channels']["solar_grid_type"] + + fh_param['channels']["terran_grid_type"]): assert res[ch].shape == (GRID_TYPE_INFO_FOR_TEST_CONTENT[grid_type]['nrows'], GRID_TYPE_INFO_FOR_TEST_CONTENT[grid_type]['ncols']) assert res[ch].dtype == np.float64 @@ -441,20 +456,20 @@ def test_load_radiance(self, reader_configs, filehandler, channels, filenames, else: numpy.testing.assert_array_equal(res[ch], 15) - @pytest.mark.parametrize('filehandler,channels,filenames,expected_res_n', [ - (FakeFCIFileHandlerFDHSI, _chans_fdhsi, _test_filenames['fdhsi'], 8), - (FakeFCIFileHandlerHRFI, _chans_hrfi, _test_filenames['hrfi'], 2) + @pytest.mark.parametrize('fh_param,expected_res_n', [ + (fh_param_for_filetype['fdhsi'], 8), + (fh_param_for_filetype['hrfi'], 2) ]) - def test_load_reflectance(self, reader_configs, filehandler, channels, filenames, + def test_load_reflectance(self, reader_configs, fh_param, expected_res_n): """Test loading with reflectance.""" - with mocked_basefilehandler(filehandler): - reader = _get_reader_with_filehandlers(filenames, reader_configs) + with mocked_basefilehandler(fh_param['filehandler']): + reader = _get_reader_with_filehandlers(fh_param['filenames'], reader_configs) res = reader.load( [make_dataid(name=name, calibration="reflectance") for name in - channels["solar"]], pad_data=False) + fh_param['channels']["solar"]], pad_data=False) assert expected_res_n == len(res) - for ch, grid_type in zip(channels["solar"], channels["solar_grid_type"]): + for ch, grid_type in zip(fh_param['channels']["solar"], fh_param['channels']["solar_grid_type"]): assert res[ch].shape == (GRID_TYPE_INFO_FOR_TEST_CONTENT[grid_type]['nrows'], GRID_TYPE_INFO_FOR_TEST_CONTENT[grid_type]['ncols']) assert res[ch].dtype == np.float64 @@ -462,22 +477,22 @@ def test_load_reflectance(self, reader_configs, filehandler, channels, filenames assert res[ch].attrs["units"] == "%" numpy.testing.assert_array_almost_equal(res[ch], 100 * 15 * 1 * np.pi / 50) - @pytest.mark.parametrize('filehandler,channels,filenames,expected_res_n', [ - (FakeFCIFileHandlerFDHSI, _chans_fdhsi, _test_filenames['fdhsi'], 8), - (FakeFCIFileHandlerHRFI, _chans_hrfi, _test_filenames['hrfi'], 2) + @pytest.mark.parametrize('fh_param,expected_res_n', [ + (fh_param_for_filetype['fdhsi'], 8), + (fh_param_for_filetype['hrfi'], 2) ]) - def test_load_bt(self, reader_configs, caplog, filehandler, channels, filenames, + def test_load_bt(self, reader_configs, caplog, fh_param, expected_res_n): """Test loading with bt.""" - with mocked_basefilehandler(filehandler): - reader = _get_reader_with_filehandlers(filenames, reader_configs) + with mocked_basefilehandler(fh_param['filehandler']): + reader = _get_reader_with_filehandlers(fh_param['filenames'], reader_configs) with caplog.at_level(logging.WARNING): res = reader.load( [make_dataid(name=name, calibration="brightness_temperature") for - name in channels["terran"]], pad_data=False) + name in fh_param['channels']["terran"]], pad_data=False) assert caplog.text == "" assert expected_res_n == len(res) - for ch, grid_type in zip(channels["terran"], channels["terran_grid_type"]): + for ch, grid_type in zip(fh_param['channels']["terran"], fh_param['channels']["terran_grid_type"]): assert res[ch].shape == (GRID_TYPE_INFO_FOR_TEST_CONTENT[grid_type]['nrows'], GRID_TYPE_INFO_FOR_TEST_CONTENT[grid_type]['ncols']) assert res[ch].dtype == np.float64 @@ -490,19 +505,19 @@ def test_load_bt(self, reader_configs, caplog, filehandler, channels, filenames, else: numpy.testing.assert_array_almost_equal(res[ch], 209.68274099) - @pytest.mark.parametrize('filehandler,channels,filenames', [ - (FakeFCIFileHandlerFDHSI, _chans_fdhsi, _test_filenames['fdhsi']), - (FakeFCIFileHandlerHRFI, _chans_hrfi, _test_filenames['hrfi']) + @pytest.mark.parametrize('fh_param', [ + (fh_param_for_filetype['fdhsi']), + (fh_param_for_filetype['hrfi']) ]) - def test_orbital_parameters_attr(self, reader_configs, filehandler, channels, filenames): + def test_orbital_parameters_attr(self, reader_configs, fh_param): """Test the orbital parameter attribute.""" - with mocked_basefilehandler(filehandler): - reader = _get_reader_with_filehandlers(filenames, reader_configs) + with mocked_basefilehandler(fh_param['filehandler']): + reader = _get_reader_with_filehandlers(fh_param['filenames'], reader_configs) res = reader.load( [make_dataid(name=name) for name in - channels["solar"] + channels["terran"]], pad_data=False) + fh_param['channels']["solar"] + fh_param['channels']["terran"]], pad_data=False) - for ch in channels["solar"] + channels["terran"]: + for ch in fh_param['channels']["solar"] + fh_param['channels']["terran"]: assert res[ch].attrs["orbital_parameters"] == { 'satellite_actual_longitude': np.mean(np.arange(6000)), 'satellite_actual_latitude': np.mean(np.arange(6000)), @@ -515,40 +530,41 @@ def test_orbital_parameters_attr(self, reader_configs, filehandler, channels, fi 'projection_altitude': 35786400.0, } - @pytest.mark.parametrize('filehandler,channels,filenames,expected_res_n', [ - (FakeFCIFileHandlerFDHSI, _chans_fdhsi, _test_filenames['fdhsi'], 16), - (FakeFCIFileHandlerHRFI, _chans_hrfi, _test_filenames['hrfi'], 4) + @pytest.mark.parametrize('fh_param,expected_res_n', [ + (fh_param_for_filetype['fdhsi'], 16), + (fh_param_for_filetype['hrfi'], 4) ]) - def test_load_index_map(self, reader_configs, filehandler, channels, filenames, expected_res_n): + def test_load_index_map(self, reader_configs, fh_param, expected_res_n): """Test loading of index_map.""" - with mocked_basefilehandler(filehandler): - reader = _get_reader_with_filehandlers(filenames, reader_configs) + with mocked_basefilehandler(fh_param['filehandler']): + reader = _get_reader_with_filehandlers(fh_param['filenames'], reader_configs) res = reader.load( [name + '_index_map' for name in - channels["solar"] + channels["terran"]], pad_data=False) + fh_param['channels']["solar"] + fh_param['channels']["terran"]], pad_data=False) assert expected_res_n == len(res) - for ch, grid_type in zip(channels["solar"] + channels["terran"], - channels["solar_grid_type"] + channels["terran_grid_type"]): + for ch, grid_type in zip(fh_param['channels']["solar"] + fh_param['channels']["terran"], + fh_param['channels']["solar_grid_type"] + + fh_param['channels']["terran_grid_type"]): assert res[ch + '_index_map'].shape == (GRID_TYPE_INFO_FOR_TEST_CONTENT[grid_type]['nrows'], GRID_TYPE_INFO_FOR_TEST_CONTENT[grid_type]['ncols']) numpy.testing.assert_array_equal(res[ch + '_index_map'][1, 1], 110) - @pytest.mark.parametrize('filehandler,channels,filenames', [ - (FakeFCIFileHandlerFDHSI, _chans_fdhsi, _test_filenames['fdhsi']), - (FakeFCIFileHandlerHRFI, _chans_hrfi, _test_filenames['hrfi']) + @pytest.mark.parametrize('fh_param', [ + (fh_param_for_filetype['fdhsi']), + (fh_param_for_filetype['hrfi']) ]) - def test_load_aux_data(self, reader_configs, filehandler, channels, filenames): + def test_load_aux_data(self, reader_configs, fh_param): """Test loading of auxiliary data.""" from satpy.readers.fci_l1c_nc import AUX_DATA - with mocked_basefilehandler(filehandler): - reader = _get_reader_with_filehandlers(filenames, reader_configs) - res = reader.load([channels['solar'][0] + '_' + key for key in AUX_DATA.keys()], + with mocked_basefilehandler(fh_param['filehandler']): + reader = _get_reader_with_filehandlers(fh_param['filenames'], reader_configs) + res = reader.load([fh_param['channels']['solar'][0] + '_' + key for key in AUX_DATA.keys()], pad_data=False) - grid_type = channels['solar_grid_type'][0] - for aux in [channels['solar'][0] + '_' + key for key in AUX_DATA.keys()]: + grid_type = fh_param['channels']['solar_grid_type'][0] + for aux in [fh_param['channels']['solar'][0] + '_' + key for key in AUX_DATA.keys()]: assert res[aux].shape == (GRID_TYPE_INFO_FOR_TEST_CONTENT[grid_type]['nrows'], GRID_TYPE_INFO_FOR_TEST_CONTENT[grid_type]['ncols']) - if aux == channels['solar'][0] + '_earth_sun_distance': + if aux == fh_param['channels']['solar'][0] + '_earth_sun_distance': numpy.testing.assert_array_equal(res[aux][1, 1], 149597870.7) else: numpy.testing.assert_array_equal(res[aux][1, 1], 10) @@ -564,37 +580,38 @@ def test_load_composite(self): assert len(comps["fci"]) > 0 assert len(mods["fci"]) > 0 - @pytest.mark.parametrize('filehandler,channels,filenames,expected_res_n', [ - (FakeFCIFileHandlerFDHSI, _chans_fdhsi, _test_filenames['fdhsi'], 16), - (FakeFCIFileHandlerHRFI, _chans_hrfi, _test_filenames['hrfi'], 4) + @pytest.mark.parametrize('fh_param,expected_res_n', [ + (fh_param_for_filetype['fdhsi'], 16), + (fh_param_for_filetype['hrfi'], 4) ]) - def test_load_quality_only(self, reader_configs, filehandler, channels, filenames, expected_res_n): + def test_load_quality_only(self, reader_configs, fh_param, expected_res_n): """Test that loading quality only works.""" - with mocked_basefilehandler(filehandler): - reader = _get_reader_with_filehandlers(filenames, reader_configs) + with mocked_basefilehandler(fh_param['filehandler']): + reader = _get_reader_with_filehandlers(fh_param['filenames'], reader_configs) res = reader.load( [name + '_pixel_quality' for name in - channels["solar"] + channels["terran"]], pad_data=False) + fh_param['channels']["solar"] + fh_param['channels']["terran"]], pad_data=False) assert expected_res_n == len(res) - for ch, grid_type in zip(channels["solar"] + channels["terran"], - channels["solar_grid_type"] + channels["terran_grid_type"]): + for ch, grid_type in zip(fh_param['channels']["solar"] + fh_param['channels']["terran"], + fh_param['channels']["solar_grid_type"] + + fh_param['channels']["terran_grid_type"]): assert res[ch + '_pixel_quality'].shape == (GRID_TYPE_INFO_FOR_TEST_CONTENT[grid_type]['nrows'], GRID_TYPE_INFO_FOR_TEST_CONTENT[grid_type]['ncols']) numpy.testing.assert_array_equal(res[ch + '_pixel_quality'][1, 1], 3) assert res[ch + '_pixel_quality'].attrs["name"] == ch + '_pixel_quality' - @pytest.mark.parametrize('filehandler,filenames', [ - (FakeFCIFileHandlerFDHSI, _test_filenames['fdhsi']), - (FakeFCIFileHandlerHRFI, _test_filenames['hrfi']), + @pytest.mark.parametrize('fh_param', [ + (fh_param_for_filetype['fdhsi']), + (fh_param_for_filetype['hrfi']) ]) - def test_platform_name(self, reader_configs, filehandler, filenames): + def test_platform_name(self, reader_configs, fh_param): """Test that platform name is exposed. Test that the FCI reader exposes the platform name. Corresponds to GH issue 1014. """ - with mocked_basefilehandler(filehandler): - reader = _get_reader_with_filehandlers(filenames, reader_configs) + with mocked_basefilehandler(fh_param['filehandler']): + reader = _get_reader_with_filehandlers(fh_param['filenames'], reader_configs) res = reader.load(["vis_06"], pad_data=False) assert res["vis_06"].attrs["platform_name"] == "MTG-I1" @@ -610,14 +627,14 @@ def test_excs(self, reader_configs): make_dataid(name="ir_123", calibration="unknown"), {"units": "unknown"}) - @pytest.mark.parametrize('filehandler,filenames, expected_area', [ - (FakeFCIFileHandlerFDHSI, _test_filenames['fdhsi'], ['mtg_fci_fdss_1km', 'mtg_fci_fdss_2km']), - (FakeFCIFileHandlerHRFI, _test_filenames['hrfi'], ['mtg_fci_fdss_500m', 'mtg_fci_fdss_1km']), + @pytest.mark.parametrize('fh_param, expected_area', [ + (fh_param_for_filetype['fdhsi'], ['mtg_fci_fdss_1km', 'mtg_fci_fdss_2km']), + (fh_param_for_filetype['hrfi'], ['mtg_fci_fdss_500m', 'mtg_fci_fdss_1km']), ]) - def test_area_definition_computation(self, reader_configs, filehandler, filenames, expected_area): + def test_area_definition_computation(self, reader_configs, fh_param, expected_area): """Test that the geolocation computation is correct.""" - with mocked_basefilehandler(filehandler): - reader = _get_reader_with_filehandlers(filenames, reader_configs) + with mocked_basefilehandler(fh_param['filehandler']): + reader = _get_reader_with_filehandlers(fh_param['filenames'], reader_configs) res = reader.load(['ir_105', 'vis_06'], pad_data=False) # test that area_ids are harmonisation-conform ___ From c224148728831df84db171803fa2dfa88a8d70f2 Mon Sep 17 00:00:00 2001 From: andream Date: Mon, 28 Nov 2022 14:13:11 +0100 Subject: [PATCH 21/26] add test for get_segment_position_info --- satpy/readers/fci_l1c_nc.py | 1 + satpy/tests/reader_tests/test_fci_l1c_nc.py | 36 +++++++++++++++++++-- 2 files changed, 34 insertions(+), 3 deletions(-) diff --git a/satpy/readers/fci_l1c_nc.py b/satpy/readers/fci_l1c_nc.py index b9e7f55b21..df76f5150d 100644 --- a/satpy/readers/fci_l1c_nc.py +++ b/satpy/readers/fci_l1c_nc.py @@ -226,6 +226,7 @@ def get_segment_position_info(self): As the final array is composed by stacking chunks (aka segments) vertically, the position of a chunk inside the array is defined by the numbers of the start (lowest) and end (highest) row of the chunk. + The row numbering is assumed to start with 1. This info is used in the GEOVariableSegmentYAMLReader to compute optimal chunk sizes for missing chunks. """ vis_06_measured_path = self.get_channel_measured_group_path('vis_06') diff --git a/satpy/tests/reader_tests/test_fci_l1c_nc.py b/satpy/tests/reader_tests/test_fci_l1c_nc.py index b42bab60be..d005e719f8 100644 --- a/satpy/tests/reader_tests/test_fci_l1c_nc.py +++ b/satpy/tests/reader_tests/test_fci_l1c_nc.py @@ -123,8 +123,8 @@ def _get_test_image_data_for_channel(data, ch_str, n_rows_cols): def _get_test_chunk_position_for_channel(data, ch_str, n_rows_cols): pos = "data/{:s}/measured/{:s}_position_{:s}" - data[pos.format(ch_str, "start", "row")] = xr.DataArray(0) - data[pos.format(ch_str, "start", "column")] = xr.DataArray(0) + data[pos.format(ch_str, "start", "row")] = xr.DataArray(1) + data[pos.format(ch_str, "start", "column")] = xr.DataArray(1) data[pos.format(ch_str, "end", "row")] = xr.DataArray(n_rows_cols[0]) data[pos.format(ch_str, "end", "column")] = xr.DataArray(n_rows_cols[1]) @@ -212,7 +212,6 @@ def _get_global_attributes(): def _get_test_content_for_channel(ch_str, grid_type): - nrows = GRID_TYPE_INFO_FOR_TEST_CONTENT[grid_type]['nrows'] ncols = GRID_TYPE_INFO_FOR_TEST_CONTENT[grid_type]['ncols'] n_rows_cols = (nrows, ncols) @@ -530,6 +529,37 @@ def test_orbital_parameters_attr(self, reader_configs, fh_param): 'projection_altitude': 35786400.0, } + expected_pos_info_for_filetype = { + 'fdhsi': {'1km': {'start_position_row': 1, + 'end_position_row': 200, + 'segment_height': 200, + 'grid_width': 11136}, + '2km': {'start_position_row': 1, + 'end_position_row': 100, + 'segment_height': 100, + 'grid_width': 5568}}, + 'hrfi': {'500m': {'start_position_row': 1, + 'end_position_row': 400, + 'segment_height': 400, + 'grid_width': 22272}, + '1km': {'start_position_row': 1, + 'end_position_row': 200, + 'grid_width': 11136, + 'segment_height': 200}} + } + + @pytest.mark.parametrize('fh_param, expected_pos_info', [ + (fh_param_for_filetype['fdhsi'], expected_pos_info_for_filetype['fdhsi']), + (fh_param_for_filetype['hrfi'], expected_pos_info_for_filetype['hrfi']) + ]) + def test_get_segment_position_info(self, reader_configs, fh_param, expected_pos_info): + """Test the segment position info method.""" + with mocked_basefilehandler(fh_param['filehandler']): + reader = _get_reader_with_filehandlers(fh_param['filenames'], reader_configs) + for filetype_handler in list(reader.file_handlers.values())[0]: + segpos_info = filetype_handler.get_segment_position_info() + assert segpos_info == expected_pos_info + @pytest.mark.parametrize('fh_param,expected_res_n', [ (fh_param_for_filetype['fdhsi'], 16), (fh_param_for_filetype['hrfi'], 4) From bc55a44a039c044c5ac098a8ea8cbda3bb704ce9 Mon Sep 17 00:00:00 2001 From: andream Date: Mon, 28 Nov 2022 17:33:52 +0100 Subject: [PATCH 22/26] refactor fci tests to make use of lazy_fixtures --- satpy/tests/reader_tests/test_fci_l1c_nc.py | 400 ++++++++++---------- 1 file changed, 205 insertions(+), 195 deletions(-) diff --git a/satpy/tests/reader_tests/test_fci_l1c_nc.py b/satpy/tests/reader_tests/test_fci_l1c_nc.py index d005e719f8..5401063618 100644 --- a/satpy/tests/reader_tests/test_fci_l1c_nc.py +++ b/satpy/tests/reader_tests/test_fci_l1c_nc.py @@ -28,6 +28,7 @@ import pytest import xarray as xr from netCDF4 import default_fillvals +from pytest_lazyfixture import lazy_fixture from satpy.readers.fci_l1c_nc import FCIL1cNCFileHandler from satpy.tests.reader_tests.test_netcdf_utils import FakeNetCDF4FileHandler @@ -365,20 +366,41 @@ def mocked_basefilehandler(filehandler): yield +@pytest.fixture +def FakeFCIFileHandlerHRFI_fixture(): + """Get a fixture for the fake HRFI filehandler, including channel and file names.""" + with mocked_basefilehandler(FakeFCIFileHandlerHRFI): + param_dict = { + 'channels': _chans_hrfi, + 'filenames': _test_filenames['hrfi'] + } + yield param_dict + + +@pytest.fixture +def FakeFCIFileHandlerFDHSI_fixture(): + """Get a fixture for the fake FDHSI filehandler, including channel and file names.""" + with mocked_basefilehandler(FakeFCIFileHandlerFDHSI): + param_dict = { + 'channels': _chans_fdhsi, + 'filenames': _test_filenames['fdhsi'] + } + yield param_dict + + class TestFCIL1cNCReader: """Test FCI L1c NetCDF reader with nominal data.""" fh_param_for_filetype = { + 'hrfi': { + 'channels': _chans_hrfi, + 'filenames': _test_filenames['hrfi'] + }, 'fdhsi': { - 'filehandler': FakeFCIFileHandlerFDHSI, 'channels': _chans_fdhsi, 'filenames': _test_filenames['fdhsi'] }, - 'hrfi': { - 'filehandler': FakeFCIFileHandlerHRFI, - 'channels': _chans_hrfi, - 'filenames': _test_filenames['hrfi'] - } + } @pytest.mark.parametrize('filenames', [_test_filenames['fdhsi'], _test_filenames['hrfi']]) @@ -401,133 +423,128 @@ def test_file_pattern_for_TRAIL_file(self, reader_configs, filenames): assert len(files) == 0 @pytest.mark.parametrize('fh_param,expected_res_n', [ - (fh_param_for_filetype['fdhsi'], 16), - (fh_param_for_filetype['hrfi'], 4) + (lazy_fixture('FakeFCIFileHandlerFDHSI_fixture'), 16), + (lazy_fixture('FakeFCIFileHandlerHRFI_fixture'), 4) ]) def test_load_counts(self, reader_configs, fh_param, expected_res_n): """Test loading with counts.""" - with mocked_basefilehandler(fh_param['filehandler']): - reader = _get_reader_with_filehandlers(fh_param['filenames'], reader_configs) - res = reader.load( - [make_dataid(name=name, calibration="counts") for name in - fh_param['channels']["solar"] + fh_param['channels']["terran"]], pad_data=False) - assert expected_res_n == len(res) - for ch, grid_type in zip(fh_param['channels']["solar"] + fh_param['channels']["terran"], - fh_param['channels']["solar_grid_type"] + - fh_param['channels']["terran_grid_type"]): - assert res[ch].shape == (GRID_TYPE_INFO_FOR_TEST_CONTENT[grid_type]['nrows'], - GRID_TYPE_INFO_FOR_TEST_CONTENT[grid_type]['ncols']) - assert res[ch].dtype == np.uint16 - assert res[ch].attrs["calibration"] == "counts" - assert res[ch].attrs["units"] == "count" - if ch == 'ir_38': - numpy.testing.assert_array_equal(res[ch][-1], 1) - numpy.testing.assert_array_equal(res[ch][0], 5000) - else: - numpy.testing.assert_array_equal(res[ch], 1) + reader = _get_reader_with_filehandlers(fh_param['filenames'], reader_configs) + res = reader.load( + [make_dataid(name=name, calibration="counts") for name in + fh_param['channels']["solar"] + fh_param['channels']["terran"]], pad_data=False) + assert expected_res_n == len(res) + for ch, grid_type in zip(fh_param['channels']["solar"] + fh_param['channels']["terran"], + fh_param['channels']["solar_grid_type"] + + fh_param['channels']["terran_grid_type"]): + assert res[ch].shape == (GRID_TYPE_INFO_FOR_TEST_CONTENT[grid_type]['nrows'], + GRID_TYPE_INFO_FOR_TEST_CONTENT[grid_type]['ncols']) + assert res[ch].dtype == np.uint16 + assert res[ch].attrs["calibration"] == "counts" + assert res[ch].attrs["units"] == "count" + if ch == 'ir_38': + numpy.testing.assert_array_equal(res[ch][-1], 1) + numpy.testing.assert_array_equal(res[ch][0], 5000) + else: + numpy.testing.assert_array_equal(res[ch], 1) @pytest.mark.parametrize('fh_param,expected_res_n', [ - (fh_param_for_filetype['fdhsi'], 16), - (fh_param_for_filetype['hrfi'], 4) + (lazy_fixture('FakeFCIFileHandlerFDHSI_fixture'), 16), + (lazy_fixture('FakeFCIFileHandlerHRFI_fixture'), 4) ]) def test_load_radiance(self, reader_configs, fh_param, expected_res_n): """Test loading with radiance.""" - with mocked_basefilehandler(fh_param['filehandler']): - reader = _get_reader_with_filehandlers(fh_param['filenames'], reader_configs) - res = reader.load( - [make_dataid(name=name, calibration="radiance") for name in - fh_param['channels']["solar"] + fh_param['channels']["terran"]], pad_data=False) - assert expected_res_n == len(res) - for ch, grid_type in zip(fh_param['channels']["solar"] + fh_param['channels']["terran"], - fh_param['channels']["solar_grid_type"] + - fh_param['channels']["terran_grid_type"]): - assert res[ch].shape == (GRID_TYPE_INFO_FOR_TEST_CONTENT[grid_type]['nrows'], - GRID_TYPE_INFO_FOR_TEST_CONTENT[grid_type]['ncols']) - assert res[ch].dtype == np.float64 - assert res[ch].attrs["calibration"] == "radiance" - assert res[ch].attrs["units"] == 'mW m-2 sr-1 (cm-1)-1' - assert res[ch].attrs["radiance_unit_conversion_coefficient"] == 1234.56 - if ch == 'ir_38': - numpy.testing.assert_array_equal(res[ch][-1], 15) - numpy.testing.assert_array_equal(res[ch][0], 9700) - else: - numpy.testing.assert_array_equal(res[ch], 15) + reader = _get_reader_with_filehandlers(fh_param['filenames'], reader_configs) + res = reader.load( + [make_dataid(name=name, calibration="radiance") for name in + fh_param['channels']["solar"] + fh_param['channels']["terran"]], pad_data=False) + assert expected_res_n == len(res) + for ch, grid_type in zip(fh_param['channels']["solar"] + fh_param['channels']["terran"], + fh_param['channels']["solar_grid_type"] + + fh_param['channels']["terran_grid_type"]): + assert res[ch].shape == (GRID_TYPE_INFO_FOR_TEST_CONTENT[grid_type]['nrows'], + GRID_TYPE_INFO_FOR_TEST_CONTENT[grid_type]['ncols']) + assert res[ch].dtype == np.float64 + assert res[ch].attrs["calibration"] == "radiance" + assert res[ch].attrs["units"] == 'mW m-2 sr-1 (cm-1)-1' + assert res[ch].attrs["radiance_unit_conversion_coefficient"] == 1234.56 + if ch == 'ir_38': + numpy.testing.assert_array_equal(res[ch][-1], 15) + numpy.testing.assert_array_equal(res[ch][0], 9700) + else: + numpy.testing.assert_array_equal(res[ch], 15) @pytest.mark.parametrize('fh_param,expected_res_n', [ - (fh_param_for_filetype['fdhsi'], 8), - (fh_param_for_filetype['hrfi'], 2) + (lazy_fixture('FakeFCIFileHandlerFDHSI_fixture'), 8), + (lazy_fixture('FakeFCIFileHandlerHRFI_fixture'), 2) ]) def test_load_reflectance(self, reader_configs, fh_param, expected_res_n): """Test loading with reflectance.""" - with mocked_basefilehandler(fh_param['filehandler']): - reader = _get_reader_with_filehandlers(fh_param['filenames'], reader_configs) - res = reader.load( - [make_dataid(name=name, calibration="reflectance") for name in - fh_param['channels']["solar"]], pad_data=False) - assert expected_res_n == len(res) - for ch, grid_type in zip(fh_param['channels']["solar"], fh_param['channels']["solar_grid_type"]): - assert res[ch].shape == (GRID_TYPE_INFO_FOR_TEST_CONTENT[grid_type]['nrows'], - GRID_TYPE_INFO_FOR_TEST_CONTENT[grid_type]['ncols']) - assert res[ch].dtype == np.float64 - assert res[ch].attrs["calibration"] == "reflectance" - assert res[ch].attrs["units"] == "%" - numpy.testing.assert_array_almost_equal(res[ch], 100 * 15 * 1 * np.pi / 50) + reader = _get_reader_with_filehandlers(fh_param['filenames'], reader_configs) + res = reader.load( + [make_dataid(name=name, calibration="reflectance") for name in + fh_param['channels']["solar"]], pad_data=False) + assert expected_res_n == len(res) + for ch, grid_type in zip(fh_param['channels']["solar"], fh_param['channels']["solar_grid_type"]): + assert res[ch].shape == (GRID_TYPE_INFO_FOR_TEST_CONTENT[grid_type]['nrows'], + GRID_TYPE_INFO_FOR_TEST_CONTENT[grid_type]['ncols']) + assert res[ch].dtype == np.float64 + assert res[ch].attrs["calibration"] == "reflectance" + assert res[ch].attrs["units"] == "%" + numpy.testing.assert_array_almost_equal(res[ch], 100 * 15 * 1 * np.pi / 50) @pytest.mark.parametrize('fh_param,expected_res_n', [ - (fh_param_for_filetype['fdhsi'], 8), - (fh_param_for_filetype['hrfi'], 2) + (lazy_fixture('FakeFCIFileHandlerFDHSI_fixture'), 8), + (lazy_fixture('FakeFCIFileHandlerHRFI_fixture'), 2) ]) def test_load_bt(self, reader_configs, caplog, fh_param, expected_res_n): """Test loading with bt.""" - with mocked_basefilehandler(fh_param['filehandler']): - reader = _get_reader_with_filehandlers(fh_param['filenames'], reader_configs) - with caplog.at_level(logging.WARNING): - res = reader.load( - [make_dataid(name=name, calibration="brightness_temperature") for - name in fh_param['channels']["terran"]], pad_data=False) - assert caplog.text == "" - assert expected_res_n == len(res) - for ch, grid_type in zip(fh_param['channels']["terran"], fh_param['channels']["terran_grid_type"]): - assert res[ch].shape == (GRID_TYPE_INFO_FOR_TEST_CONTENT[grid_type]['nrows'], - GRID_TYPE_INFO_FOR_TEST_CONTENT[grid_type]['ncols']) - assert res[ch].dtype == np.float64 - assert res[ch].attrs["calibration"] == "brightness_temperature" - assert res[ch].attrs["units"] == "K" - - if ch == 'ir_38': - numpy.testing.assert_array_almost_equal(res[ch][-1], 209.68274099) - numpy.testing.assert_array_almost_equal(res[ch][0], 1888.851296) - else: - numpy.testing.assert_array_almost_equal(res[ch], 209.68274099) + reader = _get_reader_with_filehandlers(fh_param['filenames'], reader_configs) + with caplog.at_level(logging.WARNING): + res = reader.load( + [make_dataid(name=name, calibration="brightness_temperature") for + name in fh_param['channels']["terran"]], pad_data=False) + assert caplog.text == "" + assert expected_res_n == len(res) + for ch, grid_type in zip(fh_param['channels']["terran"], fh_param['channels']["terran_grid_type"]): + assert res[ch].shape == (GRID_TYPE_INFO_FOR_TEST_CONTENT[grid_type]['nrows'], + GRID_TYPE_INFO_FOR_TEST_CONTENT[grid_type]['ncols']) + assert res[ch].dtype == np.float64 + assert res[ch].attrs["calibration"] == "brightness_temperature" + assert res[ch].attrs["units"] == "K" + + if ch == 'ir_38': + numpy.testing.assert_array_almost_equal(res[ch][-1], 209.68274099) + numpy.testing.assert_array_almost_equal(res[ch][0], 1888.851296) + else: + numpy.testing.assert_array_almost_equal(res[ch], 209.68274099) @pytest.mark.parametrize('fh_param', [ - (fh_param_for_filetype['fdhsi']), - (fh_param_for_filetype['hrfi']) + (lazy_fixture('FakeFCIFileHandlerFDHSI_fixture')), + (lazy_fixture('FakeFCIFileHandlerHRFI_fixture')) ]) def test_orbital_parameters_attr(self, reader_configs, fh_param): """Test the orbital parameter attribute.""" - with mocked_basefilehandler(fh_param['filehandler']): - reader = _get_reader_with_filehandlers(fh_param['filenames'], reader_configs) - res = reader.load( - [make_dataid(name=name) for name in - fh_param['channels']["solar"] + fh_param['channels']["terran"]], pad_data=False) - - for ch in fh_param['channels']["solar"] + fh_param['channels']["terran"]: - assert res[ch].attrs["orbital_parameters"] == { - 'satellite_actual_longitude': np.mean(np.arange(6000)), - 'satellite_actual_latitude': np.mean(np.arange(6000)), - 'satellite_actual_altitude': np.mean(np.arange(6000)), - 'satellite_nominal_longitude': 0.0, - 'satellite_nominal_latitude': 0, - 'satellite_nominal_altitude': 35786400.0, - 'projection_longitude': 0.0, - 'projection_latitude': 0, - 'projection_altitude': 35786400.0, - } + reader = _get_reader_with_filehandlers(fh_param['filenames'], reader_configs) + res = reader.load( + [make_dataid(name=name) for name in + fh_param['channels']["solar"] + fh_param['channels']["terran"]], pad_data=False) + + for ch in fh_param['channels']["solar"] + fh_param['channels']["terran"]: + assert res[ch].attrs["orbital_parameters"] == { + 'satellite_actual_longitude': np.mean(np.arange(6000)), + 'satellite_actual_latitude': np.mean(np.arange(6000)), + 'satellite_actual_altitude': np.mean(np.arange(6000)), + 'satellite_nominal_longitude': 0.0, + 'satellite_nominal_latitude': 0, + 'satellite_nominal_altitude': 35786400.0, + 'projection_longitude': 0.0, + 'projection_latitude': 0, + 'projection_altitude': 35786400.0, + } expected_pos_info_for_filetype = { 'fdhsi': {'1km': {'start_position_row': 1, @@ -549,55 +566,52 @@ def test_orbital_parameters_attr(self, reader_configs, fh_param): } @pytest.mark.parametrize('fh_param, expected_pos_info', [ - (fh_param_for_filetype['fdhsi'], expected_pos_info_for_filetype['fdhsi']), - (fh_param_for_filetype['hrfi'], expected_pos_info_for_filetype['hrfi']) + (lazy_fixture('FakeFCIFileHandlerFDHSI_fixture'), expected_pos_info_for_filetype['fdhsi']), + (lazy_fixture('FakeFCIFileHandlerHRFI_fixture'), expected_pos_info_for_filetype['hrfi']) ]) def test_get_segment_position_info(self, reader_configs, fh_param, expected_pos_info): """Test the segment position info method.""" - with mocked_basefilehandler(fh_param['filehandler']): - reader = _get_reader_with_filehandlers(fh_param['filenames'], reader_configs) - for filetype_handler in list(reader.file_handlers.values())[0]: - segpos_info = filetype_handler.get_segment_position_info() - assert segpos_info == expected_pos_info + reader = _get_reader_with_filehandlers(fh_param['filenames'], reader_configs) + for filetype_handler in list(reader.file_handlers.values())[0]: + segpos_info = filetype_handler.get_segment_position_info() + assert segpos_info == expected_pos_info @pytest.mark.parametrize('fh_param,expected_res_n', [ - (fh_param_for_filetype['fdhsi'], 16), - (fh_param_for_filetype['hrfi'], 4) + (lazy_fixture('FakeFCIFileHandlerFDHSI_fixture'), 16), + (lazy_fixture('FakeFCIFileHandlerHRFI_fixture'), 4) ]) def test_load_index_map(self, reader_configs, fh_param, expected_res_n): """Test loading of index_map.""" - with mocked_basefilehandler(fh_param['filehandler']): - reader = _get_reader_with_filehandlers(fh_param['filenames'], reader_configs) - res = reader.load( - [name + '_index_map' for name in - fh_param['channels']["solar"] + fh_param['channels']["terran"]], pad_data=False) - assert expected_res_n == len(res) - for ch, grid_type in zip(fh_param['channels']["solar"] + fh_param['channels']["terran"], - fh_param['channels']["solar_grid_type"] + - fh_param['channels']["terran_grid_type"]): - assert res[ch + '_index_map'].shape == (GRID_TYPE_INFO_FOR_TEST_CONTENT[grid_type]['nrows'], - GRID_TYPE_INFO_FOR_TEST_CONTENT[grid_type]['ncols']) - numpy.testing.assert_array_equal(res[ch + '_index_map'][1, 1], 110) + reader = _get_reader_with_filehandlers(fh_param['filenames'], reader_configs) + res = reader.load( + [name + '_index_map' for name in + fh_param['channels']["solar"] + fh_param['channels']["terran"]], pad_data=False) + assert expected_res_n == len(res) + for ch, grid_type in zip(fh_param['channels']["solar"] + fh_param['channels']["terran"], + fh_param['channels']["solar_grid_type"] + + fh_param['channels']["terran_grid_type"]): + assert res[ch + '_index_map'].shape == (GRID_TYPE_INFO_FOR_TEST_CONTENT[grid_type]['nrows'], + GRID_TYPE_INFO_FOR_TEST_CONTENT[grid_type]['ncols']) + numpy.testing.assert_array_equal(res[ch + '_index_map'][1, 1], 110) @pytest.mark.parametrize('fh_param', [ - (fh_param_for_filetype['fdhsi']), - (fh_param_for_filetype['hrfi']) + (lazy_fixture('FakeFCIFileHandlerFDHSI_fixture')), + (lazy_fixture('FakeFCIFileHandlerHRFI_fixture')) ]) def test_load_aux_data(self, reader_configs, fh_param): """Test loading of auxiliary data.""" from satpy.readers.fci_l1c_nc import AUX_DATA - with mocked_basefilehandler(fh_param['filehandler']): - reader = _get_reader_with_filehandlers(fh_param['filenames'], reader_configs) - res = reader.load([fh_param['channels']['solar'][0] + '_' + key for key in AUX_DATA.keys()], - pad_data=False) - grid_type = fh_param['channels']['solar_grid_type'][0] - for aux in [fh_param['channels']['solar'][0] + '_' + key for key in AUX_DATA.keys()]: - assert res[aux].shape == (GRID_TYPE_INFO_FOR_TEST_CONTENT[grid_type]['nrows'], - GRID_TYPE_INFO_FOR_TEST_CONTENT[grid_type]['ncols']) - if aux == fh_param['channels']['solar'][0] + '_earth_sun_distance': - numpy.testing.assert_array_equal(res[aux][1, 1], 149597870.7) - else: - numpy.testing.assert_array_equal(res[aux][1, 1], 10) + reader = _get_reader_with_filehandlers(fh_param['filenames'], reader_configs) + res = reader.load([fh_param['channels']['solar'][0] + '_' + key for key in AUX_DATA.keys()], + pad_data=False) + grid_type = fh_param['channels']['solar_grid_type'][0] + for aux in [fh_param['channels']['solar'][0] + '_' + key for key in AUX_DATA.keys()]: + assert res[aux].shape == (GRID_TYPE_INFO_FOR_TEST_CONTENT[grid_type]['nrows'], + GRID_TYPE_INFO_FOR_TEST_CONTENT[grid_type]['ncols']) + if aux == fh_param['channels']['solar'][0] + '_earth_sun_distance': + numpy.testing.assert_array_equal(res[aux][1, 1], 149597870.7) + else: + numpy.testing.assert_array_equal(res[aux][1, 1], 10) def test_load_composite(self): """Test that composites are loadable.""" @@ -611,28 +625,27 @@ def test_load_composite(self): assert len(mods["fci"]) > 0 @pytest.mark.parametrize('fh_param,expected_res_n', [ - (fh_param_for_filetype['fdhsi'], 16), - (fh_param_for_filetype['hrfi'], 4) + (lazy_fixture('FakeFCIFileHandlerFDHSI_fixture'), 16), + (lazy_fixture('FakeFCIFileHandlerHRFI_fixture'), 4) ]) def test_load_quality_only(self, reader_configs, fh_param, expected_res_n): """Test that loading quality only works.""" - with mocked_basefilehandler(fh_param['filehandler']): - reader = _get_reader_with_filehandlers(fh_param['filenames'], reader_configs) - res = reader.load( - [name + '_pixel_quality' for name in - fh_param['channels']["solar"] + fh_param['channels']["terran"]], pad_data=False) - assert expected_res_n == len(res) - for ch, grid_type in zip(fh_param['channels']["solar"] + fh_param['channels']["terran"], - fh_param['channels']["solar_grid_type"] + - fh_param['channels']["terran_grid_type"]): - assert res[ch + '_pixel_quality'].shape == (GRID_TYPE_INFO_FOR_TEST_CONTENT[grid_type]['nrows'], - GRID_TYPE_INFO_FOR_TEST_CONTENT[grid_type]['ncols']) - numpy.testing.assert_array_equal(res[ch + '_pixel_quality'][1, 1], 3) - assert res[ch + '_pixel_quality'].attrs["name"] == ch + '_pixel_quality' + reader = _get_reader_with_filehandlers(fh_param['filenames'], reader_configs) + res = reader.load( + [name + '_pixel_quality' for name in + fh_param['channels']["solar"] + fh_param['channels']["terran"]], pad_data=False) + assert expected_res_n == len(res) + for ch, grid_type in zip(fh_param['channels']["solar"] + fh_param['channels']["terran"], + fh_param['channels']["solar_grid_type"] + + fh_param['channels']["terran_grid_type"]): + assert res[ch + '_pixel_quality'].shape == (GRID_TYPE_INFO_FOR_TEST_CONTENT[grid_type]['nrows'], + GRID_TYPE_INFO_FOR_TEST_CONTENT[grid_type]['ncols']) + numpy.testing.assert_array_equal(res[ch + '_pixel_quality'][1, 1], 3) + assert res[ch + '_pixel_quality'].attrs["name"] == ch + '_pixel_quality' @pytest.mark.parametrize('fh_param', [ - (fh_param_for_filetype['fdhsi']), - (fh_param_for_filetype['hrfi']) + (lazy_fixture('FakeFCIFileHandlerFDHSI_fixture')), + (lazy_fixture('FakeFCIFileHandlerHRFI_fixture')) ]) def test_platform_name(self, reader_configs, fh_param): """Test that platform name is exposed. @@ -640,51 +653,48 @@ def test_platform_name(self, reader_configs, fh_param): Test that the FCI reader exposes the platform name. Corresponds to GH issue 1014. """ - with mocked_basefilehandler(fh_param['filehandler']): - reader = _get_reader_with_filehandlers(fh_param['filenames'], reader_configs) - res = reader.load(["vis_06"], pad_data=False) - assert res["vis_06"].attrs["platform_name"] == "MTG-I1" + reader = _get_reader_with_filehandlers(fh_param['filenames'], reader_configs) + res = reader.load(["vis_06"], pad_data=False) + assert res["vis_06"].attrs["platform_name"] == "MTG-I1" - def test_excs(self, reader_configs): + def test_excs(self, reader_configs, FakeFCIFileHandlerFDHSI_fixture): """Test that exceptions are raised where expected.""" - with mocked_basefilehandler(FakeFCIFileHandlerFDHSI): - reader = _get_reader_with_filehandlers(_test_filenames['fdhsi'], reader_configs) + reader = _get_reader_with_filehandlers(_test_filenames['fdhsi'], reader_configs) - with pytest.raises(ValueError): - reader.file_handlers["fci_l1c_fdhsi"][0].get_dataset(make_dataid(name="invalid"), {}) - with pytest.raises(ValueError): - reader.file_handlers["fci_l1c_fdhsi"][0].get_dataset( - make_dataid(name="ir_123", calibration="unknown"), - {"units": "unknown"}) + with pytest.raises(ValueError): + reader.file_handlers["fci_l1c_fdhsi"][0].get_dataset(make_dataid(name="invalid"), {}) + with pytest.raises(ValueError): + reader.file_handlers["fci_l1c_fdhsi"][0].get_dataset( + make_dataid(name="ir_123", calibration="unknown"), + {"units": "unknown"}) @pytest.mark.parametrize('fh_param, expected_area', [ - (fh_param_for_filetype['fdhsi'], ['mtg_fci_fdss_1km', 'mtg_fci_fdss_2km']), - (fh_param_for_filetype['hrfi'], ['mtg_fci_fdss_500m', 'mtg_fci_fdss_1km']), + (lazy_fixture('FakeFCIFileHandlerFDHSI_fixture'), ['mtg_fci_fdss_1km', 'mtg_fci_fdss_2km']), + (lazy_fixture('FakeFCIFileHandlerHRFI_fixture'), ['mtg_fci_fdss_500m', 'mtg_fci_fdss_1km']), ]) def test_area_definition_computation(self, reader_configs, fh_param, expected_area): """Test that the geolocation computation is correct.""" - with mocked_basefilehandler(fh_param['filehandler']): - reader = _get_reader_with_filehandlers(fh_param['filenames'], reader_configs) - res = reader.load(['ir_105', 'vis_06'], pad_data=False) - - # test that area_ids are harmonisation-conform ___ - assert res['vis_06'].attrs['area'].area_id == expected_area[0] - assert res['ir_105'].attrs['area'].area_id == expected_area[1] - - area_def = res['ir_105'].attrs['area'] - # test area extents computation - np.testing.assert_array_almost_equal(np.array(area_def.area_extent), - np.array([-5567999.994203, -5367999.994411, - 5567999.994203, -5567999.994203]), - decimal=2) - - # check that the projection is read in properly - assert area_def.crs.coordinate_operation.method_name == 'Geostationary Satellite (Sweep Y)' - assert area_def.crs.coordinate_operation.params[0].value == 0.0 # projection origin longitude - assert area_def.crs.coordinate_operation.params[1].value == 35786400.0 # projection height - assert area_def.crs.ellipsoid.semi_major_metre == 6378137.0 - assert area_def.crs.ellipsoid.inverse_flattening == 298.257223563 - assert area_def.crs.ellipsoid.is_semi_minor_computed + reader = _get_reader_with_filehandlers(fh_param['filenames'], reader_configs) + res = reader.load(['ir_105', 'vis_06'], pad_data=False) + + # test that area_ids are harmonisation-conform ___ + assert res['vis_06'].attrs['area'].area_id == expected_area[0] + assert res['ir_105'].attrs['area'].area_id == expected_area[1] + + area_def = res['ir_105'].attrs['area'] + # test area extents computation + np.testing.assert_array_almost_equal(np.array(area_def.area_extent), + np.array([-5567999.994203, -5367999.994411, + 5567999.994203, -5567999.994203]), + decimal=2) + + # check that the projection is read in properly + assert area_def.crs.coordinate_operation.method_name == 'Geostationary Satellite (Sweep Y)' + assert area_def.crs.coordinate_operation.params[0].value == 0.0 # projection origin longitude + assert area_def.crs.coordinate_operation.params[1].value == 35786400.0 # projection height + assert area_def.crs.ellipsoid.semi_major_metre == 6378137.0 + assert area_def.crs.ellipsoid.inverse_flattening == 298.257223563 + assert area_def.crs.ellipsoid.is_semi_minor_computed class TestFCIL1cNCReaderBadData: From 20534baa1f9a0a6c8ca701bd7be8569cc346b3d0 Mon Sep 17 00:00:00 2001 From: andream Date: Tue, 29 Nov 2022 18:47:42 +0100 Subject: [PATCH 23/26] add double fdhsi and hrfi check to excs test, cosmetic changes --- satpy/readers/yaml_reader.py | 9 +- satpy/tests/reader_tests/test_fci_l1c_nc.py | 148 +++++++++----------- 2 files changed, 74 insertions(+), 83 deletions(-) diff --git a/satpy/readers/yaml_reader.py b/satpy/readers/yaml_reader.py index 658371fd42..0fb46379f9 100644 --- a/satpy/readers/yaml_reader.py +++ b/satpy/readers/yaml_reader.py @@ -1390,17 +1390,16 @@ def _extract_segment_location_dicts(self, filetype): def _collect_segment_position_infos(self, filetype): # collect the segment positioning infos for all available segments - filetype_fhs = self.file_handlers[filetype] - for fh in filetype_fhs: + for fh in self.file_handlers[filetype]: chk_infos = fh.get_segment_position_info() chk_infos.update({'segment_nr': fh.filename_info['segment'] - 1}) self.segment_infos[filetype]['available_segment_infos'].append(chk_infos) def _initialise_segment_infos(self, filetype): # initialise the segment info for this filetype - filetype_fhs = self.file_handlers[filetype] - exp_segment_nr = filetype_fhs[0].filetype_info['expected_segments'] - grid_width_to_grid_type = _get_grid_width_to_grid_type(filetype_fhs[0].get_segment_position_info()) + filetype_fhs_sample = self.file_handlers[filetype][0] + exp_segment_nr = filetype_fhs_sample.filetype_info['expected_segments'] + grid_width_to_grid_type = _get_grid_width_to_grid_type(filetype_fhs_sample.get_segment_position_info()) self.segment_infos.update({filetype: {'available_segment_infos': [], 'expected_segments': exp_segment_nr, 'grid_width_to_grid_type': grid_width_to_grid_type}}) diff --git a/satpy/tests/reader_tests/test_fci_l1c_nc.py b/satpy/tests/reader_tests/test_fci_l1c_nc.py index 5401063618..aecde4b37f 100644 --- a/satpy/tests/reader_tests/test_fci_l1c_nc.py +++ b/satpy/tests/reader_tests/test_fci_l1c_nc.py @@ -56,6 +56,10 @@ } +# ---------------------------------------------------- +# Filehandlers preparation --------------------------- +# ---------------------------------------------------- + def _get_test_calib_for_channel_ir(data, meas_path): from pyspectral.blackbody import C_SPEED as c from pyspectral.blackbody import H_PLANCK as h @@ -316,6 +320,10 @@ class FakeFCIFileHandlerHRFI(FakeFCIFileHandlerBase): } +# ---------------------------------------------------- +# Fixtures preparation ------------------------------- +# ---------------------------------------------------- + @pytest.fixture def reader_configs(): """Return reader configs for FCI.""" @@ -366,42 +374,42 @@ def mocked_basefilehandler(filehandler): yield +@pytest.fixture +def FakeFCIFileHandlerFDHSI_fixture(): + """Get a fixture for the fake FDHSI filehandler, including channel and file names.""" + with mocked_basefilehandler(FakeFCIFileHandlerFDHSI): + param_dict = { + 'filetype': 'fci_l1c_fdhsi', + 'channels': _chans_fdhsi, + 'filenames': _test_filenames['fdhsi'] + } + yield param_dict + + @pytest.fixture def FakeFCIFileHandlerHRFI_fixture(): """Get a fixture for the fake HRFI filehandler, including channel and file names.""" with mocked_basefilehandler(FakeFCIFileHandlerHRFI): param_dict = { + 'filetype': 'fci_l1c_hrfi', 'channels': _chans_hrfi, 'filenames': _test_filenames['hrfi'] } yield param_dict -@pytest.fixture -def FakeFCIFileHandlerFDHSI_fixture(): - """Get a fixture for the fake FDHSI filehandler, including channel and file names.""" - with mocked_basefilehandler(FakeFCIFileHandlerFDHSI): - param_dict = { - 'channels': _chans_fdhsi, - 'filenames': _test_filenames['fdhsi'] - } - yield param_dict +# ---------------------------------------------------- +# Tests ---------------------------------------------- +# ---------------------------------------------------- class TestFCIL1cNCReader: """Test FCI L1c NetCDF reader with nominal data.""" - fh_param_for_filetype = { - 'hrfi': { - 'channels': _chans_hrfi, - 'filenames': _test_filenames['hrfi'] - }, - 'fdhsi': { - 'channels': _chans_fdhsi, - 'filenames': _test_filenames['fdhsi'] - }, - - } + fh_param_for_filetype = {'hrfi': {'channels': _chans_hrfi, + 'filenames': _test_filenames['hrfi']}, + 'fdhsi': {'channels': _chans_fdhsi, + 'filenames': _test_filenames['fdhsi']}} @pytest.mark.parametrize('filenames', [_test_filenames['fdhsi'], _test_filenames['hrfi']]) def test_file_pattern(self, reader_configs, filenames): @@ -422,10 +430,8 @@ def test_file_pattern_for_TRAIL_file(self, reader_configs, filenames): files = reader.select_files_from_pathnames(filenames) assert len(files) == 0 - @pytest.mark.parametrize('fh_param,expected_res_n', [ - (lazy_fixture('FakeFCIFileHandlerFDHSI_fixture'), 16), - (lazy_fixture('FakeFCIFileHandlerHRFI_fixture'), 4) - ]) + @pytest.mark.parametrize('fh_param,expected_res_n', [(lazy_fixture('FakeFCIFileHandlerFDHSI_fixture'), 16), + (lazy_fixture('FakeFCIFileHandlerHRFI_fixture'), 4)]) def test_load_counts(self, reader_configs, fh_param, expected_res_n): """Test loading with counts.""" @@ -448,10 +454,8 @@ def test_load_counts(self, reader_configs, fh_param, else: numpy.testing.assert_array_equal(res[ch], 1) - @pytest.mark.parametrize('fh_param,expected_res_n', [ - (lazy_fixture('FakeFCIFileHandlerFDHSI_fixture'), 16), - (lazy_fixture('FakeFCIFileHandlerHRFI_fixture'), 4) - ]) + @pytest.mark.parametrize('fh_param,expected_res_n', [(lazy_fixture('FakeFCIFileHandlerFDHSI_fixture'), 16), + (lazy_fixture('FakeFCIFileHandlerHRFI_fixture'), 4)]) def test_load_radiance(self, reader_configs, fh_param, expected_res_n): """Test loading with radiance.""" @@ -475,10 +479,8 @@ def test_load_radiance(self, reader_configs, fh_param, else: numpy.testing.assert_array_equal(res[ch], 15) - @pytest.mark.parametrize('fh_param,expected_res_n', [ - (lazy_fixture('FakeFCIFileHandlerFDHSI_fixture'), 8), - (lazy_fixture('FakeFCIFileHandlerHRFI_fixture'), 2) - ]) + @pytest.mark.parametrize('fh_param,expected_res_n', [(lazy_fixture('FakeFCIFileHandlerFDHSI_fixture'), 8), + (lazy_fixture('FakeFCIFileHandlerHRFI_fixture'), 2)]) def test_load_reflectance(self, reader_configs, fh_param, expected_res_n): """Test loading with reflectance.""" @@ -495,10 +497,8 @@ def test_load_reflectance(self, reader_configs, fh_param, assert res[ch].attrs["units"] == "%" numpy.testing.assert_array_almost_equal(res[ch], 100 * 15 * 1 * np.pi / 50) - @pytest.mark.parametrize('fh_param,expected_res_n', [ - (lazy_fixture('FakeFCIFileHandlerFDHSI_fixture'), 8), - (lazy_fixture('FakeFCIFileHandlerHRFI_fixture'), 2) - ]) + @pytest.mark.parametrize('fh_param,expected_res_n', [(lazy_fixture('FakeFCIFileHandlerFDHSI_fixture'), 8), + (lazy_fixture('FakeFCIFileHandlerHRFI_fixture'), 2)]) def test_load_bt(self, reader_configs, caplog, fh_param, expected_res_n): """Test loading with bt.""" @@ -522,10 +522,8 @@ def test_load_bt(self, reader_configs, caplog, fh_param, else: numpy.testing.assert_array_almost_equal(res[ch], 209.68274099) - @pytest.mark.parametrize('fh_param', [ - (lazy_fixture('FakeFCIFileHandlerFDHSI_fixture')), - (lazy_fixture('FakeFCIFileHandlerHRFI_fixture')) - ]) + @pytest.mark.parametrize('fh_param', [(lazy_fixture('FakeFCIFileHandlerFDHSI_fixture')), + (lazy_fixture('FakeFCIFileHandlerHRFI_fixture'))]) def test_orbital_parameters_attr(self, reader_configs, fh_param): """Test the orbital parameter attribute.""" reader = _get_reader_with_filehandlers(fh_param['filenames'], reader_configs) @@ -576,10 +574,8 @@ def test_get_segment_position_info(self, reader_configs, fh_param, expected_pos_ segpos_info = filetype_handler.get_segment_position_info() assert segpos_info == expected_pos_info - @pytest.mark.parametrize('fh_param,expected_res_n', [ - (lazy_fixture('FakeFCIFileHandlerFDHSI_fixture'), 16), - (lazy_fixture('FakeFCIFileHandlerHRFI_fixture'), 4) - ]) + @pytest.mark.parametrize('fh_param,expected_res_n', [(lazy_fixture('FakeFCIFileHandlerFDHSI_fixture'), 16), + (lazy_fixture('FakeFCIFileHandlerHRFI_fixture'), 4)]) def test_load_index_map(self, reader_configs, fh_param, expected_res_n): """Test loading of index_map.""" reader = _get_reader_with_filehandlers(fh_param['filenames'], reader_configs) @@ -594,10 +590,8 @@ def test_load_index_map(self, reader_configs, fh_param, expected_res_n): GRID_TYPE_INFO_FOR_TEST_CONTENT[grid_type]['ncols']) numpy.testing.assert_array_equal(res[ch + '_index_map'][1, 1], 110) - @pytest.mark.parametrize('fh_param', [ - (lazy_fixture('FakeFCIFileHandlerFDHSI_fixture')), - (lazy_fixture('FakeFCIFileHandlerHRFI_fixture')) - ]) + @pytest.mark.parametrize('fh_param', [(lazy_fixture('FakeFCIFileHandlerFDHSI_fixture')), + (lazy_fixture('FakeFCIFileHandlerHRFI_fixture'))]) def test_load_aux_data(self, reader_configs, fh_param): """Test loading of auxiliary data.""" from satpy.readers.fci_l1c_nc import AUX_DATA @@ -613,21 +607,8 @@ def test_load_aux_data(self, reader_configs, fh_param): else: numpy.testing.assert_array_equal(res[aux][1, 1], 10) - def test_load_composite(self): - """Test that composites are loadable.""" - # when dedicated composites for FCI are implemented in satpy, - # this method should probably move to a dedicated class and module - # in the tests.compositor_tests package - - from satpy.composites.config_loader import load_compositor_configs_for_sensors - comps, mods = load_compositor_configs_for_sensors(['fci']) - assert len(comps["fci"]) > 0 - assert len(mods["fci"]) > 0 - - @pytest.mark.parametrize('fh_param,expected_res_n', [ - (lazy_fixture('FakeFCIFileHandlerFDHSI_fixture'), 16), - (lazy_fixture('FakeFCIFileHandlerHRFI_fixture'), 4) - ]) + @pytest.mark.parametrize('fh_param,expected_res_n', [(lazy_fixture('FakeFCIFileHandlerFDHSI_fixture'), 16), + (lazy_fixture('FakeFCIFileHandlerHRFI_fixture'), 4)]) def test_load_quality_only(self, reader_configs, fh_param, expected_res_n): """Test that loading quality only works.""" reader = _get_reader_with_filehandlers(fh_param['filenames'], reader_configs) @@ -643,10 +624,8 @@ def test_load_quality_only(self, reader_configs, fh_param, expected_res_n): numpy.testing.assert_array_equal(res[ch + '_pixel_quality'][1, 1], 3) assert res[ch + '_pixel_quality'].attrs["name"] == ch + '_pixel_quality' - @pytest.mark.parametrize('fh_param', [ - (lazy_fixture('FakeFCIFileHandlerFDHSI_fixture')), - (lazy_fixture('FakeFCIFileHandlerHRFI_fixture')) - ]) + @pytest.mark.parametrize('fh_param', [(lazy_fixture('FakeFCIFileHandlerFDHSI_fixture')), + (lazy_fixture('FakeFCIFileHandlerHRFI_fixture'))]) def test_platform_name(self, reader_configs, fh_param): """Test that platform name is exposed. @@ -657,17 +636,6 @@ def test_platform_name(self, reader_configs, fh_param): res = reader.load(["vis_06"], pad_data=False) assert res["vis_06"].attrs["platform_name"] == "MTG-I1" - def test_excs(self, reader_configs, FakeFCIFileHandlerFDHSI_fixture): - """Test that exceptions are raised where expected.""" - reader = _get_reader_with_filehandlers(_test_filenames['fdhsi'], reader_configs) - - with pytest.raises(ValueError): - reader.file_handlers["fci_l1c_fdhsi"][0].get_dataset(make_dataid(name="invalid"), {}) - with pytest.raises(ValueError): - reader.file_handlers["fci_l1c_fdhsi"][0].get_dataset( - make_dataid(name="ir_123", calibration="unknown"), - {"units": "unknown"}) - @pytest.mark.parametrize('fh_param, expected_area', [ (lazy_fixture('FakeFCIFileHandlerFDHSI_fixture'), ['mtg_fci_fdss_1km', 'mtg_fci_fdss_2km']), (lazy_fixture('FakeFCIFileHandlerHRFI_fixture'), ['mtg_fci_fdss_500m', 'mtg_fci_fdss_1km']), @@ -696,6 +664,30 @@ def test_area_definition_computation(self, reader_configs, fh_param, expected_ar assert area_def.crs.ellipsoid.inverse_flattening == 298.257223563 assert area_def.crs.ellipsoid.is_semi_minor_computed + @pytest.mark.parametrize('fh_param', [(lazy_fixture('FakeFCIFileHandlerFDHSI_fixture')), + (lazy_fixture('FakeFCIFileHandlerHRFI_fixture'))]) + def test_excs(self, reader_configs, fh_param): + """Test that exceptions are raised where expected.""" + reader = _get_reader_with_filehandlers(fh_param['filenames'], reader_configs) + + with pytest.raises(ValueError): + reader.file_handlers[fh_param['filetype']][0].get_dataset(make_dataid(name="invalid"), {}) + with pytest.raises(ValueError): + reader.file_handlers[fh_param['filetype']][0].get_dataset( + make_dataid(name="ir_123", calibration="unknown"), + {"units": "unknown"}) + + def test_load_composite(self): + """Test that composites are loadable.""" + # when dedicated composites for FCI are implemented in satpy, + # this method should probably move to a dedicated class and module + # in the tests.compositor_tests package + + from satpy.composites.config_loader import load_compositor_configs_for_sensors + comps, mods = load_compositor_configs_for_sensors(['fci']) + assert len(comps["fci"]) > 0 + assert len(mods["fci"]) > 0 + class TestFCIL1cNCReaderBadData: """Test the FCI L1c NetCDF Reader for bad data input.""" From 32e69aa0caa073ced6cb25f6698958346b2e9fc8 Mon Sep 17 00:00:00 2001 From: andream Date: Thu, 1 Dec 2022 14:43:31 +0100 Subject: [PATCH 24/26] update docstring --- satpy/readers/fci_l1c_nc.py | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/satpy/readers/fci_l1c_nc.py b/satpy/readers/fci_l1c_nc.py index df76f5150d..6e3f7cf9a7 100644 --- a/satpy/readers/fci_l1c_nc.py +++ b/satpy/readers/fci_l1c_nc.py @@ -20,16 +20,23 @@ This module defines the :class:`FCIL1cNCFileHandler` file handler, to be used for reading Meteosat Third Generation (MTG) Flexible Combined Imager (FCI) Level-1c data. FCI will fly -on the MTG Imager (MTG-I) series of satellites, scheduled to be launched -in 2022 by the earliest. For more information about FCI, see `EUMETSAT`_. +on the MTG Imager (MTG-I) series of satellites, with the first satellite (MTG-I1) +scheduled to be launched on the 13th of December 2022. +For more information about FCI, see `EUMETSAT`_. -For simulated test data to be used with this reader, see `test data release`_. +For simulated test data to be used with this reader, see `test data releases`_. For the Product User Guide (PUG) of the FCI L1c data, see `PUG`_. .. note:: This reader currently supports Full Disk High Spectral Resolution Imagery - (FDHSI) and High Spatial Resolution Fast Imagery (HRFI) data. RSS data is - not supported yet. + (FDHSI) and High Spatial Resolution Fast Imagery (HRFI) data in full-disc ("FD") scanning mode. + If the user provides a list of both FDHSI and HRFI files from the same repeat cycle to the Satpy ``Scene``, + Satpy will automatically read the channels from the source with the finest resolution, + i.e. from the HRFI files for the vis_06, nir_22, ir_38, and ir_105 channels. + If needed, the desired resolution can be explicitly requested using e.g.: + ``scn.load(['vis_06'], resolution=1000)``. + + Note that RSS data is not supported yet. Geolocation is based on information from the data files. It uses: @@ -99,7 +106,7 @@ .. _PUG: https://www-cdn.eumetsat.int/files/2020-07/pdf_mtg_fci_l1_pug.pdf .. _EUMETSAT: https://www.eumetsat.int/mtg-flexible-combined-imager # noqa: E501 -.. _test data release: https://www.eumetsat.int/simulated-mtg-fci-l1c-enhanced-non-nominal-datasets +.. _test data releases: https://www.eumetsat.int/mtg-test-data """ from __future__ import absolute_import, division, print_function, unicode_literals From ebb999fa590f6613a84a400607b61be21e8c35e3 Mon Sep 17 00:00:00 2001 From: andream Date: Thu, 1 Dec 2022 17:37:00 +0100 Subject: [PATCH 25/26] add required_netcdf_variables for hrfi case after merging --- satpy/etc/readers/fci_l1c_nc.yaml | 50 +++++++++++++++++++-- satpy/tests/reader_tests/test_fci_l1c_nc.py | 2 +- 2 files changed, 47 insertions(+), 5 deletions(-) diff --git a/satpy/etc/readers/fci_l1c_nc.yaml b/satpy/etc/readers/fci_l1c_nc.yaml index 4076879bbf..483f675cd9 100644 --- a/satpy/etc/readers/fci_l1c_nc.yaml +++ b/satpy/etc/readers/fci_l1c_nc.yaml @@ -18,10 +18,6 @@ file_types: file_reader: !!python/name:satpy.readers.fci_l1c_nc.FCIL1cNCFileHandler file_patterns: [ '{pflag}_{location_indicator},{data_designator},MTI{spacecraft_id:1d}+{data_source}-1C-RRAD-FDHSI-FD-{subsetting}-{component1}-BODY-{component3}-{purpose}-{format}_{oflag}_{originator}_{processing_time:%Y%m%d%H%M%S}_{facility_or_tool}_{environment}_{start_time:%Y%m%d%H%M%S}_{end_time:%Y%m%d%H%M%S}_{processing_mode}_{special_compression}_{disposition_mode}_{repeat_cycle_in_day:>04d}_{count_in_repeat_cycle:>04d}.nc' ] expected_segments: 40 - fci_l1c_hrfi: - file_reader: !!python/name:satpy.readers.fci_l1c_nc.FCIL1cNCFileHandler - file_patterns: [ '{pflag}_{location_indicator},{data_designator},MTI{spacecraft_id:1d}+{data_source}-1C-RRAD-HRFI-FD-{subsetting}-{component1}-BODY-{component3}-{purpose}-{format}_{oflag}_{originator}_{processing_time:%Y%m%d%H%M%S}_{facility_or_tool}_{environment}_{start_time:%Y%m%d%H%M%S}_{end_time:%Y%m%d%H%M%S}_{processing_mode}_{special_compression}_{disposition_mode}_{repeat_cycle_in_day:>04d}_{count_in_repeat_cycle:>04d}.nc' ] - expected_segments: 40 required_netcdf_variables: - attr/platform - data/mtg_geos_projection @@ -76,6 +72,52 @@ file_types: - ir_105 - ir_123 - ir_133 + fci_l1c_hrfi: + file_reader: !!python/name:satpy.readers.fci_l1c_nc.FCIL1cNCFileHandler + file_patterns: [ '{pflag}_{location_indicator},{data_designator},MTI{spacecraft_id:1d}+{data_source}-1C-RRAD-HRFI-FD-{subsetting}-{component1}-BODY-{component3}-{purpose}-{format}_{oflag}_{originator}_{processing_time:%Y%m%d%H%M%S}_{facility_or_tool}_{environment}_{start_time:%Y%m%d%H%M%S}_{end_time:%Y%m%d%H%M%S}_{processing_mode}_{special_compression}_{disposition_mode}_{repeat_cycle_in_day:>04d}_{count_in_repeat_cycle:>04d}.nc' ] + expected_segments: 40 + required_netcdf_variables: + - attr/platform + - data/mtg_geos_projection + - data/{channel_name}/measured/start_position_row + - data/{channel_name}/measured/end_position_row + - data/{channel_name}/measured/radiance_to_bt_conversion_coefficient_wavenumber + - data/{channel_name}/measured/radiance_to_bt_conversion_coefficient_a + - data/{channel_name}/measured/radiance_to_bt_conversion_coefficient_b + - data/{channel_name}/measured/radiance_to_bt_conversion_constant_c1 + - data/{channel_name}/measured/radiance_to_bt_conversion_constant_c2 + - data/{channel_name}/measured/radiance_unit_conversion_coefficient + - data/{channel_name}/measured/channel_effective_solar_irradiance + - data/{channel_name}/measured/effective_radiance + - data/{channel_name}/measured/x + - data/{channel_name}/measured/y + - data/{channel_name}/measured/pixel_quality + - data/{channel_name}/measured/index_map + - data/mtg_geos_projection/attr/inverse_flattening + - data/mtg_geos_projection/attr/longitude_of_projection_origin + - data/mtg_geos_projection/attr/longitude_of_projection_origin + - data/mtg_geos_projection/attr/perspective_point_height + - data/mtg_geos_projection/attr/perspective_point_height + - data/mtg_geos_projection/attr/perspective_point_height + - data/mtg_geos_projection/attr/semi_major_axis + - data/swath_direction + - data/swath_number + - index + - state/celestial/earth_sun_distance + - state/celestial/earth_sun_distance + - state/celestial/subsolar_latitude + - state/celestial/subsolar_longitude + - state/celestial/sun_satellite_distance + - state/platform/platform_altitude + - state/platform/subsatellite_latitude + - state/platform/subsatellite_longitude + - time + variable_name_replacements: + channel_name: + - vis_06_hr + - nir_22_hr + - ir_38_hr + - ir_105_hr datasets: vis_04: diff --git a/satpy/tests/reader_tests/test_fci_l1c_nc.py b/satpy/tests/reader_tests/test_fci_l1c_nc.py index c4a373ca1c..ae883b0430 100644 --- a/satpy/tests/reader_tests/test_fci_l1c_nc.py +++ b/satpy/tests/reader_tests/test_fci_l1c_nc.py @@ -212,7 +212,7 @@ def _get_global_attributes(): data = {} attrs = {"platform": "MTI1"} for (k, v) in attrs.items(): - data["/attr/" + k] = v + data["attr/" + k] = v return data From d73bdde89d5ba21a5ef9ceaf75aa97650e47783c Mon Sep 17 00:00:00 2001 From: andream Date: Thu, 1 Dec 2022 17:54:12 +0100 Subject: [PATCH 26/26] remove all usages of the word "chunk" in favour of "segment", and add a small explanation in the get_segment_position_info docstring --- satpy/readers/fci_l1c_nc.py | 13 +++--- satpy/readers/yaml_reader.py | 12 +++--- satpy/tests/reader_tests/test_fci_l1c_nc.py | 4 +- satpy/tests/test_yaml_reader.py | 46 ++++++++++----------- 4 files changed, 39 insertions(+), 36 deletions(-) diff --git a/satpy/readers/fci_l1c_nc.py b/satpy/readers/fci_l1c_nc.py index 0e541cd430..bb9b6dda10 100644 --- a/satpy/readers/fci_l1c_nc.py +++ b/satpy/readers/fci_l1c_nc.py @@ -229,12 +229,15 @@ def get_channel_measured_group_path(self, channel): return measured_group_path def get_segment_position_info(self): - """Get information about the size and the position of the chunk inside the final image array. + """Get information about the size and the position of the segment inside the final image array. - As the final array is composed by stacking chunks (aka segments) vertically, the position of a chunk - inside the array is defined by the numbers of the start (lowest) and end (highest) row of the chunk. + As the final array is composed by stacking segments vertically, the position of a segment + inside the array is defined by the numbers of the start (lowest) and end (highest) row of the segment. The row numbering is assumed to start with 1. - This info is used in the GEOVariableSegmentYAMLReader to compute optimal chunk sizes for missing chunks. + This info is used in the GEOVariableSegmentYAMLReader to compute optimal segment sizes for missing segments. + + Note: in the FCI terminology, a segment is actually called "chunk". To avoid confusion with the dask concept + of chunk, and to be consistent with SEVIRI, we opt to use the word segment. """ vis_06_measured_path = self.get_channel_measured_group_path('vis_06') ir_105_measured_path = self.get_channel_measured_group_path('ir_105') @@ -350,7 +353,7 @@ def _get_dataset_measurand(self, key, info=None): @cached_property def orbital_param(self): - """Compute the orbital parameters for the current chunk.""" + """Compute the orbital parameters for the current segment.""" actual_subsat_lon = float(np.nanmean(self._get_aux_data_lut_vector('subsatellite_longitude'))) actual_subsat_lat = float(np.nanmean(self._get_aux_data_lut_vector('subsatellite_latitude'))) actual_sat_alt = float(np.nanmean(self._get_aux_data_lut_vector('platform_altitude'))) diff --git a/satpy/readers/yaml_reader.py b/satpy/readers/yaml_reader.py index b64f2e1d36..dffa4c7189 100644 --- a/satpy/readers/yaml_reader.py +++ b/satpy/readers/yaml_reader.py @@ -1358,7 +1358,7 @@ def _get_empty_segment_with_height(empty_segment, new_height, dim): class GEOVariableSegmentYAMLReader(GEOSegmentYAMLReader): - """GEOVariableSegmentYAMLReader for handling chunked/segmented GEO products with segments of variable height. + """GEOVariableSegmentYAMLReader for handling segmented GEO products with segments of variable height. This YAMLReader overrides parts of the GEOSegmentYAMLReader to account for formats where the segments can have variable heights. It computes the sizes of the padded segments using the information available in the @@ -1367,7 +1367,7 @@ class GEOVariableSegmentYAMLReader(GEOSegmentYAMLReader): This implementation was motivated by the FCI L1c format, where the segments (called chunks in the FCI world) can have variable heights. It is however generic, so that any future reader can use it. The requirement for the reader is to have a method called `get_segment_position_info`, returning a dictionary containing - the positioning info for each chunk (see example in + the positioning info for each segment (see example in :func:`satpy.readers.fci_l1c_nc.FCIL1cNCFileHandler.get_segment_position_info`). For more information on please see the documentation of :func:`satpy.readers.yaml_reader.GEOSegmentYAMLReader`. @@ -1501,13 +1501,13 @@ def _init_positioning_arrays_for_variable_padding(chk_infos, grid_type, exp_segm segment_start_rows = np.zeros(exp_segment_nr) segment_end_rows = np.zeros(exp_segment_nr) - _populate_positioning_arrays_with_available_chunk_info(chk_infos, grid_type, segment_start_rows, segment_end_rows, - segment_heights) + _populate_positioning_arrays_with_available_segment_info(chk_infos, grid_type, segment_start_rows, segment_end_rows, + segment_heights) return segment_start_rows, segment_end_rows, segment_heights -def _populate_positioning_arrays_with_available_chunk_info(chk_infos, grid_type, segment_start_rows, segment_end_rows, - segment_heights): +def _populate_positioning_arrays_with_available_segment_info(chk_infos, grid_type, segment_start_rows, segment_end_rows, + segment_heights): for chk_info in chk_infos: current_fh_segment_nr = chk_info['segment_nr'] segment_heights[current_fh_segment_nr] = chk_info[grid_type]['segment_height'] diff --git a/satpy/tests/reader_tests/test_fci_l1c_nc.py b/satpy/tests/reader_tests/test_fci_l1c_nc.py index ae883b0430..3995a36ece 100644 --- a/satpy/tests/reader_tests/test_fci_l1c_nc.py +++ b/satpy/tests/reader_tests/test_fci_l1c_nc.py @@ -126,7 +126,7 @@ def _get_test_image_data_for_channel(data, ch_str, n_rows_cols): data[ch_path + '/shape'] = n_rows_cols -def _get_test_chunk_position_for_channel(data, ch_str, n_rows_cols): +def _get_test_segment_position_for_channel(data, ch_str, n_rows_cols): pos = "data/{:s}/measured/{:s}_position_{:s}" data[pos.format(ch_str, "start", "row")] = xr.DataArray(1) data[pos.format(ch_str, "start", "column")] = xr.DataArray(1) @@ -228,7 +228,7 @@ def _get_test_content_for_channel(ch_str, grid_type): _get_test_geolocation_for_channel(data, ch_str, grid_type, n_rows_cols) _get_test_pixel_quality_for_channel(data, ch_str, n_rows_cols) _get_test_index_map_for_channel(data, ch_str, n_rows_cols) - _get_test_chunk_position_for_channel(data, ch_str, n_rows_cols) + _get_test_segment_position_for_channel(data, ch_str, n_rows_cols) return data diff --git a/satpy/tests/test_yaml_reader.py b/satpy/tests/test_yaml_reader.py index 479d91ffb4..0d2e057f32 100644 --- a/satpy/tests/test_yaml_reader.py +++ b/satpy/tests/test_yaml_reader.py @@ -1039,7 +1039,7 @@ def test_get_expected_segments(self, cfh): es = created_fhs['ft1'][0].filetype_info['expected_segments'] self.assertEqual(es, 3) - # check correct FCI chunk number reading into segment + # check correct FCI segment (aka chunk in the FCI world) number reading into segment fake_fh.filename_info = {'count_in_repeat_cycle': 5} created_fhs = reader.create_filehandlers(['fake.nc']) es = created_fhs['ft1'][0].filename_info['segment'] @@ -1352,9 +1352,9 @@ def test_pad_earlier_segments_area(self, GVSYReader, fake_adef): res = GVSYReader._pad_earlier_segments_area([fh_2], dataid, area_defs) assert len(res) == 2 - # The later vertical chunk (nr. 2) size is 278, which is exactly double the size - # of the gap left by the missing first chunk (139, as the second chunk starts at line 140). - # Therefore, the new vertical area extent for the first chunk should be + # The later vertical segment (nr. 2) size is 278, which is exactly double the size + # of the gap left by the missing first segment (139, as the second segment starts at line 140). + # Therefore, the new vertical area extent for the first segment should be # half of the previous size (1000-500)/2=250. # The new area extent lower-left row is therefore 500-250=250 seg1_extent = (0, 500, 200, 250) @@ -1384,8 +1384,8 @@ def test_pad_later_segments_area(self, GVSYReader, fake_adef): res = GVSYReader._pad_later_segments_area([fh_1], dataid) assert len(res) == 2 - # The previous chunk size is 556, which is exactly double the size of the gap left - # by the missing last chunk (278, as the second-to-last chunk ends at line 11136 - 278 ) + # The previous segment size is 556, which is exactly double the size of the gap left + # by the missing last segment (278, as the second-to-last segment ends at line 11136 - 278 ) # therefore, the new vertical area extent should be half of the previous size (1000-500)/2=250. # The new area extent lower-left row is therefore 1000+250=1250 seg2_extent = (0, 1250, 200, 1000) @@ -1393,8 +1393,8 @@ def test_pad_later_segments_area(self, GVSYReader, fake_adef): seg2_extent) fake_adef.assert_called_once_with(*expected_call) - def test_pad_later_segments_area_for_multiple_chunks_gap(self, GVSYReader, fake_adef): - """Test _pad_later_segments_area() in the variable padding case for multiple gaps with multiple chunks.""" + def test_pad_later_segments_area_for_multiple_segments_gap(self, GVSYReader, fake_adef): + """Test _pad_later_segments_area() in the variable padding case for multiple gaps with multiple segments.""" def side_effect_areadef(a, b, c, crs, width, height, aex): m = MagicMock() @@ -1448,30 +1448,30 @@ def side_effect_areadef(a, b, c, crs, width, height, aex): res = GVSYReader._pad_later_segments_area([fh_1, fh_4, fh_8], dataid) assert len(res) == 8 - # Regarding the chunk sizes: - # First group of missing chunks: - # The end position row of the gap is the start row of the last available chunk-1:11136-300-100+1-1=10736 - # The start position row of the gap is the end row fo the first available chunk+1: 11136-600+1=10837 + # Regarding the segment sizes: + # First group of missing segments: + # The end position row of the gap is the start row of the last available segment-1:11136-300-100+1-1=10736 + # The start position row of the gap is the end row fo the first available segment+1: 11136-600+1=10837 # hence the gap is 10736-10537+1=200 px high - # The 200px have to be split between two missing chunks, the most equal way to do it is with + # The 200px have to be split between two missing segments, the most equal way to do it is with # sizes 100: 100+100=200 # Second group: - # The end position row of the gap is the start row of the last chunk -1: 11136-100+1-1=11036 - # The start position row of the gap is the end row fo the first chunk +1: 11136-300+1=10837 + # The end position row of the gap is the start row of the last segment -1: 11136-100+1-1=11036 + # The start position row of the gap is the end row fo the first segment +1: 11136-300+1=10837 # hence the gap is 11036-10837+1=200 px high - # The 200px have to be split between three missing chunks, the most equal way to do it is with + # The 200px have to be split between three missing segments, the most equal way to do it is with # sizes 66 and 67: 66+67+67=200 # Regarding the heights: # First group: - # The first chunk has 100px height and 500 area extent height. - # The first padded chunk has 100px height -> 500*100/100=500 area extent height ->1000+500=1500 - # The second padded chunk has 100px height -> 500*100/100=500 area extent height ->1500+500=2000 + # The first segment has 100px height and 500 area extent height. + # The first padded segment has 100px height -> 500*100/100=500 area extent height ->1000+500=1500 + # The second padded segment has 100px height -> 500*100/100=500 area extent height ->1500+500=2000 # Second group: - # The first chunk has 100px height and 500 area extent height. - # The first padded chunk has 66px height -> 500*66/100=330 area extent height ->1000+330=1330 - # The second padded chunk has 67px height -> 500*67/100=335 area extent height ->1330+335=1665 - # The first padded chunk has 67px height -> 500*67/100=335 area extent height ->1665+335=2000 + # The first segment has 100px height and 500 area extent height. + # The first padded segment has 66px height -> 500*66/100=330 area extent height ->1000+330=1330 + # The second padded segment has 67px height -> 500*67/100=335 area extent height ->1330+335=1665 + # The first padded segment has 67px height -> 500*67/100=335 area extent height ->1665+335=2000 assert fake_adef.call_count == 5 expected_call1 = ('fill', 'fill', 'fill', 'some_crs', 11136, 100, (0, 1500.0, 200, 1000))