diff --git a/satpy/_compat.py b/satpy/_compat.py index 6a2a4fd528..b49b5a961b 100644 --- a/satpy/_compat.py +++ b/satpy/_compat.py @@ -50,7 +50,7 @@ def __get__(self, instance, owner=None): # noqa raise TypeError( "Cannot use cached_property instance without calling __set_name__ on it.") try: - cache = instance.__dict__ + cache = instance.__dict__ # noqa except AttributeError: # not all objects have __dict__ (e.g. class defines slots) msg = ( f"No '__dict__' attribute on {type(instance).__name__!r} " @@ -88,3 +88,9 @@ def __get__(self, instance, owner=None): # noqa # numpy <1.20 from numpy import dtype as DTypeLike # noqa from numpy import ndarray as ArrayLike # noqa + + +try: + from functools import cache # type: ignore +except ImportError: + from functools import lru_cache as cache # noqa diff --git a/satpy/etc/areas.yaml b/satpy/etc/areas.yaml index 04233d059d..9b1cc738ba 100644 --- a/satpy/etc/areas.yaml +++ b/satpy/etc/areas.yaml @@ -335,6 +335,26 @@ SouthAmerica: # ---------- Meteosat Third Generation (MTG) / FCI Instrument ----------------- # Full disk +mtg_fci_fdss_500m: + description: + MTG FCI Full Disk Scanning Service area definition + with 1 km resolution + projection: + proj: geos + lon_0: 0 + h: 35786400 + x_0: 0 + y_0: 0 + ellps: WGS84 + no_defs: null + shape: + height: 22272 + width: 22272 + area_extent: + lower_left_xy: [-5567999.999637696, -5567999.999637696] + upper_right_xy: [5567999.999637678, 5567999.999637678] + units: m + mtg_fci_fdss_1km: description: MTG FCI Full Disk Scanning Service area definition @@ -351,8 +371,8 @@ mtg_fci_fdss_1km: height: 11136 width: 11136 area_extent: - lower_left_xy: [-5567999.998577303, -5567999.998577303] - upper_right_xy: [5567999.998527619, 5567999.998527619] + lower_left_xy: [-5567999.998550739, -5567999.998550739] + upper_right_xy: [5567999.998550762, 5567999.998550762] units: m mtg_fci_fdss_2km: @@ -371,8 +391,8 @@ mtg_fci_fdss_2km: height: 5568 width: 5568 area_extent: - lower_left_xy: [-5567999.994200589, -5567999.994200589] - upper_right_xy: [5567999.994206558, 5567999.994206558] + lower_left_xy: [-5567999.994203018, -5567999.994203018] + upper_right_xy: [5567999.994203017, 5567999.994203017] units: m # Full disk - segmented products @@ -392,8 +412,8 @@ mtg_fci_fdss_6km: height: 1856 width: 1856 area_extent: - lower_left_xy: [-5567999.994200589, -5567999.994200589] - upper_right_xy: [5567999.994206558, 5567999.994206558] + lower_left_xy: [-5567999.994203018, -5567999.994203018] + upper_right_xy: [5567999.994203017, 5567999.994203017] units: m mtg_fci_fdss_32km: @@ -412,8 +432,8 @@ mtg_fci_fdss_32km: height: 348 width: 348 area_extent: - lower_left_xy: [-5567999.994200589, -5567999.994200589] - upper_right_xy: [5567999.994206558, 5567999.994206558] + lower_left_xy: [-5567999.994203018, -5567999.994203018] + upper_right_xy: [5567999.994203017, 5567999.994203017] units: m # Geostationary Operational Environmental Satellite (GOES) / ABI Instrument diff --git a/satpy/etc/readers/fci_l1c_nc.yaml b/satpy/etc/readers/fci_l1c_nc.yaml index 5cc49c095b..483f675cd9 100644 --- a/satpy/etc/readers/fci_l1c_nc.yaml +++ b/satpy/etc/readers/fci_l1c_nc.yaml @@ -6,7 +6,7 @@ reader: Reader for FCI L1c data in NetCDF4 format. Used to read Meteosat Third Generation (MTG) Flexible Combined Imager (FCI) L1c data. - status: Beta for FDHSI, HRFI not supported yet + status: Beta for full-disc FDHSI and HRFI, RSS not supported yet supports_fsspec: false reader: !!python/name:satpy.readers.yaml_reader.GEOVariableSegmentYAMLReader sensors: [ fci ] @@ -16,7 +16,7 @@ reader: file_types: fci_l1c_fdhsi: file_reader: !!python/name:satpy.readers.fci_l1c_nc.FCIL1cNCFileHandler - file_patterns: [ '{pflag}_{location_indicator},{data_designator},MTI{spacecraft_id:1d}+{data_source}-1C-RRAD-FDHSI-{coverage}-{subsetting}-{component1}-BODY-{component3}-{purpose}-{format}_{oflag}_{originator}_{processing_time:%Y%m%d%H%M%S}_{facility_or_tool}_{environment}_{start_time:%Y%m%d%H%M%S}_{end_time:%Y%m%d%H%M%S}_{processing_mode}_{special_compression}_{disposition_mode}_{repeat_cycle_in_day:>04d}_{count_in_repeat_cycle:>04d}.nc' ] + file_patterns: [ '{pflag}_{location_indicator},{data_designator},MTI{spacecraft_id:1d}+{data_source}-1C-RRAD-FDHSI-FD-{subsetting}-{component1}-BODY-{component3}-{purpose}-{format}_{oflag}_{originator}_{processing_time:%Y%m%d%H%M%S}_{facility_or_tool}_{environment}_{start_time:%Y%m%d%H%M%S}_{end_time:%Y%m%d%H%M%S}_{processing_mode}_{special_compression}_{disposition_mode}_{repeat_cycle_in_day:>04d}_{count_in_repeat_cycle:>04d}.nc' ] expected_segments: 40 required_netcdf_variables: - attr/platform @@ -72,6 +72,52 @@ file_types: - ir_105 - ir_123 - ir_133 + fci_l1c_hrfi: + file_reader: !!python/name:satpy.readers.fci_l1c_nc.FCIL1cNCFileHandler + file_patterns: [ '{pflag}_{location_indicator},{data_designator},MTI{spacecraft_id:1d}+{data_source}-1C-RRAD-HRFI-FD-{subsetting}-{component1}-BODY-{component3}-{purpose}-{format}_{oflag}_{originator}_{processing_time:%Y%m%d%H%M%S}_{facility_or_tool}_{environment}_{start_time:%Y%m%d%H%M%S}_{end_time:%Y%m%d%H%M%S}_{processing_mode}_{special_compression}_{disposition_mode}_{repeat_cycle_in_day:>04d}_{count_in_repeat_cycle:>04d}.nc' ] + expected_segments: 40 + required_netcdf_variables: + - attr/platform + - data/mtg_geos_projection + - data/{channel_name}/measured/start_position_row + - data/{channel_name}/measured/end_position_row + - data/{channel_name}/measured/radiance_to_bt_conversion_coefficient_wavenumber + - data/{channel_name}/measured/radiance_to_bt_conversion_coefficient_a + - data/{channel_name}/measured/radiance_to_bt_conversion_coefficient_b + - data/{channel_name}/measured/radiance_to_bt_conversion_constant_c1 + - data/{channel_name}/measured/radiance_to_bt_conversion_constant_c2 + - data/{channel_name}/measured/radiance_unit_conversion_coefficient + - data/{channel_name}/measured/channel_effective_solar_irradiance + - data/{channel_name}/measured/effective_radiance + - data/{channel_name}/measured/x + - data/{channel_name}/measured/y + - data/{channel_name}/measured/pixel_quality + - data/{channel_name}/measured/index_map + - data/mtg_geos_projection/attr/inverse_flattening + - data/mtg_geos_projection/attr/longitude_of_projection_origin + - data/mtg_geos_projection/attr/longitude_of_projection_origin + - data/mtg_geos_projection/attr/perspective_point_height + - data/mtg_geos_projection/attr/perspective_point_height + - data/mtg_geos_projection/attr/perspective_point_height + - data/mtg_geos_projection/attr/semi_major_axis + - data/swath_direction + - data/swath_number + - index + - state/celestial/earth_sun_distance + - state/celestial/earth_sun_distance + - state/celestial/subsolar_latitude + - state/celestial/subsolar_longitude + - state/celestial/sun_satellite_distance + - state/platform/platform_altitude + - state/platform/subsatellite_latitude + - state/platform/subsatellite_longitude + - time + variable_name_replacements: + channel_name: + - vis_06_hr + - nir_22_hr + - ir_38_hr + - ir_105_hr datasets: vis_04: @@ -112,7 +158,9 @@ datasets: name: vis_06 sensor: fci wavelength: [0.590, 0.640, 0.690] - resolution: 1000 + resolution: + 500: {file_type: fci_l1c_hrfi} + 1000: {file_type: fci_l1c_fdhsi} calibration: counts: standard_name: counts @@ -123,7 +171,6 @@ datasets: reflectance: standard_name: toa_bidirectional_reflectance units: "%" - file_type: fci_l1c_fdhsi vis_08: name: vis_08 @@ -197,7 +244,9 @@ datasets: name: nir_22 sensor: fci wavelength: [2.200, 2.250, 2.300] - resolution: 1000 + resolution: + 500: {file_type: fci_l1c_hrfi} + 1000: {file_type: fci_l1c_fdhsi} calibration: counts: standard_name: counts @@ -208,13 +257,14 @@ datasets: reflectance: standard_name: toa_bidirectional_reflectance units: "%" - file_type: fci_l1c_fdhsi ir_38: name: ir_38 sensor: fci wavelength: [3.400, 3.800, 4.200] - resolution: 2000 + resolution: + 1000: {file_type: fci_l1c_hrfi} + 2000: {file_type: fci_l1c_fdhsi} calibration: counts: standard_name: counts @@ -225,7 +275,6 @@ datasets: brightness_temperature: standard_name: toa_brightness_temperature units: "K" - file_type: fci_l1c_fdhsi wv_63: name: wv_63 @@ -299,7 +348,9 @@ datasets: name: ir_105 sensor: fci wavelength: [9.800, 10.500, 11.200] - resolution: 2000 + resolution: + 1000: {file_type: fci_l1c_hrfi} + 2000: {file_type: fci_l1c_fdhsi} calibration: counts: standard_name: counts @@ -310,7 +361,6 @@ datasets: brightness_temperature: standard_name: toa_brightness_temperature units: "K" - file_type: fci_l1c_fdhsi ir_123: name: ir_123 @@ -361,8 +411,9 @@ datasets: vis_06_pixel_quality: name: vis_06_pixel_quality sensor: fci - resolution: 1000 - file_type: fci_l1c_fdhsi + resolution: + 500: {file_type: fci_l1c_hrfi} + 1000: {file_type: fci_l1c_fdhsi} vis_08_pixel_quality: name: vis_08_pixel_quality @@ -391,14 +442,16 @@ datasets: nir_22_pixel_quality: name: nir_22_pixel_quality sensor: fci - resolution: 1000 - file_type: fci_l1c_fdhsi + resolution: + 500: {file_type: fci_l1c_hrfi} + 1000: {file_type: fci_l1c_fdhsi} ir_38_pixel_quality: name: ir_38_pixel_quality sensor: fci - resolution: 2000 - file_type: fci_l1c_fdhsi + resolution: + 1000: {file_type: fci_l1c_hrfi} + 2000: {file_type: fci_l1c_fdhsi} wv_63_pixel_quality: name: wv_63_pixel_quality @@ -427,8 +480,9 @@ datasets: ir_105_pixel_quality: name: ir_105_pixel_quality sensor: fci - resolution: 2000 - file_type: fci_l1c_fdhsi + resolution: + 1000: {file_type: fci_l1c_hrfi} + 2000: {file_type: fci_l1c_fdhsi} ir_123_pixel_quality: name: ir_123_pixel_quality @@ -457,8 +511,9 @@ datasets: vis_06_index_map: name: vis_06_index_map sensor: fci - resolution: 1000 - file_type: fci_l1c_fdhsi + resolution: + 500: {file_type: fci_l1c_hrfi} + 1000: {file_type: fci_l1c_fdhsi} vis_08_index_map: name: vis_08_index_map @@ -487,14 +542,16 @@ datasets: nir_22_index_map: name: nir_22_index_map sensor: fci - resolution: 1000 - file_type: fci_l1c_fdhsi + resolution: + 500: {file_type: fci_l1c_hrfi} + 1000: {file_type: fci_l1c_fdhsi} ir_38_index_map: name: ir_38_index_map sensor: fci - resolution: 2000 - file_type: fci_l1c_fdhsi + resolution: + 1000: {file_type: fci_l1c_hrfi} + 2000: {file_type: fci_l1c_fdhsi} wv_63_index_map: name: wv_63_index_map @@ -523,8 +580,9 @@ datasets: ir_105_index_map: name: ir_105_index_map sensor: fci - resolution: 2000 - file_type: fci_l1c_fdhsi + resolution: + 1000: {file_type: fci_l1c_hrfi} + 2000: {file_type: fci_l1c_fdhsi} ir_123_index_map: name: ir_123_index_map @@ -556,8 +614,9 @@ datasets: name: vis_06_time units: s sensor: fci - resolution: 1000 - file_type: fci_l1c_fdhsi + resolution: + 500: {file_type: fci_l1c_hrfi} + 1000: {file_type: fci_l1c_fdhsi} vis_08_time: name: vis_08_time @@ -591,15 +650,17 @@ datasets: name: nir_22_time units: s sensor: fci - resolution: 1000 - file_type: fci_l1c_fdhsi + resolution: + 500: {file_type: fci_l1c_hrfi} + 1000: {file_type: fci_l1c_fdhsi} ir_38_time: name: ir_38_time units: s sensor: fci - resolution: 2000 - file_type: fci_l1c_fdhsi + resolution: + 1000: {file_type: fci_l1c_hrfi} + 2000: {file_type: fci_l1c_fdhsi} wv_63_time: name: wv_63_time @@ -633,8 +694,9 @@ datasets: name: ir_105_time units: s sensor: fci - resolution: 2000 - file_type: fci_l1c_fdhsi + resolution: + 1000: {file_type: fci_l1c_hrfi} + 2000: {file_type: fci_l1c_fdhsi} ir_123_time: name: ir_123_time @@ -665,8 +727,9 @@ datasets: vis_06_swath_direction: name: vis_06_swath_direction sensor: fci - resolution: 1000 - file_type: fci_l1c_fdhsi + resolution: + 500: {file_type: fci_l1c_hrfi} + 1000: {file_type: fci_l1c_fdhsi} vis_08_swath_direction: name: vis_08_swath_direction @@ -695,14 +758,16 @@ datasets: nir_22_swath_direction: name: nir_22_swath_direction sensor: fci - resolution: 1000 - file_type: fci_l1c_fdhsi + resolution: + 500: {file_type: fci_l1c_hrfi} + 1000: {file_type: fci_l1c_fdhsi} ir_38_swath_direction: name: ir_38_swath_direction sensor: fci - resolution: 2000 - file_type: fci_l1c_fdhsi + resolution: + 1000: {file_type: fci_l1c_hrfi} + 2000: {file_type: fci_l1c_fdhsi} wv_63_swath_direction: name: wv_63_swath_direction @@ -731,8 +796,9 @@ datasets: ir_105_swath_direction: name: ir_105_swath_direction sensor: fci - resolution: 2000 - file_type: fci_l1c_fdhsi + resolution: + 1000: {file_type: fci_l1c_hrfi} + 2000: {file_type: fci_l1c_fdhsi} ir_123_swath_direction: name: ir_123_swath_direction @@ -761,8 +827,9 @@ datasets: vis_06_swath_number: name: vis_06_swath_number sensor: fci - resolution: 1000 - file_type: fci_l1c_fdhsi + resolution: + 500: {file_type: fci_l1c_hrfi} + 1000: {file_type: fci_l1c_fdhsi} vis_08_swath_number: name: vis_08_swath_number @@ -791,14 +858,16 @@ datasets: nir_22_swath_number: name: nir_22_swath_number sensor: fci - resolution: 1000 - file_type: fci_l1c_fdhsi + resolution: + 500: {file_type: fci_l1c_hrfi} + 1000: {file_type: fci_l1c_fdhsi} ir_38_swath_number: name: ir_38_swath_number sensor: fci - resolution: 2000 - file_type: fci_l1c_fdhsi + resolution: + 1000: {file_type: fci_l1c_hrfi} + 2000: {file_type: fci_l1c_fdhsi} wv_63_swath_number: name: wv_63_swath_number @@ -827,8 +896,9 @@ datasets: ir_105_swath_number: name: ir_105_swath_number sensor: fci - resolution: 2000 - file_type: fci_l1c_fdhsi + resolution: + 1000: {file_type: fci_l1c_hrfi} + 2000: {file_type: fci_l1c_fdhsi} ir_123_swath_number: name: ir_123_swath_number @@ -860,8 +930,9 @@ datasets: name: vis_06_subsatellite_latitude units: deg sensor: fci - resolution: 1000 - file_type: fci_l1c_fdhsi + resolution: + 500: {file_type: fci_l1c_hrfi} + 1000: {file_type: fci_l1c_fdhsi} vis_08_subsatellite_latitude: name: vis_08_subsatellite_latitude @@ -895,15 +966,17 @@ datasets: name: nir_22_subsatellite_latitude units: deg sensor: fci - resolution: 1000 - file_type: fci_l1c_fdhsi + resolution: + 500: {file_type: fci_l1c_hrfi} + 1000: {file_type: fci_l1c_fdhsi} ir_38_subsatellite_latitude: name: ir_38_subsatellite_latitude units: deg sensor: fci - resolution: 2000 - file_type: fci_l1c_fdhsi + resolution: + 1000: {file_type: fci_l1c_hrfi} + 2000: {file_type: fci_l1c_fdhsi} wv_63_subsatellite_latitude: name: wv_63_subsatellite_latitude @@ -937,8 +1010,9 @@ datasets: name: ir_105_subsatellite_latitude units: deg sensor: fci - resolution: 2000 - file_type: fci_l1c_fdhsi + resolution: + 1000: {file_type: fci_l1c_hrfi} + 2000: {file_type: fci_l1c_fdhsi} ir_123_subsatellite_latitude: name: ir_123_subsatellite_latitude @@ -972,8 +1046,9 @@ datasets: name: vis_06_subsatellite_longitude units: deg sensor: fci - resolution: 1000 - file_type: fci_l1c_fdhsi + resolution: + 500: {file_type: fci_l1c_hrfi} + 1000: {file_type: fci_l1c_fdhsi} vis_08_subsatellite_longitude: name: vis_08_subsatellite_longitude @@ -1007,15 +1082,17 @@ datasets: name: nir_22_subsatellite_longitude units: deg sensor: fci - resolution: 1000 - file_type: fci_l1c_fdhsi + resolution: + 500: {file_type: fci_l1c_hrfi} + 1000: {file_type: fci_l1c_fdhsi} ir_38_subsatellite_longitude: name: ir_38_subsatellite_longitude units: deg sensor: fci - resolution: 2000 - file_type: fci_l1c_fdhsi + resolution: + 1000: {file_type: fci_l1c_hrfi} + 2000: {file_type: fci_l1c_fdhsi} wv_63_subsatellite_longitude: name: wv_63_subsatellite_longitude @@ -1049,8 +1126,9 @@ datasets: name: ir_105_subsatellite_longitude units: deg sensor: fci - resolution: 2000 - file_type: fci_l1c_fdhsi + resolution: + 1000: {file_type: fci_l1c_hrfi} + 2000: {file_type: fci_l1c_fdhsi} ir_123_subsatellite_longitude: name: ir_123_subsatellite_longitude @@ -1084,8 +1162,9 @@ datasets: name: vis_06_subsolar_latitude units: deg sensor: fci - resolution: 1000 - file_type: fci_l1c_fdhsi + resolution: + 500: {file_type: fci_l1c_hrfi} + 1000: {file_type: fci_l1c_fdhsi} vis_08_subsolar_latitude: name: vis_08_subsolar_latitude @@ -1119,15 +1198,17 @@ datasets: name: nir_22_subsolar_latitude units: deg sensor: fci - resolution: 1000 - file_type: fci_l1c_fdhsi + resolution: + 500: {file_type: fci_l1c_hrfi} + 1000: {file_type: fci_l1c_fdhsi} ir_38_subsolar_latitude: name: ir_38_subsolar_latitude units: deg sensor: fci - resolution: 2000 - file_type: fci_l1c_fdhsi + resolution: + 1000: {file_type: fci_l1c_hrfi} + 2000: {file_type: fci_l1c_fdhsi} wv_63_subsolar_latitude: name: wv_63_subsolar_latitude @@ -1161,8 +1242,9 @@ datasets: name: ir_105_subsolar_latitude units: deg sensor: fci - resolution: 2000 - file_type: fci_l1c_fdhsi + resolution: + 1000: {file_type: fci_l1c_hrfi} + 2000: {file_type: fci_l1c_fdhsi} ir_123_subsolar_latitude: name: ir_123_subsolar_latitude @@ -1196,8 +1278,9 @@ datasets: name: vis_06_subsolar_longitude units: deg sensor: fci - resolution: 1000 - file_type: fci_l1c_fdhsi + resolution: + 500: {file_type: fci_l1c_hrfi} + 1000: {file_type: fci_l1c_fdhsi} vis_08_subsolar_longitude: name: vis_08_subsolar_longitude @@ -1231,15 +1314,17 @@ datasets: name: nir_22_subsolar_longitude units: deg sensor: fci - resolution: 1000 - file_type: fci_l1c_fdhsi + resolution: + 500: {file_type: fci_l1c_hrfi} + 1000: {file_type: fci_l1c_fdhsi} ir_38_subsolar_longitude: name: ir_38_subsolar_longitude units: deg sensor: fci - resolution: 2000 - file_type: fci_l1c_fdhsi + resolution: + 1000: {file_type: fci_l1c_hrfi} + 2000: {file_type: fci_l1c_fdhsi} wv_63_subsolar_longitude: name: wv_63_subsolar_longitude @@ -1273,8 +1358,9 @@ datasets: name: ir_105_subsolar_longitude units: deg sensor: fci - resolution: 2000 - file_type: fci_l1c_fdhsi + resolution: + 1000: {file_type: fci_l1c_hrfi} + 2000: {file_type: fci_l1c_fdhsi} ir_123_subsolar_longitude: name: ir_123_subsolar_longitude @@ -1309,8 +1395,9 @@ datasets: name: vis_06_platform_altitude units: m sensor: fci - resolution: 1000 - file_type: fci_l1c_fdhsi + resolution: + 500: {file_type: fci_l1c_hrfi} + 1000: {file_type: fci_l1c_fdhsi} vis_08_platform_altitude: name: vis_08_platform_altitude @@ -1344,15 +1431,17 @@ datasets: name: nir_22_platform_altitude units: m sensor: fci - resolution: 1000 - file_type: fci_l1c_fdhsi + resolution: + 500: {file_type: fci_l1c_hrfi} + 1000: {file_type: fci_l1c_fdhsi} ir_38_platform_altitude: name: ir_38_platform_altitude units: m sensor: fci - resolution: 2000 - file_type: fci_l1c_fdhsi + resolution: + 1000: {file_type: fci_l1c_hrfi} + 2000: {file_type: fci_l1c_fdhsi} wv_63_platform_altitude: name: wv_63_platform_altitude @@ -1386,8 +1475,9 @@ datasets: name: ir_105_platform_altitude units: m sensor: fci - resolution: 2000 - file_type: fci_l1c_fdhsi + resolution: + 1000: {file_type: fci_l1c_hrfi} + 2000: {file_type: fci_l1c_fdhsi} ir_123_platform_altitude: name: ir_123_platform_altitude @@ -1421,8 +1511,9 @@ datasets: name: vis_06_earth_sun_distance units: km sensor: fci - resolution: 1000 - file_type: fci_l1c_fdhsi + resolution: + 500: {file_type: fci_l1c_hrfi} + 1000: {file_type: fci_l1c_fdhsi} vis_08_earth_sun_distance: name: vis_08_earth_sun_distance @@ -1456,15 +1547,17 @@ datasets: name: nir_22_earth_sun_distance units: km sensor: fci - resolution: 1000 - file_type: fci_l1c_fdhsi + resolution: + 500: {file_type: fci_l1c_hrfi} + 1000: {file_type: fci_l1c_fdhsi} ir_38_earth_sun_distance: name: ir_38_earth_sun_distance units: km sensor: fci - resolution: 2000 - file_type: fci_l1c_fdhsi + resolution: + 1000: {file_type: fci_l1c_hrfi} + 2000: {file_type: fci_l1c_fdhsi} wv_63_earth_sun_distance: name: wv_63_earth_sun_distance @@ -1498,8 +1591,9 @@ datasets: name: ir_105_earth_sun_distance units: km sensor: fci - resolution: 2000 - file_type: fci_l1c_fdhsi + resolution: + 1000: {file_type: fci_l1c_hrfi} + 2000: {file_type: fci_l1c_fdhsi} ir_123_earth_sun_distance: name: ir_123_earth_sun_distance @@ -1533,8 +1627,9 @@ datasets: name: vis_06_sun_satellite_distance units: km sensor: fci - resolution: 1000 - file_type: fci_l1c_fdhsi + resolution: + 500: {file_type: fci_l1c_hrfi} + 1000: {file_type: fci_l1c_fdhsi} vis_08_sun_satellite_distance: name: vis_08_sun_satellite_distance @@ -1568,15 +1663,17 @@ datasets: name: nir_22_sun_satellite_distance units: km sensor: fci - resolution: 1000 - file_type: fci_l1c_fdhsi + resolution: + 500: {file_type: fci_l1c_hrfi} + 1000: {file_type: fci_l1c_fdhsi} ir_38_sun_satellite_distance: name: ir_38_sun_satellite_distance units: km sensor: fci - resolution: 2000 - file_type: fci_l1c_fdhsi + resolution: + 1000: {file_type: fci_l1c_hrfi} + 2000: {file_type: fci_l1c_fdhsi} wv_63_sun_satellite_distance: name: wv_63_sun_satellite_distance @@ -1610,8 +1707,9 @@ datasets: name: ir_105_sun_satellite_distance units: km sensor: fci - resolution: 2000 - file_type: fci_l1c_fdhsi + resolution: + 1000: {file_type: fci_l1c_hrfi} + 2000: {file_type: fci_l1c_fdhsi} ir_123_sun_satellite_distance: name: ir_123_sun_satellite_distance diff --git a/satpy/readers/fci_l1c_nc.py b/satpy/readers/fci_l1c_nc.py index 27e2987e19..bb9b6dda10 100644 --- a/satpy/readers/fci_l1c_nc.py +++ b/satpy/readers/fci_l1c_nc.py @@ -20,16 +20,23 @@ This module defines the :class:`FCIL1cNCFileHandler` file handler, to be used for reading Meteosat Third Generation (MTG) Flexible Combined Imager (FCI) Level-1c data. FCI will fly -on the MTG Imager (MTG-I) series of satellites, scheduled to be launched -in 2022 by the earliest. For more information about FCI, see `EUMETSAT`_. +on the MTG Imager (MTG-I) series of satellites, with the first satellite (MTG-I1) +scheduled to be launched on the 13th of December 2022. +For more information about FCI, see `EUMETSAT`_. -For simulated test data to be used with this reader, see `test data release`_. +For simulated test data to be used with this reader, see `test data releases`_. For the Product User Guide (PUG) of the FCI L1c data, see `PUG`_. .. note:: This reader currently supports Full Disk High Spectral Resolution Imagery - (FDHSI) files. Support for High Spatial Resolution Fast Imagery (HRFI) files - will be implemented when corresponding test datasets will be available. + (FDHSI) and High Spatial Resolution Fast Imagery (HRFI) data in full-disc ("FD") scanning mode. + If the user provides a list of both FDHSI and HRFI files from the same repeat cycle to the Satpy ``Scene``, + Satpy will automatically read the channels from the source with the finest resolution, + i.e. from the HRFI files for the vis_06, nir_22, ir_38, and ir_105 channels. + If needed, the desired resolution can be explicitly requested using e.g.: + ``scn.load(['vis_06'], resolution=1000)``. + + Note that RSS data is not supported yet. Geolocation is based on information from the data files. It uses: @@ -99,7 +106,7 @@ .. _PUG: https://www-cdn.eumetsat.int/files/2020-07/pdf_mtg_fci_l1_pug.pdf .. _EUMETSAT: https://www.eumetsat.int/mtg-flexible-combined-imager # noqa: E501 -.. _test data release: https://www.eumetsat.int/simulated-mtg-fci-l1c-enhanced-non-nominal-datasets +.. _test data releases: https://www.eumetsat.int/mtg-test-data """ from __future__ import absolute_import, division, print_function, unicode_literals @@ -134,6 +141,15 @@ 'swath_direction': 'data/swath_direction', } +HIGH_RES_GRID_INFO = {'fci_l1c_hrfi': {'grid_type': '500m', + 'grid_width': 22272}, + 'fci_l1c_fdhsi': {'grid_type': '1km', + 'grid_width': 11136}} +LOW_RES_GRID_INFO = {'fci_l1c_hrfi': {'grid_type': '1km', + 'grid_width': 11136}, + 'fci_l1c_fdhsi': {'grid_type': '2km', + 'grid_width': 5568}} + def _get_aux_data_name_from_dsname(dsname): aux_data_name = [key for key in AUX_DATA.keys() if key in dsname] @@ -204,22 +220,45 @@ def end_time(self): """Get end time.""" return self.filename_info['end_time'] + def get_channel_measured_group_path(self, channel): + """Get the channel's measured group path.""" + if self.filetype_info['file_type'] == 'fci_l1c_hrfi': + channel += '_hr' + measured_group_path = 'data/{}/measured'.format(channel) + + return measured_group_path + def get_segment_position_info(self): - """Get the vertical position and size information of the chunk (aka segment) for both 1km and 2km grids. + """Get information about the size and the position of the segment inside the final image array. + + As the final array is composed by stacking segments vertically, the position of a segment + inside the array is defined by the numbers of the start (lowest) and end (highest) row of the segment. + The row numbering is assumed to start with 1. + This info is used in the GEOVariableSegmentYAMLReader to compute optimal segment sizes for missing segments. - This is used in the GEOVariableSegmentYAMLReader to compute optimal chunk sizes for missing chunks. + Note: in the FCI terminology, a segment is actually called "chunk". To avoid confusion with the dask concept + of chunk, and to be consistent with SEVIRI, we opt to use the word segment. """ + vis_06_measured_path = self.get_channel_measured_group_path('vis_06') + ir_105_measured_path = self.get_channel_measured_group_path('ir_105') + + file_type = self.filetype_info['file_type'] + segment_position_info = { - '1km': {'start_position_row': self.get_and_cache_npxr("data/vis_04/measured/start_position_row").item(), - 'end_position_row': self.get_and_cache_npxr('data/vis_04/measured/end_position_row').item(), - 'segment_height': self.get_and_cache_npxr('data/vis_04/measured/end_position_row').item() - - self.get_and_cache_npxr('data/vis_04/measured/start_position_row').item() + 1, - 'segment_width': 11136}, - '2km': {'start_position_row': self.get_and_cache_npxr('data/ir_105/measured/start_position_row').item(), - 'end_position_row': self.get_and_cache_npxr('data/ir_105/measured/end_position_row').item(), - 'segment_height': self.get_and_cache_npxr('data/ir_105/measured/end_position_row').item() - - self.get_and_cache_npxr('data/ir_105/measured/start_position_row').item() + 1, - 'segment_width': 5568} + HIGH_RES_GRID_INFO[file_type]['grid_type']: { + 'start_position_row': self.get_and_cache_npxr(vis_06_measured_path + '/start_position_row').item(), + 'end_position_row': self.get_and_cache_npxr(vis_06_measured_path + '/end_position_row').item(), + 'segment_height': self.get_and_cache_npxr(vis_06_measured_path + '/end_position_row').item() - + self.get_and_cache_npxr(vis_06_measured_path + '/start_position_row').item() + 1, + 'grid_width': HIGH_RES_GRID_INFO[file_type]['grid_width'] + }, + LOW_RES_GRID_INFO[file_type]['grid_type']: { + 'start_position_row': self.get_and_cache_npxr(ir_105_measured_path + '/start_position_row').item(), + 'end_position_row': self.get_and_cache_npxr(ir_105_measured_path + '/end_position_row').item(), + 'segment_height': self.get_and_cache_npxr(ir_105_measured_path + '/end_position_row').item() - + self.get_and_cache_npxr(ir_105_measured_path + '/start_position_row').item() + 1, + 'grid_width': LOW_RES_GRID_INFO[file_type]['grid_width'] + } } return segment_position_info @@ -314,7 +353,7 @@ def _get_dataset_measurand(self, key, info=None): @cached_property def orbital_param(self): - """Compute the orbital parameters for the current chunk.""" + """Compute the orbital parameters for the current segment.""" actual_subsat_lon = float(np.nanmean(self._get_aux_data_lut_vector('subsatellite_longitude'))) actual_subsat_lat = float(np.nanmean(self._get_aux_data_lut_vector('subsatellite_latitude'))) actual_sat_alt = float(np.nanmean(self._get_aux_data_lut_vector('platform_altitude'))) @@ -386,13 +425,6 @@ def _get_dataset_aux_data(self, dsname): return aux - @staticmethod - def get_channel_measured_group_path(channel): - """Get the channel's measured group path.""" - measured_group_path = 'data/{}/measured'.format(channel) - - return measured_group_path - def calc_area_extent(self, key): """Calculate area extent for a dataset.""" # if a user requests a pixel quality or index map before the channel data, the @@ -413,7 +445,7 @@ def calc_area_extent(self, key): extents = {} for coord in "xy": - coord_radian = self.get_and_cache_npxr("data/{:s}/measured/{:s}".format(channel_name, coord)) + coord_radian = self.get_and_cache_npxr(measured + "/{:s}".format(coord)) # TODO remove this check when old versions of IDPF test data ( 0: diff --git a/satpy/readers/yaml_reader.py b/satpy/readers/yaml_reader.py index ba1aeadbea..dffa4c7189 100644 --- a/satpy/readers/yaml_reader.py +++ b/satpy/readers/yaml_reader.py @@ -26,7 +26,6 @@ from collections import OrderedDict, deque from contextlib import suppress from fnmatch import fnmatch -from functools import cached_property from weakref import WeakValueDictionary import numpy as np @@ -38,6 +37,7 @@ from yaml import UnsafeLoader from satpy import DatasetDict +from satpy._compat import cache from satpy.aux_download import DataDownloadMixin from satpy.dataset import DataID, DataQuery, get_key from satpy.dataset.dataid import default_co_keys_config, default_id_keys_config, get_keys_from_config @@ -1358,7 +1358,7 @@ def _get_empty_segment_with_height(empty_segment, new_height, dim): class GEOVariableSegmentYAMLReader(GEOSegmentYAMLReader): - """GEOVariableSegmentYAMLReader for handling chunked/segmented GEO products with segments of variable height. + """GEOVariableSegmentYAMLReader for handling segmented GEO products with segments of variable height. This YAMLReader overrides parts of the GEOSegmentYAMLReader to account for formats where the segments can have variable heights. It computes the sizes of the padded segments using the information available in the @@ -1367,59 +1367,59 @@ class GEOVariableSegmentYAMLReader(GEOSegmentYAMLReader): This implementation was motivated by the FCI L1c format, where the segments (called chunks in the FCI world) can have variable heights. It is however generic, so that any future reader can use it. The requirement for the reader is to have a method called `get_segment_position_info`, returning a dictionary containing - the positioning info for each chunk (see example in + the positioning info for each segment (see example in :func:`satpy.readers.fci_l1c_nc.FCIL1cNCFileHandler.get_segment_position_info`). For more information on please see the documentation of :func:`satpy.readers.yaml_reader.GEOSegmentYAMLReader`. """ - def create_filehandlers(self, filenames, fh_kwargs=None): - """Create file handler objects and collect the location information.""" - created_fhs = super().create_filehandlers(filenames, fh_kwargs=fh_kwargs) - self._extract_segment_location_dicts(created_fhs) - return created_fhs - - def _extract_segment_location_dicts(self, created_fhs): + def __init__(self, + config_dict, + filter_parameters=None, + filter_filenames=True, + **kwargs): + """Initialise the GEOVariableSegmentYAMLReader object.""" + super().__init__(config_dict, filter_parameters, filter_filenames, **kwargs) + self.segment_heights = cache(self._segment_heights) self.segment_infos = dict() - for filetype, filetype_fhs in created_fhs.items(): - self._initialise_segment_infos(filetype, filetype_fhs) - self._collect_segment_position_infos(filetype, filetype_fhs) + + def _extract_segment_location_dicts(self, filetype): + self._initialise_segment_infos(filetype) + self._collect_segment_position_infos(filetype) return - def _collect_segment_position_infos(self, filetype, filetype_fhs): + def _collect_segment_position_infos(self, filetype): # collect the segment positioning infos for all available segments - for fh in filetype_fhs: + for fh in self.file_handlers[filetype]: chk_infos = fh.get_segment_position_info() chk_infos.update({'segment_nr': fh.filename_info['segment'] - 1}) self.segment_infos[filetype]['available_segment_infos'].append(chk_infos) - def _initialise_segment_infos(self, filetype, filetype_fhs): + def _initialise_segment_infos(self, filetype): # initialise the segment info for this filetype - exp_segment_nr = filetype_fhs[0].filetype_info['expected_segments'] - width_to_grid_type = _get_width_to_grid_type(filetype_fhs[0].get_segment_position_info()) + filetype_fhs_sample = self.file_handlers[filetype][0] + exp_segment_nr = filetype_fhs_sample.filetype_info['expected_segments'] + grid_width_to_grid_type = _get_grid_width_to_grid_type(filetype_fhs_sample.get_segment_position_info()) self.segment_infos.update({filetype: {'available_segment_infos': [], 'expected_segments': exp_segment_nr, - 'width_to_grid_type': width_to_grid_type}}) + 'grid_width_to_grid_type': grid_width_to_grid_type}}) def _get_empty_segment(self, dim=None, idx=None, filetype=None): - grid_type = self.segment_infos[filetype]['width_to_grid_type'][self.empty_segment.shape[1]] - segment_height = self.segment_heights[filetype][grid_type][idx] + grid_width = self.empty_segment.shape[1] + segment_height = self.segment_heights(filetype, grid_width)[idx] return _get_empty_segment_with_height(self.empty_segment, segment_height, dim=dim) - @cached_property - def segment_heights(self): + def _segment_heights(self, filetype, grid_width): """Compute optimal padded segment heights (in number of pixels) based on the location of available segments.""" - segment_heights = dict() - for filetype, filetype_seginfos in self.segment_infos.items(): - filetype_seg_heights = {'1km': _compute_optimal_missing_segment_heights(filetype_seginfos, '1km', 11136), - '2km': _compute_optimal_missing_segment_heights(filetype_seginfos, '2km', 5568)} - segment_heights.update({filetype: filetype_seg_heights}) + self._extract_segment_location_dicts(filetype) + grid_type = self.segment_infos[filetype]['grid_width_to_grid_type'][grid_width] + segment_heights = _compute_optimal_missing_segment_heights(self.segment_infos[filetype], grid_type, grid_width) return segment_heights def _get_new_areadef_heights(self, previous_area, previous_seg_size, segment_n=None, filetype=None): # retrieve the segment height in number of pixels - grid_type = self.segment_infos[filetype]['width_to_grid_type'][previous_seg_size[1]] - new_height_px = self.segment_heights[filetype][grid_type][segment_n - 1] + grid_width = previous_seg_size[1] + new_height_px = self.segment_heights(filetype, grid_width)[segment_n - 1] # scale the previous vertical area extent using the new pixel height prev_area_extent = previous_area.area_extent[1] - previous_area.area_extent[3] new_height_proj_coord = prev_area_extent * new_height_px / previous_seg_size[0] @@ -1427,11 +1427,11 @@ def _get_new_areadef_heights(self, previous_area, previous_seg_size, segment_n=N return new_height_proj_coord, new_height_px -def _get_width_to_grid_type(seg_info): - width_to_grid_type = dict() +def _get_grid_width_to_grid_type(seg_info): + grid_width_to_grid_type = dict() for grid_type, grid_type_seg_info in seg_info.items(): - width_to_grid_type.update({grid_type_seg_info['segment_width']: grid_type}) - return width_to_grid_type + grid_width_to_grid_type.update({grid_type_seg_info['grid_width']: grid_type}) + return grid_width_to_grid_type def _compute_optimal_missing_segment_heights(seg_infos, grid_type, expected_vertical_size): @@ -1501,13 +1501,13 @@ def _init_positioning_arrays_for_variable_padding(chk_infos, grid_type, exp_segm segment_start_rows = np.zeros(exp_segment_nr) segment_end_rows = np.zeros(exp_segment_nr) - _populate_positioning_arrays_with_available_chunk_info(chk_infos, grid_type, segment_start_rows, segment_end_rows, - segment_heights) + _populate_positioning_arrays_with_available_segment_info(chk_infos, grid_type, segment_start_rows, segment_end_rows, + segment_heights) return segment_start_rows, segment_end_rows, segment_heights -def _populate_positioning_arrays_with_available_chunk_info(chk_infos, grid_type, segment_start_rows, segment_end_rows, - segment_heights): +def _populate_positioning_arrays_with_available_segment_info(chk_infos, grid_type, segment_start_rows, segment_end_rows, + segment_heights): for chk_info in chk_infos: current_fh_segment_nr = chk_info['segment_nr'] segment_heights[current_fh_segment_nr] = chk_info[grid_type]['segment_height'] diff --git a/satpy/tests/reader_tests/test_fci_l1c_nc.py b/satpy/tests/reader_tests/test_fci_l1c_nc.py index ae0b74ae0c..3995a36ece 100644 --- a/satpy/tests/reader_tests/test_fci_l1c_nc.py +++ b/satpy/tests/reader_tests/test_fci_l1c_nc.py @@ -16,10 +16,10 @@ # You should have received a copy of the GNU General Public License along with # satpy. If not, see . """Tests for the 'fci_l1c_nc' reader.""" - +import contextlib import logging import os -from typing import Dict +from typing import Dict, List, Union from unittest import mock import dask.array as da @@ -27,248 +27,304 @@ import numpy.testing import pytest import xarray as xr +from netCDF4 import default_fillvals +from pytest_lazyfixture import lazy_fixture +from satpy.readers.fci_l1c_nc import FCIL1cNCFileHandler from satpy.tests.reader_tests.test_netcdf_utils import FakeNetCDF4FileHandler - - -class FakeNetCDF4FileHandler2(FakeNetCDF4FileHandler): - """Class for faking the NetCDF4 Filehandler.""" - - cached_file_content: Dict[str, xr.DataArray] = {} - - def _get_test_calib_for_channel_ir(self, chroot, meas): - from pyspectral.blackbody import C_SPEED as c - from pyspectral.blackbody import H_PLANCK as h - from pyspectral.blackbody import K_BOLTZMANN as k - xrda = xr.DataArray - data = {} - data[meas + "/radiance_to_bt_conversion_coefficient_wavenumber"] = xrda(955) - data[meas + "/radiance_to_bt_conversion_coefficient_a"] = xrda(1) - data[meas + "/radiance_to_bt_conversion_coefficient_b"] = xrda(0.4) - data[meas + "/radiance_to_bt_conversion_constant_c1"] = xrda(1e11 * 2 * h * c ** 2) - data[meas + "/radiance_to_bt_conversion_constant_c2"] = xrda(1e2 * h * c / k) - return data - - def _get_test_calib_for_channel_vis(self, chroot, meas): - xrda = xr.DataArray - data = {} - data["state/celestial/earth_sun_distance"] = xrda(da.repeat(da.array([149597870.7]), 6000)) - data[meas + "/channel_effective_solar_irradiance"] = xrda(50) - return data - - def _get_test_content_for_channel(self, pat, ch): - xrda = xr.DataArray - nrows = 200 - ncols = 11136 - chroot = "data/{:s}" - meas = chroot + "/measured" - rad = meas + "/effective_radiance" - qual = meas + "/pixel_quality" - index_map = meas + "/index_map" - rad_conv_coeff = meas + "/radiance_unit_conversion_coefficient" - pos = meas + "/{:s}_position_{:s}" - shp = rad + "/shape" - x = meas + "/x" - y = meas + "/y" - data = {} - ch_str = pat.format(ch) - ch_path = rad.format(ch_str) - - common_attrs = { - "scale_factor": 5, - "add_offset": 10, - "long_name": "Effective Radiance", - "units": "mW.m-2.sr-1.(cm-1)-1", - "ancillary_variables": "pixel_quality" - } - if ch == 38: - fire_line = da.ones((1, ncols), dtype="uint16", chunks=1024) * 5000 - data_without_fires = da.ones((nrows - 1, ncols), dtype="uint16", chunks=1024) - d = xrda( - da.concatenate([fire_line, data_without_fires], axis=0), - dims=("y", "x"), - attrs={ - "valid_range": [0, 8191], - "warm_scale_factor": 2, - "warm_add_offset": -300, - **common_attrs - } - ) - else: - d = xrda( - da.ones((nrows, ncols), dtype="uint16", chunks=1024), - dims=("y", "x"), - attrs={ - "valid_range": [0, 4095], - "warm_scale_factor": 1, - "warm_add_offset": 0, - **common_attrs - } - ) - - data[ch_path] = d - data[x.format(ch_str)] = xrda( - da.arange(1, ncols + 1, dtype="uint16"), - dims=("x",), +from satpy.tests.utils import make_dataid + +GRID_TYPE_INFO_FOR_TEST_CONTENT = { + '500m': { + 'nrows': 400, + 'ncols': 22272, + 'scale_factor': 1.39717881644274e-05, + 'add_offset': 1.55596818893146e-01, + }, + '1km': { + 'nrows': 200, + 'ncols': 11136, + 'scale_factor': 2.79435763233999e-05, + 'add_offset': 1.55603804756852e-01, + }, + '2km': { + 'nrows': 100, + 'ncols': 5568, + 'scale_factor': 5.58871526031607e-05, + 'add_offset': 1.55617776423501e-01, + }, +} + + +# ---------------------------------------------------- +# Filehandlers preparation --------------------------- +# ---------------------------------------------------- + +def _get_test_calib_for_channel_ir(data, meas_path): + from pyspectral.blackbody import C_SPEED as c + from pyspectral.blackbody import H_PLANCK as h + from pyspectral.blackbody import K_BOLTZMANN as k + data[meas_path + "/radiance_to_bt_conversion_coefficient_wavenumber"] = xr.DataArray(955) + data[meas_path + "/radiance_to_bt_conversion_coefficient_a"] = xr.DataArray(1) + data[meas_path + "/radiance_to_bt_conversion_coefficient_b"] = xr.DataArray(0.4) + data[meas_path + "/radiance_to_bt_conversion_constant_c1"] = xr.DataArray(1e11 * 2 * h * c ** 2) + data[meas_path + "/radiance_to_bt_conversion_constant_c2"] = xr.DataArray(1e2 * h * c / k) + return data + + +def _get_test_calib_for_channel_vis(data, meas): + data["state/celestial/earth_sun_distance"] = xr.DataArray(da.repeat(da.array([149597870.7]), 6000)) + data[meas + "/channel_effective_solar_irradiance"] = xr.DataArray(50) + return data + + +def _get_test_calib_data_for_channel(data, ch_str): + meas_path = "data/{:s}/measured".format(ch_str) + if ch_str.startswith("ir") or ch_str.startswith("wv"): + _get_test_calib_for_channel_ir(data, meas_path) + elif ch_str.startswith("vis") or ch_str.startswith("nir"): + _get_test_calib_for_channel_vis(data, meas_path) + data[meas_path + "/radiance_unit_conversion_coefficient"] = xr.DataArray(1234.56) + + +def _get_test_image_data_for_channel(data, ch_str, n_rows_cols): + ch_path = "data/{:s}/measured/effective_radiance".format(ch_str) + + common_attrs = { + "scale_factor": 5, + "add_offset": 10, + "long_name": "Effective Radiance", + "units": "mW.m-2.sr-1.(cm-1)-1", + "ancillary_variables": "pixel_quality" + } + if "38" in ch_path: + fire_line = da.ones((1, n_rows_cols[1]), dtype="uint16", chunks=1024) * 5000 + data_without_fires = da.ones((n_rows_cols[0] - 1, n_rows_cols[1]), dtype="uint16", chunks=1024) + d = xr.DataArray( + da.concatenate([fire_line, data_without_fires], axis=0), + dims=("y", "x"), attrs={ - "scale_factor": -5.58877772833e-05, - "add_offset": 0.155619515845, + "valid_range": [0, 8191], + "warm_scale_factor": 2, + "warm_add_offset": -300, + **common_attrs } ) - data[y.format(ch_str)] = xrda( - da.arange(1, nrows + 1, dtype="uint16"), - dims=("y",), + else: + d = xr.DataArray( + da.ones(n_rows_cols, dtype="uint16", chunks=1024), + dims=("y", "x"), attrs={ - "scale_factor": -5.58877772833e-05, - "add_offset": 0.155619515845, + "valid_range": [0, 4095], + "warm_scale_factor": 1, + "warm_add_offset": 0, + **common_attrs } ) - data[qual.format(ch_str)] = xrda( - da.arange(nrows * ncols, dtype="uint8").reshape(nrows, ncols) % 128, - dims=("y", "x")) - # add dummy data for index map starting from 100 - data[index_map.format(ch_str)] = xrda( - (da.arange(nrows * ncols, dtype="uint16").reshape(nrows, ncols) % 6000) + 100, - dims=("y", "x")) - - data[rad_conv_coeff.format(ch_str)] = xrda(1234.56) - data[pos.format(ch_str, "start", "row")] = xrda(0) - data[pos.format(ch_str, "start", "column")] = xrda(0) - data[pos.format(ch_str, "end", "row")] = xrda(nrows) - data[pos.format(ch_str, "end", "column")] = xrda(ncols) - if pat.startswith("ir") or pat.startswith("wv"): - data.update(self._get_test_calib_for_channel_ir(chroot.format(ch_str), - meas.format(ch_str))) - elif pat.startswith("vis") or pat.startswith("nir"): - data.update(self._get_test_calib_for_channel_vis(chroot.format(ch_str), - meas.format(ch_str))) - data[shp.format(ch_str)] = (nrows, ncols) - return data - def _get_test_content_all_channels(self): - chan_patterns = { - "vis_{:>02d}": (4, 5, 6, 8, 9), - "nir_{:>02d}": (13, 16, 22), - "ir_{:>02d}": (38, 87, 97, 105, 123, 133), - "wv_{:>02d}": (63, 73), + data[ch_path] = d + data[ch_path + '/shape'] = n_rows_cols + + +def _get_test_segment_position_for_channel(data, ch_str, n_rows_cols): + pos = "data/{:s}/measured/{:s}_position_{:s}" + data[pos.format(ch_str, "start", "row")] = xr.DataArray(1) + data[pos.format(ch_str, "start", "column")] = xr.DataArray(1) + data[pos.format(ch_str, "end", "row")] = xr.DataArray(n_rows_cols[0]) + data[pos.format(ch_str, "end", "column")] = xr.DataArray(n_rows_cols[1]) + + +def _get_test_index_map_for_channel(data, ch_str, n_rows_cols): + index_map_path = "data/{:s}/measured/index_map".format(ch_str) + data[index_map_path] = xr.DataArray((da.ones(n_rows_cols)) * 110, dims=("y", "x")) + + +def _get_test_pixel_quality_for_channel(data, ch_str, n_rows_cols): + qual_path = "data/{:s}/measured/pixel_quality".format(ch_str) + data[qual_path] = xr.DataArray((da.ones(n_rows_cols)) * 3, dims=("y", "x")) + + +def _get_test_geolocation_for_channel(data, ch_str, grid_type, n_rows_cols): + x_path = "data/{:s}/measured/x".format(ch_str) + data[x_path] = xr.DataArray( + da.arange(1, n_rows_cols[1] + 1, dtype=np.dtype("uint16")), + dims=("x",), + attrs={ + "scale_factor": -GRID_TYPE_INFO_FOR_TEST_CONTENT[grid_type]['scale_factor'], + "add_offset": GRID_TYPE_INFO_FOR_TEST_CONTENT[grid_type]['add_offset'], } - data = {} - for pat in chan_patterns: - for ch_num in chan_patterns[pat]: - data.update(self._get_test_content_for_channel(pat, ch_num)) - return data + ) + + y_path = "data/{:s}/measured/y".format(ch_str) + data[y_path] = xr.DataArray( + da.arange(1, n_rows_cols[0] + 1, dtype=np.dtype("uint16")), + dims=("y",), + attrs={ + "scale_factor": GRID_TYPE_INFO_FOR_TEST_CONTENT[grid_type]['scale_factor'], + "add_offset": -GRID_TYPE_INFO_FOR_TEST_CONTENT[grid_type]['add_offset'], + } + ) - def _get_test_content_areadef(self): - data = {} - proj = "data/mtg_geos_projection" - - attrs = { - "sweep_angle_axis": "y", - "perspective_point_height": "35786400.0", - "semi_major_axis": "6378137.0", - "longitude_of_projection_origin": "0.0", - "inverse_flattening": "298.257223563", - "units": "m"} - data[proj] = xr.DataArray( - 0, - dims=(), - attrs=attrs) - - # also set attributes cached, as this may be how they are accessed with - # the NetCDF4FileHandler - for (k, v) in attrs.items(): - data[proj + "/attr/" + k] = v +def _get_test_content_areadef(): + data = {} - return data + proj = "data/mtg_geos_projection" - def _get_test_content_aux_data(self): - from satpy.readers.fci_l1c_nc import AUX_DATA - xrda = xr.DataArray - data = {} - indices_dim = 6000 - for key, value in AUX_DATA.items(): - # skip population of earth_sun_distance as this is already defined for reflectance calculation - if key == 'earth_sun_distance': - continue - data[value] = xrda(da.arange(indices_dim, dtype="float32"), dims=("index")) + attrs = { + "sweep_angle_axis": "y", + "perspective_point_height": "35786400.0", + "semi_major_axis": "6378137.0", + "longitude_of_projection_origin": "0.0", + "inverse_flattening": "298.257223563", + "units": "m"} + data[proj] = xr.DataArray( + 0, + dims=(), + attrs=attrs) - # compute the last data entry to simulate the FCI caching - data[list(AUX_DATA.values())[-1]] = data[list(AUX_DATA.values())[-1]].compute() + # also set attributes cached, as this may be how they are accessed with + # the NetCDF4FileHandler + for (k, v) in attrs.items(): + data[proj + "/attr/" + k] = v + + return data + + +def _get_test_content_aux_data(): + from satpy.readers.fci_l1c_nc import AUX_DATA + data = {} + indices_dim = 6000 + for key, value in AUX_DATA.items(): + # skip population of earth_sun_distance as this is already defined for reflectance calculation + if key == 'earth_sun_distance': + continue + data[value] = xr.DataArray(da.arange(indices_dim, dtype="float32"), dims=("index")) + + # compute the last data entry to simulate the FCI caching + data[list(AUX_DATA.values())[-1]] = data[list(AUX_DATA.values())[-1]].compute() + + data['index'] = xr.DataArray(da.ones(indices_dim, dtype="uint16") * 100, dims=("index")) + return data - data['index'] = xrda(da.arange(indices_dim, dtype="uint16")+100, dims=("index")) - return data - def _get_global_attributes(self): +def _get_global_attributes(): + data = {} + attrs = {"platform": "MTI1"} + for (k, v) in attrs.items(): + data["attr/" + k] = v + return data + + +def _get_test_content_for_channel(ch_str, grid_type): + nrows = GRID_TYPE_INFO_FOR_TEST_CONTENT[grid_type]['nrows'] + ncols = GRID_TYPE_INFO_FOR_TEST_CONTENT[grid_type]['ncols'] + n_rows_cols = (nrows, ncols) + + data = {} + + _get_test_image_data_for_channel(data, ch_str, n_rows_cols) + _get_test_calib_data_for_channel(data, ch_str) + _get_test_geolocation_for_channel(data, ch_str, grid_type, n_rows_cols) + _get_test_pixel_quality_for_channel(data, ch_str, n_rows_cols) + _get_test_index_map_for_channel(data, ch_str, n_rows_cols) + _get_test_segment_position_for_channel(data, ch_str, n_rows_cols) + + return data + + +class FakeFCIFileHandlerBase(FakeNetCDF4FileHandler): + """Class for faking the NetCDF4 Filehandler.""" + + cached_file_content: Dict[str, xr.DataArray] = {} + # overwritten by FDHSI and HRFI FIle Handlers + chan_patterns: Dict[str, Dict[str, Union[List[int], str]]] = {} + + def _get_test_content_all_channels(self): data = {} - attrs = {"platform": "MTI1"} - for (k, v) in attrs.items(): - data["attr/" + k] = v + for pat in self.chan_patterns: + for ch in self.chan_patterns[pat]['channels']: + data.update(_get_test_content_for_channel(pat.format(ch), self.chan_patterns[pat]['grid_type'])) return data def get_test_content(self, filename, filename_info, filetype_info): """Get the content of the test data.""" - # mock global attributes - # - root groups global - # - other groups global - # mock data variables - # mock dimensions - # - # ... but only what satpy is using ... - D = {} D.update(self._get_test_content_all_channels()) - D.update(self._get_test_content_areadef()) - D.update(self._get_test_content_aux_data()) - D.update(self._get_global_attributes()) + D.update(_get_test_content_areadef()) + D.update(_get_test_content_aux_data()) + D.update(_get_global_attributes()) return D -class FakeNetCDF4FileHandler3(FakeNetCDF4FileHandler2): +class FakeFCIFileHandlerFDHSI(FakeFCIFileHandlerBase): + """Mock FDHSI data.""" + + chan_patterns = { + "vis_{:>02d}": {'channels': [4, 5, 6, 8, 9], + 'grid_type': '1km'}, + "nir_{:>02d}": {'channels': [13, 16, 22], + 'grid_type': '1km'}, + "ir_{:>02d}": {'channels': [38, 87, 97, 105, 123, 133], + 'grid_type': '2km'}, + "wv_{:>02d}": {'channels': [63, 73], + 'grid_type': '2km'}, + } + + +class FakeFCIFileHandlerWithBadData(FakeFCIFileHandlerFDHSI): """Mock bad data.""" - def _get_test_calib_for_channel_ir(self, chroot, meas): - from netCDF4 import default_fillvals + def _get_test_content_all_channels(self): + data = super()._get_test_content_all_channels() v = xr.DataArray(default_fillvals["f4"]) - data = {} - data[meas + "/radiance_to_bt_conversion_coefficient_wavenumber"] = v - data[meas + "/radiance_to_bt_conversion_coefficient_a"] = v - data[meas + "/radiance_to_bt_conversion_coefficient_b"] = v - data[meas + "/radiance_to_bt_conversion_constant_c1"] = v - data[meas + "/radiance_to_bt_conversion_constant_c2"] = v - return data - def _get_test_calib_for_channel_vis(self, chroot, meas): - data = super()._get_test_calib_for_channel_vis(chroot, meas) - from netCDF4 import default_fillvals - v = xr.DataArray(default_fillvals["f4"]) - data[meas + "/channel_effective_solar_irradiance"] = v + data.update({"data/ir_105/measured/radiance_to_bt_conversion_coefficient_wavenumber": v, + "data/ir_105/measured/radiance_to_bt_conversion_coefficient_a": v, + "data/ir_105/measured/radiance_to_bt_conversion_coefficient_b": v, + "data/ir_105/measured/radiance_to_bt_conversion_constant_c1": v, + "data/ir_105/measured/radiance_to_bt_conversion_constant_c2": v, + "data/vis_06/measured/channel_effective_solar_irradiance": v}) + return data -class FakeNetCDF4FileHandler4(FakeNetCDF4FileHandler2): +class FakeFCIFileHandlerWithBadIDPFData(FakeFCIFileHandlerFDHSI): """Mock bad data for IDPF TO-DO's.""" - def _get_test_calib_for_channel_vis(self, chroot, meas): - data = super()._get_test_calib_for_channel_vis(chroot, meas) - data["state/celestial/earth_sun_distance"] = xr.DataArray(da.repeat(da.array([30000000]), 6000)) - return data - def _get_test_content_all_channels(self): data = super()._get_test_content_all_channels() - data['data/vis_04/measured/x'].attrs['scale_factor'] *= -1 - data['data/vis_04/measured/x'].attrs['scale_factor'] = \ - np.float32(data['data/vis_04/measured/x'].attrs['scale_factor']) - data['data/vis_04/measured/x'].attrs['add_offset'] = \ - np.float32(data['data/vis_04/measured/x'].attrs['add_offset']) - data['data/vis_04/measured/y'].attrs['scale_factor'] = \ - np.float32(data['data/vis_04/measured/y'].attrs['scale_factor']) - data['data/vis_04/measured/y'].attrs['add_offset'] = \ - np.float32(data['data/vis_04/measured/y'].attrs['add_offset']) + data['data/vis_06/measured/x'].attrs['scale_factor'] *= -1 + data['data/vis_06/measured/x'].attrs['scale_factor'] = \ + np.float32(data['data/vis_06/measured/x'].attrs['scale_factor']) + data['data/vis_06/measured/x'].attrs['add_offset'] = \ + np.float32(data['data/vis_06/measured/x'].attrs['add_offset']) + data['data/vis_06/measured/y'].attrs['scale_factor'] = \ + np.float32(data['data/vis_06/measured/y'].attrs['scale_factor']) + data['data/vis_06/measured/y'].attrs['add_offset'] = \ + np.float32(data['data/vis_06/measured/y'].attrs['add_offset']) + data["state/celestial/earth_sun_distance"] = xr.DataArray(da.repeat(da.array([30000000]), 6000)) return data +class FakeFCIFileHandlerHRFI(FakeFCIFileHandlerBase): + """Mock HRFI data.""" + + chan_patterns = { + "vis_{:>02d}_hr": {'channels': [6], + 'grid_type': '500m'}, + "nir_{:>02d}_hr": {'channels': [22], + 'grid_type': '500m'}, + "ir_{:>02d}_hr": {'channels': [38, 105], + 'grid_type': '1km'}, + } + + +# ---------------------------------------------------- +# Fixtures preparation ------------------------------- +# ---------------------------------------------------- + @pytest.fixture def reader_configs(): """Return reader configs for FCI.""" @@ -285,186 +341,198 @@ def _get_reader_with_filehandlers(filenames, reader_configs): return reader -class TestFCIL1cNCReader: - """Initialize the unittest TestCase for the FCI L1c NetCDF Reader.""" +_chans_fdhsi = {"solar": ["vis_04", "vis_05", "vis_06", "vis_08", "vis_09", + "nir_13", "nir_16", "nir_22"], + "solar_grid_type": ["1km"] * 8, + "terran": ["ir_38", "wv_63", "wv_73", "ir_87", "ir_97", "ir_105", + "ir_123", "ir_133"], + "terran_grid_type": ["2km"] * 8} + +_chans_hrfi = {"solar": ["vis_06", "nir_22"], + "solar_grid_type": ["500m"] * 2, + "terran": ["ir_38", "ir_105"], + "terran_grid_type": ["1km"] * 2} + +_test_filenames = {'fdhsi': [ + "W_XX-EUMETSAT-Darmstadt,IMG+SAT,MTI1+FCI-1C-RRAD-FDHSI-FD--" + "CHK-BODY--L2P-NC4E_C_EUMT_20170410114434_GTT_DEV_" + "20170410113925_20170410113934_N__C_0070_0067.nc" +], + 'hrfi': [ + "W_XX-EUMETSAT-Darmstadt,IMG+SAT,MTI1+FCI-1C-RRAD-HRFI-FD--" + "CHK-BODY--L2P-NC4E_C_EUMT_20170410114434_GTT_DEV_" + "20170410113925_20170410113934_N__C_0070_0067.nc" + ] +} + + +@contextlib.contextmanager +def mocked_basefilehandler(filehandler): + """Mock patch the base class of the FCIL1cNCFileHandler with the content of our fake files (filehandler).""" + p = mock.patch.object(FCIL1cNCFileHandler, "__bases__", (filehandler,)) + with p: + p.is_local = True + yield - yaml_file = "fci_l1c_nc.yaml" - _alt_handler = FakeNetCDF4FileHandler2 +@pytest.fixture +def FakeFCIFileHandlerFDHSI_fixture(): + """Get a fixture for the fake FDHSI filehandler, including channel and file names.""" + with mocked_basefilehandler(FakeFCIFileHandlerFDHSI): + param_dict = { + 'filetype': 'fci_l1c_fdhsi', + 'channels': _chans_fdhsi, + 'filenames': _test_filenames['fdhsi'] + } + yield param_dict - @pytest.fixture(autouse=True, scope="class") - def fake_handler(self): - """Wrap NetCDF4 FileHandler with our own fake handler.""" - # implementation strongly inspired by test_viirs_l1b.py - from satpy.readers.fci_l1c_nc import FCIL1cNCFileHandler - p = mock.patch.object( - FCIL1cNCFileHandler, - "__bases__", - (self._alt_handler,)) - with p: - p.is_local = True - yield p +@pytest.fixture +def FakeFCIFileHandlerHRFI_fixture(): + """Get a fixture for the fake HRFI filehandler, including channel and file names.""" + with mocked_basefilehandler(FakeFCIFileHandlerHRFI): + param_dict = { + 'filetype': 'fci_l1c_hrfi', + 'channels': _chans_hrfi, + 'filenames': _test_filenames['hrfi'] + } + yield param_dict -class TestFCIL1cNCReaderGoodData(TestFCIL1cNCReader): - """Test FCI L1c NetCDF reader.""" - _alt_handler = FakeNetCDF4FileHandler2 +# ---------------------------------------------------- +# Tests ---------------------------------------------- +# ---------------------------------------------------- - def test_file_pattern(self, reader_configs): + +class TestFCIL1cNCReader: + """Test FCI L1c NetCDF reader with nominal data.""" + + fh_param_for_filetype = {'hrfi': {'channels': _chans_hrfi, + 'filenames': _test_filenames['hrfi']}, + 'fdhsi': {'channels': _chans_fdhsi, + 'filenames': _test_filenames['fdhsi']}} + + @pytest.mark.parametrize('filenames', [_test_filenames['fdhsi'], _test_filenames['hrfi']]) + def test_file_pattern(self, reader_configs, filenames): """Test file pattern matching.""" from satpy.readers import load_reader - filenames = [ - "W_XX-EUMETSAT-Darmstadt,IMG+SAT,MTI1+FCI-1C-RRAD-FDHSI-FD--" - "CHK-BODY--L2P-NC4E_C_EUMT_20170410114434_GTT_DEV_" - "20170410113925_20170410113934_N__C_0070_0067.nc", - "W_XX-EUMETSAT-Darmstadt,IMG+SAT,MTI1+FCI-1C-RRAD-FDHSI-FD--" - "CHK-BODY--L2P-NC4E_C_EUMT_20170410114442_GTT_DEV_" - "20170410113934_20170410113942_N__C_0070_0068.nc", - "W_XX-EUMETSAT-Darmstadt,IMG+SAT,MTI1+FCI-1C-RRAD-FDHSI-FD--" - "CHK-BODY--L2P-NC4E_C_EUMT_20170410114451_GTT_DEV_" - "20170410113942_20170410113951_N__C_0070_0069.nc", - "W_XX-EUMETSAT-Darmstadt,IMG+SAT,MTI1+FCI-1C-RRAD-FDHSI-FD--" - "CHK-BODY--L2P-NC4E_C_EUMT_20170410114500_GTT_DEV_" - "20170410113951_20170410114000_N__C_0070_0070.nc", - "W_XX-EUMETSAT-Darmstadt,IMG+SAT,MTI1+FCI-1C-RRAD-FDHSI-FD--" - "CHK-TRAIL--L2P-NC4E_C_EUMT_20170410114600_GTT_DEV_" - "20170410113000_20170410114000_N__C_0070_0071.nc", - ] - reader = load_reader(reader_configs) files = reader.select_files_from_pathnames(filenames) - # only 4 out of 5 above should match - assert len(files) == 4 + assert len(files) == 1 + + @pytest.mark.parametrize('filenames', [_test_filenames['fdhsi'][0].replace('BODY', 'TRAIL'), + _test_filenames['hrfi'][0].replace('BODY', 'TRAIL')]) + def test_file_pattern_for_TRAIL_file(self, reader_configs, filenames): + """Test file pattern matching for TRAIL files, which should not be picked up.""" + from satpy.readers import load_reader - _chans = {"solar": ["vis_04", "vis_05", "vis_06", "vis_08", "vis_09", - "nir_13", "nir_16", "nir_22"], - "terran": ["ir_38", "wv_63", "wv_73", "ir_87", "ir_97", "ir_105", - "ir_123", "ir_133"]} + reader = load_reader(reader_configs) + files = reader.select_files_from_pathnames(filenames) + assert len(files) == 0 - def test_load_counts(self, reader_configs): + @pytest.mark.parametrize('fh_param,expected_res_n', [(lazy_fixture('FakeFCIFileHandlerFDHSI_fixture'), 16), + (lazy_fixture('FakeFCIFileHandlerHRFI_fixture'), 4)]) + def test_load_counts(self, reader_configs, fh_param, + expected_res_n): """Test loading with counts.""" - from satpy.tests.utils import make_dataid - - # testing two filenames to test correctly combined - filenames = [ - "W_XX-EUMETSAT-Darmstadt,IMG+SAT,MTI1+FCI-1C-RRAD-FDHSI-FD--" - "CHK-BODY--L2P-NC4E_C_EUMT_20170410114434_GTT_DEV_" - "20170410113925_20170410113934_N__C_0070_0067.nc", - "W_XX-EUMETSAT-Darmstadt,IMG+SAT,MTI1+FCI-1C-RRAD-FDHSI-FD--" - "CHK-BODY--L2P-NC4E_C_EUMT_20170410114442_GTT_DEV_" - "20170410113934_20170410113942_N__C_0070_0068.nc", - ] - reader = _get_reader_with_filehandlers(filenames, reader_configs) + reader = _get_reader_with_filehandlers(fh_param['filenames'], reader_configs) res = reader.load( [make_dataid(name=name, calibration="counts") for name in - self._chans["solar"] + self._chans["terran"]], pad_data=False) - assert 16 == len(res) - for ch in self._chans["solar"] + self._chans["terran"]: - assert res[ch].shape == (200 * 2, 11136) + fh_param['channels']["solar"] + fh_param['channels']["terran"]], pad_data=False) + assert expected_res_n == len(res) + for ch, grid_type in zip(fh_param['channels']["solar"] + fh_param['channels']["terran"], + fh_param['channels']["solar_grid_type"] + + fh_param['channels']["terran_grid_type"]): + assert res[ch].shape == (GRID_TYPE_INFO_FOR_TEST_CONTENT[grid_type]['nrows'], + GRID_TYPE_INFO_FOR_TEST_CONTENT[grid_type]['ncols']) assert res[ch].dtype == np.uint16 assert res[ch].attrs["calibration"] == "counts" assert res[ch].attrs["units"] == "count" if ch == 'ir_38': - numpy.testing.assert_array_equal(res[ch][~0], 1) + numpy.testing.assert_array_equal(res[ch][-1], 1) numpy.testing.assert_array_equal(res[ch][0], 5000) else: numpy.testing.assert_array_equal(res[ch], 1) - def test_load_radiance(self, reader_configs): + @pytest.mark.parametrize('fh_param,expected_res_n', [(lazy_fixture('FakeFCIFileHandlerFDHSI_fixture'), 16), + (lazy_fixture('FakeFCIFileHandlerHRFI_fixture'), 4)]) + def test_load_radiance(self, reader_configs, fh_param, + expected_res_n): """Test loading with radiance.""" - from satpy.tests.utils import make_dataid - - filenames = [ - "W_XX-EUMETSAT-Darmstadt,IMG+SAT,MTI1+FCI-1C-RRAD-FDHSI-FD--" - "CHK-BODY--L2P-NC4E_C_EUMT_20170410114434_GTT_DEV_" - "20170410113925_20170410113934_N__C_0070_0067.nc", - ] - - reader = _get_reader_with_filehandlers(filenames, reader_configs) + reader = _get_reader_with_filehandlers(fh_param['filenames'], reader_configs) res = reader.load( [make_dataid(name=name, calibration="radiance") for name in - self._chans["solar"] + self._chans["terran"]], pad_data=False) - assert 16 == len(res) - for ch in self._chans["solar"] + self._chans["terran"]: - assert res[ch].shape == (200, 11136) + fh_param['channels']["solar"] + fh_param['channels']["terran"]], pad_data=False) + assert expected_res_n == len(res) + for ch, grid_type in zip(fh_param['channels']["solar"] + fh_param['channels']["terran"], + fh_param['channels']["solar_grid_type"] + + fh_param['channels']["terran_grid_type"]): + assert res[ch].shape == (GRID_TYPE_INFO_FOR_TEST_CONTENT[grid_type]['nrows'], + GRID_TYPE_INFO_FOR_TEST_CONTENT[grid_type]['ncols']) assert res[ch].dtype == np.float64 assert res[ch].attrs["calibration"] == "radiance" assert res[ch].attrs["units"] == 'mW m-2 sr-1 (cm-1)-1' assert res[ch].attrs["radiance_unit_conversion_coefficient"] == 1234.56 if ch == 'ir_38': - numpy.testing.assert_array_equal(res[ch][~0], 15) + numpy.testing.assert_array_equal(res[ch][-1], 15) numpy.testing.assert_array_equal(res[ch][0], 9700) else: numpy.testing.assert_array_equal(res[ch], 15) - def test_load_reflectance(self, reader_configs): + @pytest.mark.parametrize('fh_param,expected_res_n', [(lazy_fixture('FakeFCIFileHandlerFDHSI_fixture'), 8), + (lazy_fixture('FakeFCIFileHandlerHRFI_fixture'), 2)]) + def test_load_reflectance(self, reader_configs, fh_param, + expected_res_n): """Test loading with reflectance.""" - from satpy.tests.utils import make_dataid - - filenames = [ - "W_XX-EUMETSAT-Darmstadt,IMG+SAT,MTI1+FCI-1C-RRAD-FDHSI-FD--" - "CHK-BODY--L2P-NC4E_C_EUMT_20170410114434_GTT_DEV_" - "20170410113925_20170410113934_N__C_0070_0067.nc", - ] - - reader = _get_reader_with_filehandlers(filenames, reader_configs) + reader = _get_reader_with_filehandlers(fh_param['filenames'], reader_configs) res = reader.load( [make_dataid(name=name, calibration="reflectance") for name in - self._chans["solar"]], pad_data=False) - assert 8 == len(res) - for ch in self._chans["solar"]: - assert res[ch].shape == (200, 11136) + fh_param['channels']["solar"]], pad_data=False) + assert expected_res_n == len(res) + for ch, grid_type in zip(fh_param['channels']["solar"], fh_param['channels']["solar_grid_type"]): + assert res[ch].shape == (GRID_TYPE_INFO_FOR_TEST_CONTENT[grid_type]['nrows'], + GRID_TYPE_INFO_FOR_TEST_CONTENT[grid_type]['ncols']) assert res[ch].dtype == np.float64 assert res[ch].attrs["calibration"] == "reflectance" assert res[ch].attrs["units"] == "%" numpy.testing.assert_array_almost_equal(res[ch], 100 * 15 * 1 * np.pi / 50) - def test_load_bt(self, reader_configs, caplog): + @pytest.mark.parametrize('fh_param,expected_res_n', [(lazy_fixture('FakeFCIFileHandlerFDHSI_fixture'), 8), + (lazy_fixture('FakeFCIFileHandlerHRFI_fixture'), 2)]) + def test_load_bt(self, reader_configs, caplog, fh_param, + expected_res_n): """Test loading with bt.""" - from satpy.tests.utils import make_dataid - - filenames = [ - "W_XX-EUMETSAT-Darmstadt,IMG+SAT,MTI1+FCI-1C-RRAD-FDHSI-FD--" - "CHK-BODY--L2P-NC4E_C_EUMT_20170410114434_GTT_DEV_" - "20170410113925_20170410113934_N__C_0070_0067.nc", - ] - - reader = _get_reader_with_filehandlers(filenames, reader_configs) + reader = _get_reader_with_filehandlers(fh_param['filenames'], reader_configs) with caplog.at_level(logging.WARNING): res = reader.load( [make_dataid(name=name, calibration="brightness_temperature") for - name in self._chans["terran"]], pad_data=False) + name in fh_param['channels']["terran"]], pad_data=False) assert caplog.text == "" - for ch in self._chans["terran"]: - assert res[ch].shape == (200, 11136) + assert expected_res_n == len(res) + for ch, grid_type in zip(fh_param['channels']["terran"], fh_param['channels']["terran_grid_type"]): + assert res[ch].shape == (GRID_TYPE_INFO_FOR_TEST_CONTENT[grid_type]['nrows'], + GRID_TYPE_INFO_FOR_TEST_CONTENT[grid_type]['ncols']) assert res[ch].dtype == np.float64 assert res[ch].attrs["calibration"] == "brightness_temperature" assert res[ch].attrs["units"] == "K" if ch == 'ir_38': - numpy.testing.assert_array_almost_equal(res[ch][~0], 209.68274099) + numpy.testing.assert_array_almost_equal(res[ch][-1], 209.68274099) numpy.testing.assert_array_almost_equal(res[ch][0], 1888.851296) else: numpy.testing.assert_array_almost_equal(res[ch], 209.68274099) - def test_orbital_parameters_attr(self, reader_configs): + @pytest.mark.parametrize('fh_param', [(lazy_fixture('FakeFCIFileHandlerFDHSI_fixture')), + (lazy_fixture('FakeFCIFileHandlerHRFI_fixture'))]) + def test_orbital_parameters_attr(self, reader_configs, fh_param): """Test the orbital parameter attribute.""" - from satpy.tests.utils import make_dataid - - filenames = [ - "W_XX-EUMETSAT-Darmstadt,IMG+SAT,MTI1+FCI-1C-RRAD-FDHSI-FD--" - "CHK-BODY--L2P-NC4E_C_EUMT_20170410114434_GTT_DEV_" - "20170410113925_20170410113934_N__C_0070_0067.nc", - ] - - reader = _get_reader_with_filehandlers(filenames, reader_configs) + reader = _get_reader_with_filehandlers(fh_param['filenames'], reader_configs) res = reader.load( [make_dataid(name=name) for name in - self._chans["solar"] + self._chans["terran"]], pad_data=False) + fh_param['channels']["solar"] + fh_param['channels']["terran"]], pad_data=False) - for ch in self._chans["solar"] + self._chans["terran"]: + for ch in fh_param['channels']["solar"] + fh_param['channels']["terran"]: assert res[ch].attrs["orbital_parameters"] == { 'satellite_actual_longitude': np.mean(np.arange(6000)), 'satellite_actual_latitude': np.mean(np.arange(6000)), @@ -477,127 +545,117 @@ def test_orbital_parameters_attr(self, reader_configs): 'projection_altitude': 35786400.0, } - def test_load_index_map(self, reader_configs): + expected_pos_info_for_filetype = { + 'fdhsi': {'1km': {'start_position_row': 1, + 'end_position_row': 200, + 'segment_height': 200, + 'grid_width': 11136}, + '2km': {'start_position_row': 1, + 'end_position_row': 100, + 'segment_height': 100, + 'grid_width': 5568}}, + 'hrfi': {'500m': {'start_position_row': 1, + 'end_position_row': 400, + 'segment_height': 400, + 'grid_width': 22272}, + '1km': {'start_position_row': 1, + 'end_position_row': 200, + 'grid_width': 11136, + 'segment_height': 200}} + } + + @pytest.mark.parametrize('fh_param, expected_pos_info', [ + (lazy_fixture('FakeFCIFileHandlerFDHSI_fixture'), expected_pos_info_for_filetype['fdhsi']), + (lazy_fixture('FakeFCIFileHandlerHRFI_fixture'), expected_pos_info_for_filetype['hrfi']) + ]) + def test_get_segment_position_info(self, reader_configs, fh_param, expected_pos_info): + """Test the segment position info method.""" + reader = _get_reader_with_filehandlers(fh_param['filenames'], reader_configs) + for filetype_handler in list(reader.file_handlers.values())[0]: + segpos_info = filetype_handler.get_segment_position_info() + assert segpos_info == expected_pos_info + + @pytest.mark.parametrize('fh_param,expected_res_n', [(lazy_fixture('FakeFCIFileHandlerFDHSI_fixture'), 16), + (lazy_fixture('FakeFCIFileHandlerHRFI_fixture'), 4)]) + def test_load_index_map(self, reader_configs, fh_param, expected_res_n): """Test loading of index_map.""" - filenames = [ - "W_XX-EUMETSAT-Darmstadt,IMG+SAT,MTI1+FCI-1C-RRAD-FDHSI-FD--" - "CHK-BODY--L2P-NC4E_C_EUMT_20170410114434_GTT_DEV_" - "20170410113925_20170410113934_N__C_0070_0067.nc" - ] - - reader = _get_reader_with_filehandlers(filenames, reader_configs) + reader = _get_reader_with_filehandlers(fh_param['filenames'], reader_configs) res = reader.load( [name + '_index_map' for name in - self._chans["solar"] + self._chans["terran"]], pad_data=False) - assert 16 == len(res) - for ch in self._chans["solar"] + self._chans["terran"]: - assert res[ch + '_index_map'].shape == (200, 11136) - numpy.testing.assert_array_equal(res[ch + '_index_map'][1, 1], 5237) - - def test_load_aux_data(self, reader_configs): + fh_param['channels']["solar"] + fh_param['channels']["terran"]], pad_data=False) + assert expected_res_n == len(res) + for ch, grid_type in zip(fh_param['channels']["solar"] + fh_param['channels']["terran"], + fh_param['channels']["solar_grid_type"] + + fh_param['channels']["terran_grid_type"]): + assert res[ch + '_index_map'].shape == (GRID_TYPE_INFO_FOR_TEST_CONTENT[grid_type]['nrows'], + GRID_TYPE_INFO_FOR_TEST_CONTENT[grid_type]['ncols']) + numpy.testing.assert_array_equal(res[ch + '_index_map'][1, 1], 110) + + @pytest.mark.parametrize('fh_param', [(lazy_fixture('FakeFCIFileHandlerFDHSI_fixture')), + (lazy_fixture('FakeFCIFileHandlerHRFI_fixture'))]) + def test_load_aux_data(self, reader_configs, fh_param): """Test loading of auxiliary data.""" from satpy.readers.fci_l1c_nc import AUX_DATA - - filenames = [ - "W_XX-EUMETSAT-Darmstadt,IMG+SAT,MTI1+FCI-1C-RRAD-FDHSI-FD--" - "CHK-BODY--L2P-NC4E_C_EUMT_20170410114434_GTT_DEV_" - "20170410113925_20170410113934_N__C_0070_0067.nc" - ] - - reader = _get_reader_with_filehandlers(filenames, reader_configs) - res = reader.load(['vis_04_' + key for key in AUX_DATA.keys()], + reader = _get_reader_with_filehandlers(fh_param['filenames'], reader_configs) + res = reader.load([fh_param['channels']['solar'][0] + '_' + key for key in AUX_DATA.keys()], pad_data=False) - for aux in ['vis_04_' + key for key in AUX_DATA.keys()]: - - assert res[aux].shape == (200, 11136) - if aux == 'vis_04_earth_sun_distance': + grid_type = fh_param['channels']['solar_grid_type'][0] + for aux in [fh_param['channels']['solar'][0] + '_' + key for key in AUX_DATA.keys()]: + assert res[aux].shape == (GRID_TYPE_INFO_FOR_TEST_CONTENT[grid_type]['nrows'], + GRID_TYPE_INFO_FOR_TEST_CONTENT[grid_type]['ncols']) + if aux == fh_param['channels']['solar'][0] + '_earth_sun_distance': numpy.testing.assert_array_equal(res[aux][1, 1], 149597870.7) else: - numpy.testing.assert_array_equal(res[aux][1, 1], 5137) - - def test_load_composite(self): - """Test that composites are loadable.""" - # when dedicated composites for FCI are implemented in satpy, - # this method should probably move to a dedicated class and module - # in the tests.compositor_tests package - - from satpy.composites.config_loader import load_compositor_configs_for_sensors - comps, mods = load_compositor_configs_for_sensors(['fci']) - assert len(comps["fci"]) > 0 - assert len(mods["fci"]) > 0 + numpy.testing.assert_array_equal(res[aux][1, 1], 10) - def test_load_quality_only(self, reader_configs): + @pytest.mark.parametrize('fh_param,expected_res_n', [(lazy_fixture('FakeFCIFileHandlerFDHSI_fixture'), 16), + (lazy_fixture('FakeFCIFileHandlerHRFI_fixture'), 4)]) + def test_load_quality_only(self, reader_configs, fh_param, expected_res_n): """Test that loading quality only works.""" - filenames = [ - "W_XX-EUMETSAT-Darmstadt,IMG+SAT,MTI1+FCI-1C-RRAD-FDHSI-FD--" - "CHK-BODY--L2P-NC4E_C_EUMT_20170410114434_GTT_DEV_" - "20170410113925_20170410113934_N__C_0070_0067.nc", - ] - - reader = _get_reader_with_filehandlers(filenames, reader_configs) + reader = _get_reader_with_filehandlers(fh_param['filenames'], reader_configs) res = reader.load( [name + '_pixel_quality' for name in - self._chans["solar"] + self._chans["terran"]], pad_data=False) - assert 16 == len(res) - for ch in self._chans["solar"] + self._chans["terran"]: - assert res[ch + '_pixel_quality'].shape == (200, 11136) - numpy.testing.assert_array_equal(res[ch + '_pixel_quality'][1, 1], 1) + fh_param['channels']["solar"] + fh_param['channels']["terran"]], pad_data=False) + assert expected_res_n == len(res) + for ch, grid_type in zip(fh_param['channels']["solar"] + fh_param['channels']["terran"], + fh_param['channels']["solar_grid_type"] + + fh_param['channels']["terran_grid_type"]): + assert res[ch + '_pixel_quality'].shape == (GRID_TYPE_INFO_FOR_TEST_CONTENT[grid_type]['nrows'], + GRID_TYPE_INFO_FOR_TEST_CONTENT[grid_type]['ncols']) + numpy.testing.assert_array_equal(res[ch + '_pixel_quality'][1, 1], 3) assert res[ch + '_pixel_quality'].attrs["name"] == ch + '_pixel_quality' - def test_platform_name(self, reader_configs): + @pytest.mark.parametrize('fh_param', [(lazy_fixture('FakeFCIFileHandlerFDHSI_fixture')), + (lazy_fixture('FakeFCIFileHandlerHRFI_fixture'))]) + def test_platform_name(self, reader_configs, fh_param): """Test that platform name is exposed. Test that the FCI reader exposes the platform name. Corresponds to GH issue 1014. """ - filenames = [ - "W_XX-EUMETSAT-Darmstadt,IMG+SAT,MTI1+FCI-1C-RRAD-FDHSI-FD--" - "CHK-BODY--L2P-NC4E_C_EUMT_20170410114434_GTT_DEV_" - "20170410113925_20170410113934_N__C_0070_0067.nc", - ] - - reader = _get_reader_with_filehandlers(filenames, reader_configs) - res = reader.load(["ir_123"], pad_data=False) - assert res["ir_123"].attrs["platform_name"] == "MTG-I1" - - def test_excs(self, reader_configs): - """Test that exceptions are raised where expected.""" - from satpy.tests.utils import make_dataid - filenames = [ - "W_XX-EUMETSAT-Darmstadt,IMG+SAT,MTI1+FCI-1C-RRAD-FDHSI-FD--" - "CHK-BODY--L2P-NC4E_C_EUMT_20170410114434_GTT_DEV_" - "20170410113925_20170410113934_N__C_0070_0067.nc", - ] - - reader = _get_reader_with_filehandlers(filenames, reader_configs) - - with pytest.raises(ValueError): - reader.file_handlers["fci_l1c_fdhsi"][0].get_dataset(make_dataid(name="invalid"), {}) - with pytest.raises(ValueError): - reader.file_handlers["fci_l1c_fdhsi"][0].get_dataset( - make_dataid(name="ir_123", calibration="unknown"), - {"units": "unknown"}) - - def test_area_definition_computation(self, reader_configs): + reader = _get_reader_with_filehandlers(fh_param['filenames'], reader_configs) + res = reader.load(["vis_06"], pad_data=False) + assert res["vis_06"].attrs["platform_name"] == "MTG-I1" + + @pytest.mark.parametrize('fh_param, expected_area', [ + (lazy_fixture('FakeFCIFileHandlerFDHSI_fixture'), ['mtg_fci_fdss_1km', 'mtg_fci_fdss_2km']), + (lazy_fixture('FakeFCIFileHandlerHRFI_fixture'), ['mtg_fci_fdss_500m', 'mtg_fci_fdss_1km']), + ]) + def test_area_definition_computation(self, reader_configs, fh_param, expected_area): """Test that the geolocation computation is correct.""" - filenames = [ - "W_XX-EUMETSAT-Darmstadt,IMG+SAT,MTI1+FCI-1C-RRAD-FDHSI-FD--" - "CHK-BODY--L2P-NC4E_C_EUMT_20170410114434_GTT_DEV_" - "20170410113925_20170410113934_N__C_0070_0067.nc", - ] - - reader = _get_reader_with_filehandlers(filenames, reader_configs) + reader = _get_reader_with_filehandlers(fh_param['filenames'], reader_configs) res = reader.load(['ir_105', 'vis_06'], pad_data=False) # test that area_ids are harmonisation-conform ___ - assert res['vis_06'].attrs['area'].area_id == 'mtg_fci_fdss_1km' - assert res['ir_105'].attrs['area'].area_id == 'mtg_fci_fdss_2km' + assert res['vis_06'].attrs['area'].area_id == expected_area[0] + assert res['ir_105'].attrs['area'].area_id == expected_area[1] area_def = res['ir_105'].attrs['area'] # test area extents computation np.testing.assert_array_almost_equal(np.array(area_def.area_extent), - np.array([-5568062.23065902, 5168057.7600648, - 16704186.692027, 5568062.23065902])) + np.array([-5567999.994203, -5367999.994411, + 5567999.994203, -5567999.994203]), + decimal=2) # check that the projection is read in properly assert area_def.crs.coordinate_operation.method_name == 'Geostationary Satellite (Sweep Y)' @@ -607,80 +665,75 @@ def test_area_definition_computation(self, reader_configs): assert area_def.crs.ellipsoid.inverse_flattening == 298.257223563 assert area_def.crs.ellipsoid.is_semi_minor_computed + @pytest.mark.parametrize('fh_param', [(lazy_fixture('FakeFCIFileHandlerFDHSI_fixture')), + (lazy_fixture('FakeFCIFileHandlerHRFI_fixture'))]) + def test_excs(self, reader_configs, fh_param): + """Test that exceptions are raised where expected.""" + reader = _get_reader_with_filehandlers(fh_param['filenames'], reader_configs) -class TestFCIL1cNCReaderBadData(TestFCIL1cNCReader): - """Test the FCI L1c NetCDF Reader for bad data input.""" + with pytest.raises(ValueError): + reader.file_handlers[fh_param['filetype']][0].get_dataset(make_dataid(name="invalid"), {}) + with pytest.raises(ValueError): + reader.file_handlers[fh_param['filetype']][0].get_dataset( + make_dataid(name="ir_123", calibration="unknown"), + {"units": "unknown"}) - _alt_handler = FakeNetCDF4FileHandler3 + def test_load_composite(self): + """Test that composites are loadable.""" + # when dedicated composites for FCI are implemented in satpy, + # this method should probably move to a dedicated class and module + # in the tests.compositor_tests package - def test_handling_bad_data_ir(self, reader_configs, caplog): - """Test handling of bad IR data.""" - from satpy.tests.utils import make_dataid + from satpy.composites.config_loader import load_compositor_configs_for_sensors + comps, mods = load_compositor_configs_for_sensors(['fci']) + assert len(comps["fci"]) > 0 + assert len(mods["fci"]) > 0 - filenames = [ - "W_XX-EUMETSAT-Darmstadt,IMG+SAT,MTI1+FCI-1C-RRAD-FDHSI-FD--" - "CHK-BODY--L2P-NC4E_C_EUMT_20170410114434_GTT_DEV_" - "20170410113925_20170410113934_N__C_0070_0067.nc", - ] - reader = _get_reader_with_filehandlers(filenames, reader_configs) - with caplog.at_level("ERROR"): - reader.load([make_dataid( - name="ir_123", - calibration="brightness_temperature")], pad_data=False) - assert "cannot produce brightness temperature" in caplog.text +class TestFCIL1cNCReaderBadData: + """Test the FCI L1c NetCDF Reader for bad data input.""" + + def test_handling_bad_data_ir(self, reader_configs, caplog): + """Test handling of bad IR data.""" + with mocked_basefilehandler(FakeFCIFileHandlerWithBadData): + reader = _get_reader_with_filehandlers(_test_filenames['fdhsi'], reader_configs) + with caplog.at_level("ERROR"): + reader.load([make_dataid( + name="ir_105", + calibration="brightness_temperature")], pad_data=False) + assert "cannot produce brightness temperature" in caplog.text def test_handling_bad_data_vis(self, reader_configs, caplog): """Test handling of bad VIS data.""" - from satpy.tests.utils import make_dataid - - filenames = [ - "W_XX-EUMETSAT-Darmstadt,IMG+SAT,MTI1+FCI-1C-RRAD-FDHSI-FD--" - "CHK-BODY--L2P-NC4E_C_EUMT_20170410114434_GTT_DEV_" - "20170410113925_20170410113934_N__C_0070_0067.nc", - ] + with mocked_basefilehandler(FakeFCIFileHandlerWithBadData): + reader = _get_reader_with_filehandlers(_test_filenames['fdhsi'], reader_configs) + with caplog.at_level("ERROR"): + reader.load([make_dataid( + name="vis_06", + calibration="reflectance")], pad_data=False) + assert "cannot produce reflectance" in caplog.text - reader = _get_reader_with_filehandlers(filenames, reader_configs) - with caplog.at_level("ERROR"): - reader.load([make_dataid( - name="vis_04", - calibration="reflectance")], pad_data=False) - assert "cannot produce reflectance" in caplog.text - -class TestFCIL1cNCReaderBadDataFromIDPF(TestFCIL1cNCReader): - """Test the FCI L1c NetCDF Reader for bad data input.""" - - _alt_handler = FakeNetCDF4FileHandler4 +class TestFCIL1cNCReaderBadDataFromIDPF: + """Test the FCI L1c NetCDF Reader for bad data input, specifically the IDPF issues.""" def test_handling_bad_earthsun_distance(self, reader_configs, caplog): """Test handling of bad earth-sun distance data.""" - from satpy.tests.utils import make_dataid - - filenames = [ - "W_XX-EUMETSAT-Darmstadt,IMG+SAT,MTI1+FCI-1C-RRAD-FDHSI-FD--" - "CHK-BODY--L2P-NC4E_C_EUMT_20170410114434_GTT_DEV_" - "20170410113925_20170410113934_N__C_0070_0067.nc", - ] - - reader = _get_reader_with_filehandlers(filenames, reader_configs) + with mocked_basefilehandler(FakeFCIFileHandlerWithBadIDPFData): + reader = _get_reader_with_filehandlers(_test_filenames['fdhsi'], reader_configs) + res = reader.load([make_dataid(name=["vis_06"], calibration="reflectance")], pad_data=False) - res = reader.load([make_dataid(name=["vis_04"], calibration="reflectance")], pad_data=False) - numpy.testing.assert_array_almost_equal(res["vis_04"], 100 * 15 * 1 * np.pi / 50) + numpy.testing.assert_array_almost_equal(res["vis_06"], 100 * 15 * 1 * np.pi / 50) def test_bad_xy_coords(self, reader_configs): """Test that the geolocation computation is correct.""" - filenames = [ - "W_XX-EUMETSAT-Darmstadt,IMG+SAT,MTI1+FCI-1C-RRAD-FDHSI-FD--" - "CHK-BODY--L2P-NC4E_C_EUMT_20170410114434_GTT_DEV_" - "20170410113925_20170410113934_N__C_0070_0067.nc", - ] - - reader = _get_reader_with_filehandlers(filenames, reader_configs) - res = reader.load(['vis_04'], pad_data=False) - - area_def = res['vis_04'].attrs['area'] - # test area extents computation - np.testing.assert_array_almost_equal(np.array(area_def.area_extent), - np.array([-5568062.270889, 5168057.806632, - 16704186.298937, 5568062.270889])) + with mocked_basefilehandler(FakeFCIFileHandlerWithBadIDPFData): + reader = _get_reader_with_filehandlers(_test_filenames['fdhsi'], reader_configs) + res = reader.load(['vis_06'], pad_data=False) + + area_def = res['vis_06'].attrs['area'] + # test area extents computation + np.testing.assert_array_almost_equal(np.array(area_def.area_extent), + np.array([-5568000.227139, -5368000.221262, + 5568000.100073, -5568000.227139]), + decimal=2) diff --git a/satpy/tests/test_yaml_reader.py b/satpy/tests/test_yaml_reader.py index 89416ab76d..0d2e057f32 100644 --- a/satpy/tests/test_yaml_reader.py +++ b/satpy/tests/test_yaml_reader.py @@ -25,9 +25,11 @@ from unittest.mock import MagicMock, call, patch import numpy as np +import pytest import xarray as xr import satpy.readers.yaml_reader as yr +from satpy._compat import cache from satpy.dataset import DataQuery from satpy.dataset.dataid import ModifierTuple from satpy.readers.file_handlers import BaseFileHandler @@ -41,7 +43,7 @@ 'default_channels': [1, 2, 3, 4, 5], 'data_identification_keys': {'name': {'required': True}, 'frequency_double_sideband': - {'type': FrequencyDoubleSideBand}, + {'type': FrequencyDoubleSideBand}, 'frequency_range': {'type': FrequencyRange}, 'resolution': None, 'polarization': {'enum': ['H', 'V']}, @@ -689,7 +691,7 @@ def test_update_ds_ids_from_file_handlers(self): # need to copy this because the dataset infos will be modified _orig_ids = {key: val.copy() for key, val in orig_ids.items()} with patch.dict(self.reader.all_ids, _orig_ids, clear=True), \ - patch.dict(self.reader.available_ids, {}, clear=True): + patch.dict(self.reader.available_ids, {}, clear=True): # Add a file handler with resolution property fh = MagicMock(filetype_info={'file_type': ftype}, resolution=resol) @@ -1037,7 +1039,7 @@ def test_get_expected_segments(self, cfh): es = created_fhs['ft1'][0].filetype_info['expected_segments'] self.assertEqual(es, 3) - # check correct FCI chunk number reading into segment + # check correct FCI segment (aka chunk in the FCI world) number reading into segment fake_fh.filename_info = {'count_in_repeat_cycle': 5} created_fhs = reader.create_filehandlers(['fake.nc']) es = created_fhs['ft1'][0].filename_info['segment'] @@ -1245,36 +1247,68 @@ def test_find_missing_segments(self): self.assertTrue(proj is projectable) -class TestGEOVariableSegmentYAMLReader(unittest.TestCase): +@pytest.fixture +@patch.object(yr.GEOVariableSegmentYAMLReader, "__init__", lambda x: None) +def GVSYReader(): + """Get a fixture of the GEOVariableSegmentYAMLReader.""" + from satpy.readers.yaml_reader import GEOVariableSegmentYAMLReader + reader = GEOVariableSegmentYAMLReader() + reader.segment_infos = dict() + reader.segment_heights = cache(reader._segment_heights) + return reader + + +@pytest.fixture +def fake_geswh(): + """Get a fixture of the patched _get_empty_segment_with_height.""" + with patch('satpy.readers.yaml_reader._get_empty_segment_with_height') as geswh: + yield geswh + + +@pytest.fixture +def fake_xr(): + """Get a fixture of the patched xarray.""" + with patch('satpy.readers.yaml_reader.xr') as xr: + yield xr + + +@pytest.fixture +def fake_mss(): + """Get a fixture of the patched _find_missing_segments.""" + with patch('satpy.readers.yaml_reader._find_missing_segments') as mss: + yield mss + + +@pytest.fixture +def fake_adef(): + """Get a fixture of the patched AreaDefinition.""" + with patch('satpy.readers.yaml_reader.AreaDefinition') as adef: + yield adef + + +class TestGEOVariableSegmentYAMLReader: """Test GEOVariableSegmentYAMLReader.""" - @patch.object(yr.FileYAMLReader, "__init__", lambda x: None) - @patch('satpy.readers.yaml_reader._get_empty_segment_with_height') - @patch('satpy.readers.yaml_reader.xr') - @patch('satpy.readers.yaml_reader._find_missing_segments') - def test_get_empty_segment(self, mss, xr, geswh): + def test_get_empty_segment(self, GVSYReader, fake_mss, fake_xr, fake_geswh): """Test execution of (overridden) get_empty_segment inside _load_dataset.""" - from satpy.readers.yaml_reader import GEOVariableSegmentYAMLReader - reader = GEOVariableSegmentYAMLReader() # Setup input, and output of mocked functions for first segment missing chk_pos_info = { '1km': {'start_position_row': 0, 'end_position_row': 0, 'segment_height': 0, - 'segment_width': 11136}, + 'grid_width': 11136}, '2km': {'start_position_row': 140, 'end_position_row': None, 'segment_height': 278, - 'segment_width': 5568} + 'grid_width': 5568} } expected_segments = 2 segment = 2 aex = [0, 1000, 200, 500] ashape = [278, 5568] - fh_2, seg2_area = _create_mocked_fh_and_areadef(aex, ashape, expected_segments, segment, chk_pos_info) + fh_2, _ = _create_mocked_fh_and_areadef(aex, ashape, expected_segments, segment, chk_pos_info) - file_handlers = {'filetype1': [fh_2]} - reader._extract_segment_location_dicts(file_handlers) + GVSYReader.file_handlers = {'filetype1': [fh_2]} counter = 2 seg = MagicMock(dims=['y', 'x']) @@ -1283,32 +1317,28 @@ def test_get_empty_segment(self, mss, xr, geswh): projectable = MagicMock() empty_segment = MagicMock() empty_segment.shape = [278, 5568] - xr.full_like.return_value = empty_segment + fake_xr.full_like.return_value = empty_segment dataid = MagicMock() ds_info = MagicMock() - mss.return_value = (counter, expected_segments, slice_list, - failure, projectable) - reader._load_dataset(dataid, ds_info, [fh_2]) + fake_mss.return_value = (counter, expected_segments, slice_list, + failure, projectable) + GVSYReader._load_dataset(dataid, ds_info, [fh_2]) # the return of get_empty_segment - geswh.assert_called_once_with(empty_segment, 139, dim='y') + fake_geswh.assert_called_once_with(empty_segment, 139, dim='y') - @patch.object(yr.FileYAMLReader, "__init__", lambda x: None) - @patch('satpy.readers.yaml_reader.AreaDefinition') - def test_pad_earlier_segments_area(self, AreaDefinition): + def test_pad_earlier_segments_area(self, GVSYReader, fake_adef): """Test _pad_earlier_segments_area() for the variable segment case.""" - from satpy.readers.yaml_reader import GEOVariableSegmentYAMLReader - reader = GEOVariableSegmentYAMLReader() # setting to 0 or None values that shouldn't be relevant chk_pos_info = { '1km': {'start_position_row': 0, 'end_position_row': 0, 'segment_height': 0, - 'segment_width': 11136}, + 'grid_width': 11136}, '2km': {'start_position_row': 140, 'end_position_row': None, 'segment_height': 278, - 'segment_width': 5568} + 'grid_width': 5568} } expected_segments = 2 segment = 2 @@ -1316,66 +1346,55 @@ def test_pad_earlier_segments_area(self, AreaDefinition): ashape = [278, 5568] fh_2, seg2_area = _create_mocked_fh_and_areadef(aex, ashape, expected_segments, segment, chk_pos_info) - file_handlers = {'filetype1': [fh_2]} - reader._extract_segment_location_dicts(file_handlers) + GVSYReader.file_handlers = {'filetype1': [fh_2]} dataid = 'dataid' area_defs = {2: seg2_area} - res = reader._pad_earlier_segments_area([fh_2], dataid, area_defs) - self.assertEqual(len(res), 2) + res = GVSYReader._pad_earlier_segments_area([fh_2], dataid, area_defs) + assert len(res) == 2 - # The later vertical chunk (nr. 2) size is 278, which is exactly double the size - # of the gap left by the missing first chunk (139, as the second chunk starts at line 140). - # Therefore, the new vertical area extent for the first chunk should be + # The later vertical segment (nr. 2) size is 278, which is exactly double the size + # of the gap left by the missing first segment (139, as the second segment starts at line 140). + # Therefore, the new vertical area extent for the first segment should be # half of the previous size (1000-500)/2=250. # The new area extent lower-left row is therefore 500-250=250 seg1_extent = (0, 500, 200, 250) expected_call = ('fill', 'fill', 'fill', 'some_crs', 5568, 139, seg1_extent) - AreaDefinition.assert_called_once_with(*expected_call) + fake_adef.assert_called_once_with(*expected_call) - @patch.object(yr.FileYAMLReader, "__init__", lambda x: None) - @patch('satpy.readers.yaml_reader.AreaDefinition') - def test_pad_later_segments_area(self, AreaDefinition): + def test_pad_later_segments_area(self, GVSYReader, fake_adef): """Test _pad_later_segments_area() in the variable padding case.""" - from satpy.readers.yaml_reader import GEOVariableSegmentYAMLReader - reader = GEOVariableSegmentYAMLReader() - chk_pos_info = { '1km': {'start_position_row': None, 'end_position_row': 11136 - 278, 'segment_height': 556, - 'segment_width': 11136}, + 'grid_width': 11136}, '2km': {'start_position_row': 0, 'end_position_row': 0, 'segment_height': 0, - 'segment_width': 5568}} + 'grid_width': 5568}} expected_segments = 2 segment = 1 aex = [0, 1000, 200, 500] ashape = [556, 11136] fh_1, _ = _create_mocked_fh_and_areadef(aex, ashape, expected_segments, segment, chk_pos_info) - file_handlers = {'filetype1': [fh_1]} - reader._extract_segment_location_dicts(file_handlers) + GVSYReader.file_handlers = {'filetype1': [fh_1]} dataid = 'dataid' - res = reader._pad_later_segments_area([fh_1], dataid) - self.assertEqual(len(res), 2) + res = GVSYReader._pad_later_segments_area([fh_1], dataid) + assert len(res) == 2 - # The previous chunk size is 556, which is exactly double the size of the gap left - # by the missing last chunk (278, as the second-to-last chunk ends at line 11136 - 278 ) + # The previous segment size is 556, which is exactly double the size of the gap left + # by the missing last segment (278, as the second-to-last segment ends at line 11136 - 278 ) # therefore, the new vertical area extent should be half of the previous size (1000-500)/2=250. # The new area extent lower-left row is therefore 1000+250=1250 seg2_extent = (0, 1250, 200, 1000) expected_call = ('fill', 'fill', 'fill', 'some_crs', 11136, 278, seg2_extent) - AreaDefinition.assert_called_once_with(*expected_call) + fake_adef.assert_called_once_with(*expected_call) - @patch.object(yr.FileYAMLReader, "__init__", lambda x: None) - @patch('satpy.readers.yaml_reader.AreaDefinition') - def test_pad_later_segments_area_for_multiple_chunks_gap(self, AreaDefinition): - """Test _pad_later_segments_area() in the variable padding case for multiple gaps with multiple chunks.""" - from satpy.readers.yaml_reader import GEOVariableSegmentYAMLReader - reader = GEOVariableSegmentYAMLReader() + def test_pad_later_segments_area_for_multiple_segments_gap(self, GVSYReader, fake_adef): + """Test _pad_later_segments_area() in the variable padding case for multiple gaps with multiple segments.""" def side_effect_areadef(a, b, c, crs, width, height, aex): m = MagicMock() @@ -1384,17 +1403,17 @@ def side_effect_areadef(a, b, c, crs, width, height, aex): m.crs = crs return m - AreaDefinition.side_effect = side_effect_areadef + fake_adef.side_effect = side_effect_areadef chk_pos_info = { '1km': {'start_position_row': 11136 - 600 - 100 + 1, 'end_position_row': 11136 - 600, 'segment_height': 100, - 'segment_width': 11136}, + 'grid_width': 11136}, '2km': {'start_position_row': 0, 'end_position_row': 0, 'segment_height': 0, - 'segment_width': 5568}} + 'grid_width': 5568}} expected_segments = 8 segment = 1 aex = [0, 1000, 200, 500] @@ -1404,11 +1423,11 @@ def side_effect_areadef(a, b, c, crs, width, height, aex): '1km': {'start_position_row': 11136 - 300 - 100 + 1, 'end_position_row': 11136 - 300, 'segment_height': 100, - 'segment_width': 11136}, + 'grid_width': 11136}, '2km': {'start_position_row': 0, 'end_position_row': 0, 'segment_height': 0, - 'segment_width': 5568}} + 'grid_width': 5568}} segment = 4 fh_4, _ = _create_mocked_fh_and_areadef(aex, ashape, expected_segments, segment, chk_pos_info) @@ -1416,46 +1435,44 @@ def side_effect_areadef(a, b, c, crs, width, height, aex): '1km': {'start_position_row': 11136 - 100 + 1, 'end_position_row': None, 'segment_height': 100, - 'segment_width': 11136}, + 'grid_width': 11136}, '2km': {'start_position_row': 0, 'end_position_row': 0, 'segment_height': 0, - 'segment_width': 5568}} + 'grid_width': 5568}} segment = 8 fh_8, _ = _create_mocked_fh_and_areadef(aex, ashape, expected_segments, segment, chk_pos_info) - file_handlers = {'filetype1': [fh_1, fh_4, fh_8]} - - reader._extract_segment_location_dicts(file_handlers) + GVSYReader.file_handlers = {'filetype1': [fh_1, fh_4, fh_8]} dataid = 'dataid' - res = reader._pad_later_segments_area([fh_1, fh_4, fh_8], dataid) - self.assertEqual(len(res), 8) + res = GVSYReader._pad_later_segments_area([fh_1, fh_4, fh_8], dataid) + assert len(res) == 8 - # Regarding the chunk sizes: - # First group of missing chunks: - # The end position row of the gap is the start row of the last available chunk-1:11136-300-100+1-1=10736 - # The start position row of the gap is the end row fo the first available chunk+1: 11136-600+1=10837 + # Regarding the segment sizes: + # First group of missing segments: + # The end position row of the gap is the start row of the last available segment-1:11136-300-100+1-1=10736 + # The start position row of the gap is the end row fo the first available segment+1: 11136-600+1=10837 # hence the gap is 10736-10537+1=200 px high - # The 200px have to be split between two missing chunks, the most equal way to do it is with + # The 200px have to be split between two missing segments, the most equal way to do it is with # sizes 100: 100+100=200 # Second group: - # The end position row of the gap is the start row of the last chunk -1: 11136-100+1-1=11036 - # The start position row of the gap is the end row fo the first chunk +1: 11136-300+1=10837 + # The end position row of the gap is the start row of the last segment -1: 11136-100+1-1=11036 + # The start position row of the gap is the end row fo the first segment +1: 11136-300+1=10837 # hence the gap is 11036-10837+1=200 px high - # The 200px have to be split between three missing chunks, the most equal way to do it is with + # The 200px have to be split between three missing segments, the most equal way to do it is with # sizes 66 and 67: 66+67+67=200 # Regarding the heights: # First group: - # The first chunk has 100px height and 500 area extent height. - # The first padded chunk has 100px height -> 500*100/100=500 area extent height ->1000+500=1500 - # The second padded chunk has 100px height -> 500*100/100=500 area extent height ->1500+500=2000 + # The first segment has 100px height and 500 area extent height. + # The first padded segment has 100px height -> 500*100/100=500 area extent height ->1000+500=1500 + # The second padded segment has 100px height -> 500*100/100=500 area extent height ->1500+500=2000 # Second group: - # The first chunk has 100px height and 500 area extent height. - # The first padded chunk has 66px height -> 500*66/100=330 area extent height ->1000+330=1330 - # The second padded chunk has 67px height -> 500*67/100=335 area extent height ->1330+335=1665 - # The first padded chunk has 67px height -> 500*67/100=335 area extent height ->1665+335=2000 - self.assertEqual(AreaDefinition.call_count, 5) + # The first segment has 100px height and 500 area extent height. + # The first padded segment has 66px height -> 500*66/100=330 area extent height ->1000+330=1330 + # The second padded segment has 67px height -> 500*67/100=335 area extent height ->1330+335=1665 + # The first padded segment has 67px height -> 500*67/100=335 area extent height ->1665+335=2000 + assert fake_adef.call_count == 5 expected_call1 = ('fill', 'fill', 'fill', 'some_crs', 11136, 100, (0, 1500.0, 200, 1000)) expected_call2 = ('fill', 'fill', 'fill', 'some_crs', 11136, 100, @@ -1467,13 +1484,13 @@ def side_effect_areadef(a, b, c, crs, width, height, aex): expected_call5 = ('fill', 'fill', 'fill', 'some_crs', 11136, 67, (0, 2000.0, 200, 1665.0)) - AreaDefinition.side_effect = None - AreaDefinition.assert_has_calls([call(*expected_call1), - call(*expected_call2), - call(*expected_call3), - call(*expected_call4), - call(*expected_call5) - ]) + fake_adef.side_effect = None + fake_adef.assert_has_calls([call(*expected_call1), + call(*expected_call2), + call(*expected_call3), + call(*expected_call4), + call(*expected_call5) + ]) def test_get_empty_segment_with_height(self): """Test _get_empty_segment_with_height()."""