diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
index c2fa26dafa..e775944b5a 100644
--- a/.github/workflows/ci.yaml
+++ b/.github/workflows/ci.yaml
@@ -103,7 +103,7 @@ jobs:
git+https://github.com/dask/distributed \
git+https://github.com/zarr-developers/zarr \
git+https://github.com/Unidata/cftime \
- git+https://github.com/mapbox/rasterio \
+ git+https://github.com/rasterio/rasterio \
git+https://github.com/pydata/bottleneck \
git+https://github.com/pydata/xarray \
git+https://github.com/astropy/astropy;
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index caeabd5305..3d36ec4301 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -20,7 +20,7 @@ repos:
- id: bandit
args: [--ini, .bandit]
- repo: https://github.com/pre-commit/mirrors-mypy
- rev: 'v0.971' # Use the sha / tag you want to point at
+ rev: 'v0.982' # Use the sha / tag you want to point at
hooks:
- id: mypy
additional_dependencies:
diff --git a/continuous_integration/environment.yaml b/continuous_integration/environment.yaml
index b86bc1e072..f8a94233bc 100644
--- a/continuous_integration/environment.yaml
+++ b/continuous_integration/environment.yaml
@@ -2,7 +2,7 @@ name: test-environment
channels:
- conda-forge
dependencies:
- - xarray
+ - xarray!=2022.9.0
- dask
- distributed
- donfig
diff --git a/satpy/composites/ahi.py b/satpy/composites/ahi.py
index edd195bbcf..bb96a94581 100644
--- a/satpy/composites/ahi.py
+++ b/satpy/composites/ahi.py
@@ -1,6 +1,4 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-# Copyright (c) 2015-2021 Satpy developers
+# Copyright (c) 2022- Satpy developers
#
# This file is part of satpy.
#
@@ -15,30 +13,8 @@
#
# You should have received a copy of the GNU General Public License along with
# satpy. If not, see .
-"""Composite classes for the AHI instrument."""
+"""Composite classes for AHI."""
-import logging
-
-from satpy.composites import GenericCompositor
-from satpy.dataset import combine_metadata
-
-LOG = logging.getLogger(__name__)
-
-
-class GreenCorrector(GenericCompositor):
- """Corrector of the AHI green band to compensate for the deficit of chlorophyll signal."""
-
- def __init__(self, *args, fractions=(0.85, 0.15), **kwargs):
- """Set default keyword argument values."""
- # XXX: Should this be 0.93 and 0.07
- self.fractions = fractions
- super(GreenCorrector, self).__init__(*args, **kwargs)
-
- def __call__(self, projectables, optional_datasets=None, **attrs):
- """Boost vegetation effect thanks to NIR (0.8µm) band."""
- LOG.info('Boosting vegetation on green band')
-
- projectables = self.match_data_arrays(projectables)
- new_green = sum(fraction * value for fraction, value in zip(self.fractions, projectables))
- new_green.attrs = combine_metadata(*projectables)
- return super(GreenCorrector, self).__call__((new_green,), **attrs)
+# The green corrector used to be defined here, but was moved to spectral.py
+# in Satpy 0.38 because it also applies to FCI.
+from .spectral import GreenCorrector # noqa: F401
diff --git a/satpy/composites/spectral.py b/satpy/composites/spectral.py
new file mode 100644
index 0000000000..d0e6dc9330
--- /dev/null
+++ b/satpy/composites/spectral.py
@@ -0,0 +1,70 @@
+# Copyright (c) 2015-2022 Satpy developers
+#
+# This file is part of satpy.
+#
+# satpy is free software: you can redistribute it and/or modify it under the
+# terms of the GNU General Public License as published by the Free Software
+# Foundation, either version 3 of the License, or (at your option) any later
+# version.
+#
+# satpy is distributed in the hope that it will be useful, but WITHOUT ANY
+# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
+# A PARTICULAR PURPOSE. See the GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along with
+# satpy. If not, see .
+"""Composite classes for spectral adjustments."""
+
+import logging
+
+from satpy.composites import GenericCompositor
+from satpy.dataset import combine_metadata
+
+LOG = logging.getLogger(__name__)
+
+
+class GreenCorrector(GenericCompositor):
+ """Corrector of the FCI or AHI green band.
+
+ The green band in FCI and AHI deliberately misses the chlorophyll peak
+ in order to focus on aerosol and ash rather than on vegetation. This
+ affects true colour RGBs, because vegetation looks brown rather than green.
+ To make vegetation look greener again, this corrector allows
+ to simulate the green band as a fraction of two or more other channels.
+
+ To be used, the composite takes two or more input channels and a parameter
+ ``fractions`` that should be a list of floats with the same length as the
+ number of channels.
+
+ For example, to simulate an FCI corrected green composite, one could use
+ a combination of 93% from the green band (vis_05) and 7% from the
+ near-infrared 0.8 µm band (vis_08)::
+
+ corrected_green:
+ compositor: !!python/name:satpy.composites.ahi.GreenCorrector
+ fractions: [0.93, 0.07]
+ prerequisites:
+ - name: vis_05
+ modifiers: [sunz_corrected, rayleigh_corrected]
+ - name: vis_08
+ modifiers: [sunz_corrected, rayleigh_corrected]
+ standard_name: toa_bidirectional_reflectance
+
+ Other examples can be found in the ``fci.yaml`` and ``ahi.yaml`` composite
+ files in the satpy distribution.
+ """
+
+ def __init__(self, *args, fractions=(0.85, 0.15), **kwargs):
+ """Set default keyword argument values."""
+ # XXX: Should this be 0.93 and 0.07
+ self.fractions = fractions
+ super(GreenCorrector, self).__init__(*args, **kwargs)
+
+ def __call__(self, projectables, optional_datasets=None, **attrs):
+ """Boost vegetation effect thanks to NIR (0.8µm) band."""
+ LOG.info('Boosting vegetation on green band')
+
+ projectables = self.match_data_arrays(projectables)
+ new_green = sum(fraction * value for fraction, value in zip(self.fractions, projectables))
+ new_green.attrs = combine_metadata(*projectables)
+ return super(GreenCorrector, self).__call__((new_green,), **attrs)
diff --git a/satpy/enhancements/viirs.py b/satpy/enhancements/viirs.py
index 6a465f161b..627fc80220 100644
--- a/satpy/enhancements/viirs.py
+++ b/satpy/enhancements/viirs.py
@@ -38,7 +38,7 @@ def water_detection(img, **kwargs):
@exclude_alpha
@using_map_blocks
def _water_detection(img_data):
- data = np.asarray(img_data)
+ data = np.asarray(img_data).copy()
data[data == 150] = 31
data[data == 199] = 18
data[data >= 200] = data[data >= 200] - 100
diff --git a/satpy/etc/composites/ahi.yaml b/satpy/etc/composites/ahi.yaml
index 6c3d1860e6..ad77eeab50 100644
--- a/satpy/etc/composites/ahi.yaml
+++ b/satpy/etc/composites/ahi.yaml
@@ -16,7 +16,7 @@ modifiers:
composites:
green:
- compositor: !!python/name:satpy.composites.ahi.GreenCorrector
+ compositor: !!python/name:satpy.composites.spectral.GreenCorrector
# FUTURE: Set a wavelength...see what happens. Dependency finding
# probably wouldn't work.
prerequisites:
@@ -31,7 +31,7 @@ composites:
green_true_color_reproduction:
# JMA True Color Reproduction green band
# http://www.jma.go.jp/jma/jma-eng/satellite/introduction/TCR.html
- compositor: !!python/name:satpy.composites.ahi.GreenCorrector
+ compositor: !!python/name:satpy.composites.spectral.GreenCorrector
fractions: [0.6321, 0.2928, 0.0751]
prerequisites:
- name: B02
@@ -43,7 +43,7 @@ composites:
standard_name: none
green_nocorr:
- compositor: !!python/name:satpy.composites.ahi.GreenCorrector
+ compositor: !!python/name:satpy.composites.spectral.GreenCorrector
# FUTURE: Set a wavelength...see what happens. Dependency finding
# probably wouldn't work.
prerequisites:
diff --git a/satpy/etc/composites/ami.yaml b/satpy/etc/composites/ami.yaml
index 6049ef0a10..0c2635d3c7 100644
--- a/satpy/etc/composites/ami.yaml
+++ b/satpy/etc/composites/ami.yaml
@@ -2,7 +2,7 @@ sensor_name: visir/ami
composites:
green_raw:
- compositor: !!python/name:satpy.composites.ahi.GreenCorrector
+ compositor: !!python/name:satpy.composites.spectral.GreenCorrector
prerequisites:
- name: VI005
modifiers: [sunz_corrected]
@@ -12,7 +12,7 @@ composites:
fractions: [0.85, 0.15]
green:
- compositor: !!python/name:satpy.composites.ahi.GreenCorrector
+ compositor: !!python/name:satpy.composites.spectral.GreenCorrector
prerequisites:
- name: VI005
modifiers: [sunz_corrected, rayleigh_corrected]
@@ -22,7 +22,7 @@ composites:
fractions: [0.85, 0.15]
green_nocorr:
- compositor: !!python/name:satpy.composites.ahi.GreenCorrector
+ compositor: !!python/name:satpy.composites.spectral.GreenCorrector
prerequisites:
- name: VI005
- name: VI008
diff --git a/satpy/etc/composites/fci.yaml b/satpy/etc/composites/fci.yaml
index a3e558e455..853fcff835 100644
--- a/satpy/etc/composites/fci.yaml
+++ b/satpy/etc/composites/fci.yaml
@@ -2,6 +2,31 @@ sensor_name: visir/fci
composites:
+ corrected_green:
+ description: >
+ The FCI green band at 0.51 µm deliberately misses the chlorophyl band, such that
+ the signal comes rather from aerosols and ash rather than vegetation. An effect
+ is that vegetation in a true colour RGB looks rather brown than green. Mixing in
+ some part of the NIR 0.8 channel reduced this effect. Note that the fractions
+ currently implemented are experimental and may change in future versions of Satpy.
+ compositor: !!python/name:satpy.composites.spectral.GreenCorrector
+ fractions: [0.93, 0.07]
+ prerequisites:
+ - name: vis_05
+ modifiers: [sunz_corrected, rayleigh_corrected]
+ - name: vis_08
+ modifiers: [sunz_corrected, rayleigh_corrected]
+ standard_name: toa_bidirectional_reflectance
+
+ corrected_green_raw:
+ description: >
+ Alternative to corrected_green, but without solar zenith or rayleigh correction.
+ compositor: !!python/name:satpy.composites.spectral.GreenCorrector
+ fractions: [0.93, 0.07]
+ prerequisites:
+ - name: vis_05
+ - name: vis_08
+ standard_name: toa_bidirectional_reflectance
binary_cloud_mask:
# This will set all clear pixels to '0', all pixles with cloudy features (meteorological/dust/ash clouds) to '1' and
@@ -11,3 +36,28 @@ composites:
- name: 'cloud_state'
lut: [.nan, 0, 1, 1, 1, 1, 1, 1, 0, .nan]
standard_name: binary_cloud_mask
+
+ true_color:
+ compositor: !!python/name:satpy.composites.SelfSharpenedRGB
+ description: >
+ FCI true color composite. The green band is simulated based on a combination of
+ channels. This simulation may change in future versions of Satpy. See the description
+ of the corrected_green composites for details.
+ prerequisites:
+ - name: vis_06
+ modifiers: [sunz_corrected, rayleigh_corrected]
+ - name: corrected_green
+ - name: vis_04
+ modifiers: [sunz_corrected, rayleigh_corrected]
+ standard_name: true_color
+
+ true_color_raw_with_corrected_green:
+ compositor: !!python/name:satpy.composites.SelfSharpenedRGB
+ description: >
+ FCI true color without solar zenith or rayleigh corrections, but with the
+ corrected green composite.
+ prerequisites:
+ - name: vis_06
+ - name: corrected_green_raw
+ - name: vis_04
+ standard_name: true_color_raw
diff --git a/satpy/etc/readers/ici_l1b_nc.yaml b/satpy/etc/readers/ici_l1b_nc.yaml
index 8d8f271867..cbf6c9f993 100644
--- a/satpy/etc/readers/ici_l1b_nc.yaml
+++ b/satpy/etc/readers/ici_l1b_nc.yaml
@@ -12,7 +12,7 @@ reader:
name:
required: true
frequency_double_sideband:
- type: !!python/name:satpy.readers.aapp_mhs_amsub_l1c.FrequencyDoubleSideBand
+ type: !!python/name:satpy.readers.pmw_channels_definitions.FrequencyDoubleSideBand
polarization:
enum:
- H
diff --git a/satpy/etc/readers/mws_l1b_nc.yaml b/satpy/etc/readers/mws_l1b_nc.yaml
index ce7fe527de..1f77bbe7a2 100644
--- a/satpy/etc/readers/mws_l1b_nc.yaml
+++ b/satpy/etc/readers/mws_l1b_nc.yaml
@@ -493,22 +493,6 @@ datasets:
- mws_lon
- mws_lat
-# --- Land surface data ---
- surface_type:
- name: surface_type
- standard_name: surface_type
- file_type: mws_l1b_nc
- coordinates:
- - mws_lon
- - mws_lat
- terrain_elevation:
- name: terrain_elevation
- standard_name: terrain_elevation
- file_type: mws_l1b_nc
- coordinates:
- - mws_lon
- - mws_lat
-
file_types:
mws_l1b_nc:
diff --git a/satpy/etc/readers/seadas_l2.yaml b/satpy/etc/readers/seadas_l2.yaml
index ca6ce3d956..99ff897bac 100644
--- a/satpy/etc/readers/seadas_l2.yaml
+++ b/satpy/etc/readers/seadas_l2.yaml
@@ -15,41 +15,53 @@ file_types:
- '{platform_indicator:1s}1.{start_time:%y%j.%H%M}.seadas.hdf'
file_reader: !!python/name:satpy.readers.seadas_l2.SEADASL2HDFFileHandler
geo_resolution: 1000
+ chlora_seadas_nc:
+ file_patterns:
+ # IMAPP-style filenames:
+ - '{platform_indicator:1s}1.{start_time:%y%j.%H%M}.seadas.nc'
+ file_reader: !!python/name:satpy.readers.seadas_l2.SEADASL2NetCDFFileHandler
+ geo_resolution: 1000
chlora_seadas_viirs:
# SEADAS_npp_d20211118_t1728125_e1739327.hdf
file_patterns:
- 'SEADAS_{platform_indicator:s}_d{start_time:%Y%m%d_t%H%M%S%f}_e{end_time:%H%M%S%f}.hdf'
file_reader: !!python/name:satpy.readers.seadas_l2.SEADASL2HDFFileHandler
geo_resolution: 750
+ chlora_seadas_viirs_nc:
+ # SEADAS_npp_d20211118_t1728125_e1739327.nc
+ file_patterns:
+ - 'SEADAS_{platform_indicator:s}_d{start_time:%Y%m%d_t%H%M%S%f}_e{end_time:%H%M%S%f}.nc'
+ file_reader: !!python/name:satpy.readers.seadas_l2.SEADASL2NetCDFFileHandler
+ geo_resolution: 750
datasets:
longitude:
name: longitude
- file_type: [chlora_seadas, seadas_hdf_viirs]
- file_key: longitude
+ file_type: [chlora_seadas_viirs_nc, chlora_seadas_nc, seadas_hdf_viirs, chlora_sedas]
+ file_key: ["navigation_data/longitude", "longitude"]
resolution:
1000:
- file_type: chlora_seadas
+ file_type: [chlora_seadas_nc, chlora_seadas]
750:
- file_type: chlora_seadas_viirs
+ file_type: [chlora_seadas_viirs_nc, chlora_seadas_viirs]
latitude:
name: latitude
- file_type: [chlora_seadas, seadas_hdf_viirs]
- file_key: latitude
+ file_type: [chlora_seadas_viirs_nc, chlora_seadas_nc, seadas_hdf_viirs, chlora_sedas]
+ file_key: ["navigation_data/latitude", "latitude"]
resolution:
1000:
- file_type: chlora_seadas
+ file_type: [chlora_seadas_nc, chlora_seadas]
750:
- file_type: chlora_seadas_viirs
+ file_type: [chlora_seadas_viirs_nc, chlora_seadas_viirs]
chlor_a:
name: chlor_a
- file_type: [chlora_seadas, seadas_hdf_viirs]
- file_key: chlor_a
+ file_type: [chlora_seadas_viirs_nc, chlora_seadas_nc, seadas_hdf_viirs, chlora_sedas]
+ file_key: ["geophysical_data/chlor_a", "chlor_a"]
resolution:
1000:
- file_type: chlora_seadas
+ file_type: [chlora_seadas_nc, chlora_seadas]
750:
- file_type: chlora_seadas_viirs
+ file_type: [chlora_seadas_viirs_nc, chlora_seadas_viirs]
coordinates: [longitude, latitude]
diff --git a/satpy/etc/writers/awips_tiled.yaml b/satpy/etc/writers/awips_tiled.yaml
index 6fcc5aa948..e761414904 100644
--- a/satpy/etc/writers/awips_tiled.yaml
+++ b/satpy/etc/writers/awips_tiled.yaml
@@ -129,6 +129,12 @@ templates:
physical_element:
raw_value: CLAVR-x Cloud Type
units: {}
+ encoding:
+ dtype: int16
+ _Unsigned: "true"
+ scale_factor: 0.5
+ add_offset: 0.0
+ _FillValue: -128
clavrx_cld_temp_acha:
reader: clavrx
name: cld_temp_acha
@@ -153,6 +159,12 @@ templates:
units: {}
physical_element:
raw_value: CLAVR-x Cloud Phase
+ encoding:
+ dtype: int16
+ _Unsigned: "true"
+ scale_factor: 0.5
+ add_offset: 0.0
+ _FillValue: -128
clavrx_cld_opd_dcomp:
reader: clavrx
name: cld_opd_dcomp
diff --git a/satpy/readers/abi_base.py b/satpy/readers/abi_base.py
index 2f97c925cc..b74c4cf728 100644
--- a/satpy/readers/abi_base.py
+++ b/satpy/readers/abi_base.py
@@ -18,6 +18,7 @@
"""Advance Baseline Imager reader base class for the Level 1b and l2+ reader."""
import logging
+from contextlib import suppress
from datetime import datetime
import numpy as np
@@ -76,7 +77,8 @@ def _rename_dims(nc):
if 't' in nc.dims or 't' in nc.coords:
nc = nc.rename({'t': 'time'})
if 'goes_lat_lon_projection' in nc:
- nc = nc.rename({'lon': 'x', 'lat': 'y'})
+ with suppress(ValueError):
+ nc = nc.rename({'lon': 'x', 'lat': 'y'})
return nc
@property
diff --git a/satpy/readers/hy2_scat_l2b_h5.py b/satpy/readers/hy2_scat_l2b_h5.py
index 6bdb8f88a4..64520bae9a 100644
--- a/satpy/readers/hy2_scat_l2b_h5.py
+++ b/satpy/readers/hy2_scat_l2b_h5.py
@@ -107,11 +107,13 @@ def get_dataset(self, key, info):
else:
dim_map = {curr_dim: new_dim for curr_dim, new_dim in zip(data.dims, dims)}
data = data.rename(dim_map)
- data = self._mask_data(key['name'], data)
- data = self._scale_data(key['name'], data)
+ data = self._mask_data(data)
+ data = self._scale_data(data)
if key['name'] in 'wvc_lon':
+ _attrs = data.attrs
data = xr.where(data > 180, data - 360., data)
+ data.attrs.update(_attrs)
data.attrs.update(info)
data.attrs.update(self.get_metadata())
data.attrs.update(self.get_variable_metadata())
@@ -120,13 +122,14 @@ def get_dataset(self, key, info):
return data
- def _scale_data(self, key_name, data):
- return data * self[key_name].attrs['scale_factor'] + self[key_name].attrs['add_offset']
+ def _scale_data(self, data):
+ return data * data.attrs['scale_factor'] + data.attrs['add_offset']
- def _mask_data(self, key_name, data):
- data = xr.where(data == self[key_name].attrs['fill_value'], np.nan, data)
-
- valid_range = self[key_name].attrs['valid_range']
+ def _mask_data(self, data):
+ _attrs = data.attrs
+ valid_range = data.attrs['valid_range']
+ data = xr.where(data == data.attrs['fill_value'], np.nan, data)
data = xr.where(data < valid_range[0], np.nan, data)
data = xr.where(data > valid_range[1], np.nan, data)
+ data.attrs.update(_attrs)
return data
diff --git a/satpy/readers/seadas_l2.py b/satpy/readers/seadas_l2.py
index 708b694fac..281a0132af 100644
--- a/satpy/readers/seadas_l2.py
+++ b/satpy/readers/seadas_l2.py
@@ -31,17 +31,16 @@
from datetime import datetime
from .hdf4_utils import HDF4FileHandler
+from .netcdf_utils import NetCDF4FileHandler
-TIME_FORMAT = "%Y%j%H%M%S"
-
-class SEADASL2HDFFileHandler(HDF4FileHandler):
+class _SEADASL2Base:
"""Simple handler of SEADAS L2 files."""
def __init__(self, filename, filename_info, filetype_info, apply_quality_flags=False):
"""Initialize file handler and determine if data quality flags should be applied."""
super().__init__(filename, filename_info, filetype_info)
- self.apply_quality_flags = apply_quality_flags and "l2_flags" in self
+ self.apply_quality_flags = apply_quality_flags and self.l2_flags_var_name in self
def _add_satpy_metadata(self, data):
data.attrs["sensor"] = self.sensor_names
@@ -57,7 +56,7 @@ def _rows_per_scan(self):
raise ValueError(f"Don't know how to read data for sensors: {self.sensor_names}")
def _platform_name(self):
- platform = self["/attr/Mission"]
+ platform = self[self.platform_attr_name]
platform_dict = {'NPP': 'Suomi-NPP',
'JPSS-1': 'NOAA-20',
'JPSS-2': 'NOAA-21'}
@@ -66,38 +65,93 @@ def _platform_name(self):
@property
def start_time(self):
"""Get the starting observation time of this file's data."""
- start_time = self["/attr/Start Time"]
- return datetime.strptime(start_time[:-3], TIME_FORMAT)
+ start_time = self[self.start_time_attr_name]
+ return datetime.strptime(start_time[:-3], self.time_format)
@property
def end_time(self):
"""Get the ending observation time of this file's data."""
- end_time = self["/attr/End Time"]
- return datetime.strptime(end_time[:-3], TIME_FORMAT)
+ end_time = self[self.end_time_attr_name]
+ return datetime.strptime(end_time[:-3], self.time_format)
@property
def sensor_names(self):
"""Get sensor for the current file's data."""
# Example: MODISA or VIIRSN or VIIRSJ1
- sensor_name = self["/attr/Sensor Name"].lower()
+ sensor_name = self[self.sensor_attr_name].lower()
if sensor_name.startswith("modis"):
return {"modis"}
return {"viirs"}
def get_dataset(self, data_id, dataset_info):
"""Get DataArray for the specified DataID."""
- file_key = dataset_info.get("file_key", data_id["name"])
- data = self[file_key]
- valid_range = data.attrs["valid_range"]
- data = data.where(valid_range[0] <= data)
- data = data.where(data <= valid_range[1])
- if self.apply_quality_flags and not ("lon" in file_key or "lat" in file_key):
- l2_flags = self["l2_flags"]
- mask = (l2_flags & 0b00000000010000000000000000000000) != 0
- data = data.where(~mask)
+ file_key, data = self._get_file_key_and_variable(data_id, dataset_info)
+ data = self._filter_by_valid_min_max(data)
+ data = self._rename_2d_dims_if_necessary(data)
+ data = self._mask_based_on_l2_flags(data)
for attr_name in ("standard_name", "long_name", "units"):
val = data.attrs[attr_name]
if val[-1] == "\x00":
data.attrs[attr_name] = data.attrs[attr_name][:-1]
data = self._add_satpy_metadata(data)
return data
+
+ def _get_file_key_and_variable(self, data_id, dataset_info):
+ file_keys = dataset_info.get("file_key", data_id["name"])
+ if not isinstance(file_keys, list):
+ file_keys = [file_keys]
+ for file_key in file_keys:
+ try:
+ data = self[file_key]
+ return file_key, data
+ except KeyError:
+ continue
+ raise KeyError(f"Unable to find any of the possible keys for {data_id}: {file_keys}")
+
+ def _rename_2d_dims_if_necessary(self, data_arr):
+ if data_arr.ndim != 2 or data_arr.dims == ("y", "x"):
+ return data_arr
+ return data_arr.rename(dict(zip(data_arr.dims, ("y", "x"))))
+
+ def _filter_by_valid_min_max(self, data_arr):
+ valid_range = self._valid_min_max(data_arr)
+ data_arr = data_arr.where(valid_range[0] <= data_arr)
+ data_arr = data_arr.where(data_arr <= valid_range[1])
+ return data_arr
+
+ def _valid_min_max(self, data_arr):
+ try:
+ return data_arr.attrs["valid_range"]
+ except KeyError:
+ return data_arr.attrs["valid_min"], data_arr.attrs["valid_max"]
+
+ def _mask_based_on_l2_flags(self, data_arr):
+ standard_name = data_arr.attrs.get("standard_name", "")
+ if self.apply_quality_flags and not ("lon" in standard_name or "lat" in standard_name):
+ l2_flags = self[self.l2_flags_var_name]
+ l2_flags = self._rename_2d_dims_if_necessary(l2_flags)
+ mask = (l2_flags & 0b00000000010000000000000000000000) != 0
+ data_arr = data_arr.where(~mask)
+ return data_arr
+
+
+class SEADASL2NetCDFFileHandler(_SEADASL2Base, NetCDF4FileHandler):
+ """Simple handler of SEADAS L2 NetCDF4 files."""
+
+ start_time_attr_name = "/attr/time_coverage_start"
+ end_time_attr_name = "/attr/time_coverage_end"
+ time_format = "%Y-%m-%dT%H:%M:%S.%f"
+ platform_attr_name = "/attr/platform"
+ sensor_attr_name = "/attr/instrument"
+ l2_flags_var_name = "geophysical_data/l2_flags"
+
+
+class SEADASL2HDFFileHandler(_SEADASL2Base, HDF4FileHandler):
+ """Simple handler of SEADAS L2 HDF4 files."""
+
+ start_time_attr_name = "/attr/Start Time"
+ end_time_attr_name = "/attr/End Time"
+ time_format = "%Y%j%H%M%S"
+ platform_attr_name = "/attr/Mission"
+ sensor_attr_name = "/attr/Sensor Name"
+ l2_flags_var_name = "l2_flags"
diff --git a/satpy/readers/viirs_edr_active_fires.py b/satpy/readers/viirs_edr_active_fires.py
index 2fdfaf6bcf..f1bcf4d3cc 100644
--- a/satpy/readers/viirs_edr_active_fires.py
+++ b/satpy/readers/viirs_edr_active_fires.py
@@ -92,12 +92,12 @@ def end_time(self):
@property
def sensor_name(self):
"""Name of sensor for this file."""
- return self["sensor"].lower()
+ return self["/attr/instrument_name"].lower()
@property
def platform_name(self):
"""Name of platform/satellite for this file."""
- return self["platform_name"]
+ return self["/attr/satellite_name"]
class VIIRSActiveFiresTextFileHandler(BaseFileHandler):
diff --git a/satpy/scene.py b/satpy/scene.py
index dd1ca61dcd..06f36e561e 100644
--- a/satpy/scene.py
+++ b/satpy/scene.py
@@ -275,7 +275,8 @@ def _compare_swath_defs(compare_func: Callable, swath_defs: list[SwathDefinition
def _key_func(swath_def: SwathDefinition) -> tuple:
attrs = getattr(swath_def.lons, "attrs", {})
lon_ds_name = attrs.get("name")
- return swath_def.shape[1], swath_def.shape[0], lon_ds_name
+ rev_shape = swath_def.shape[::-1]
+ return rev_shape + (lon_ds_name,)
return compare_func(swath_defs, key=_key_func)
def _gather_all_areas(self, datasets):
diff --git a/satpy/tests/reader_tests/test_abi_l2_nc.py b/satpy/tests/reader_tests/test_abi_l2_nc.py
index 02aaad977b..63014685f9 100644
--- a/satpy/tests/reader_tests/test_abi_l2_nc.py
+++ b/satpy/tests/reader_tests/test_abi_l2_nc.py
@@ -279,3 +279,69 @@ def test_get_area_def_latlon(self, adef):
self.assertEqual(call_args[4], self.reader.ncols)
self.assertEqual(call_args[5], self.reader.nlines)
np.testing.assert_allclose(call_args[6], (-85.0, -20.0, -65.0, 20))
+
+
+class Test_NC_ABI_L2_area_AOD(unittest.TestCase):
+ """Test the NC_ABI_L2 reader for the AOD product."""
+
+ @mock.patch('satpy.readers.abi_base.xr')
+ def setUp(self, xr_):
+ """Create fake data for the tests."""
+ from satpy.readers.abi_l2_nc import NC_ABI_L2
+ proj = xr.DataArray(
+ [],
+ attrs={'semi_major_axis': 1.,
+ 'semi_minor_axis': 1.,
+ 'inverse_flattening': 1.,
+ 'longitude_of_prime_meridian': 0.0,
+ }
+ )
+
+ proj_ext = xr.DataArray(
+ [],
+ attrs={'geospatial_westbound_longitude': -85.0,
+ 'geospatial_eastbound_longitude': -65.0,
+ 'geospatial_northbound_latitude': 20.0,
+ 'geospatial_southbound_latitude': -20.0,
+ 'geospatial_lat_center': 0.0,
+ 'geospatial_lon_center': -75.0,
+ })
+
+ x__ = xr.DataArray(
+ [0, 1],
+ attrs={'scale_factor': 2., 'add_offset': -1.},
+ dims=('x',),
+ )
+ y__ = xr.DataArray(
+ [0, 1],
+ attrs={'scale_factor': -2., 'add_offset': 1.},
+ dims=('y',),
+ )
+ fake_dataset = xr.Dataset(
+ data_vars={
+ 'goes_lat_lon_projection': proj,
+ 'geospatial_lat_lon_extent': proj_ext,
+ 'x': x__,
+ 'y': y__,
+ 'RSR': xr.DataArray(np.ones((2, 2)), dims=('y', 'x')),
+ },
+ )
+ xr_.open_dataset.return_value = fake_dataset
+
+ self.reader = NC_ABI_L2('filename',
+ {'platform_shortname': 'G16', 'observation_type': 'RSR',
+ 'scene_abbr': 'C', 'scan_mode': 'M3'},
+ {'filetype': 'info'})
+
+ @mock.patch('satpy.readers.abi_base.geometry.AreaDefinition')
+ def test_get_area_def_xy(self, adef):
+ """Test the area generation."""
+ self.reader.get_area_def(None)
+
+ self.assertEqual(adef.call_count, 1)
+ call_args = tuple(adef.call_args)[0]
+ self.assertDictEqual(call_args[3], {'proj': 'latlong', 'a': 1.0, 'b': 1.0, 'fi': 1.0, 'pm': 0.0,
+ 'lon_0': -75.0, 'lat_0': 0.0})
+ self.assertEqual(call_args[4], self.reader.ncols)
+ self.assertEqual(call_args[5], self.reader.nlines)
+ np.testing.assert_allclose(call_args[6], (-85.0, -20.0, -65.0, 20))
diff --git a/satpy/tests/reader_tests/test_hy2_scat_l2b_h5.py b/satpy/tests/reader_tests/test_hy2_scat_l2b_h5.py
index 1664c46ed4..b2a5d4d3e1 100644
--- a/satpy/tests/reader_tests/test_hy2_scat_l2b_h5.py
+++ b/satpy/tests/reader_tests/test_hy2_scat_l2b_h5.py
@@ -40,6 +40,13 @@
class FakeHDF5FileHandler2(FakeHDF5FileHandler):
"""Swap-in HDF5 File Handler."""
+ def __getitem__(self, key):
+ """Return copy of dataarray to prevent manipulating attributes in the original."""
+ val = self.file_content[key]
+ if isinstance(val, xr.core.dataarray.DataArray):
+ val = val.copy()
+ return val
+
def _get_geo_data(self, num_rows, num_cols):
geo = {
'wvc_lon':
@@ -498,3 +505,20 @@ def test_reading_attrs_nsoas(self):
with self.assertRaises(KeyError):
self.assertEqual(res['wvc_lon'].attrs['L2B_Number_WVC_cells'], 10)
self.assertEqual(res['wvc_lon'].attrs['L2B_Expected_WVC_Cells'], 10)
+
+ def test_properties(self):
+ """Test platform_name."""
+ from datetime import datetime
+
+ from satpy.readers import load_reader
+ filenames = [
+ 'W_XX-EUMETSAT-Darmstadt,SURFACE+SATELLITE,HY2B+SM_C_EUMP_20200326------_07077_o_250_l2b.h5', ]
+
+ reader = load_reader(self.reader_configs)
+ files = reader.select_files_from_pathnames(filenames)
+ reader.create_filehandlers(files)
+ # Make sure we have some files
+ res = reader.load(['wvc_lon'])
+ self.assertEqual(res['wvc_lon'].platform_name, 'HY-2B')
+ self.assertEqual(res['wvc_lon'].start_time, datetime(2020, 3, 26, 1, 11, 7))
+ self.assertEqual(res['wvc_lon'].end_time, datetime(2020, 3, 26, 2, 55, 40))
diff --git a/satpy/tests/reader_tests/test_seadas_l2.py b/satpy/tests/reader_tests/test_seadas_l2.py
index dfa55a557d..2cc10a4344 100644
--- a/satpy/tests/reader_tests/test_seadas_l2.py
+++ b/satpy/tests/reader_tests/test_seadas_l2.py
@@ -19,7 +19,6 @@
import numpy as np
import pytest
-from pyhdf.SD import SD, SDC
from pyresample.geometry import SwathDefinition
from pytest_lazyfixture import lazy_fixture
@@ -31,7 +30,7 @@ def seadas_l2_modis_chlor_a(tmp_path_factory):
"""Create MODIS SEADAS file."""
filename = "a1.21322.1758.seadas.hdf"
full_path = str(tmp_path_factory.mktemp("seadas_l2") / filename)
- return _create_seadas_chlor_a_file(full_path, "Aqua", "MODISA")
+ return _create_seadas_chlor_a_hdf4_file(full_path, "Aqua", "MODISA")
@pytest.fixture(scope="module")
@@ -39,7 +38,7 @@ def seadas_l2_viirs_npp_chlor_a(tmp_path_factory):
"""Create VIIRS NPP SEADAS file."""
filename = "SEADAS_npp_d20211118_t1728125_e1739327.hdf"
full_path = str(tmp_path_factory.mktemp("seadas") / filename)
- return _create_seadas_chlor_a_file(full_path, "NPP", "VIIRSN")
+ return _create_seadas_chlor_a_hdf4_file(full_path, "NPP", "VIIRSN")
@pytest.fixture(scope="module")
@@ -47,10 +46,11 @@ def seadas_l2_viirs_j01_chlor_a(tmp_path_factory):
"""Create VIIRS JPSS-01 SEADAS file."""
filename = "SEADAS_j01_d20211118_t1728125_e1739327.hdf"
full_path = str(tmp_path_factory.mktemp("seadas") / filename)
- return _create_seadas_chlor_a_file(full_path, "JPSS-1", "VIIRSJ1")
+ return _create_seadas_chlor_a_hdf4_file(full_path, "JPSS-1", "VIIRSJ1")
-def _create_seadas_chlor_a_file(full_path, mission, sensor):
+def _create_seadas_chlor_a_hdf4_file(full_path, mission, sensor):
+ from pyhdf.SD import SD, SDC
h = SD(full_path, SDC.WRITE | SDC.CREATE)
setattr(h, "Sensor Name", sensor)
h.Mission = mission
@@ -79,8 +79,8 @@ def _create_seadas_chlor_a_file(full_path, mission, sensor):
"valid_range": (-90.0, 90.0),
}
}
- _add_variable_to_file(h, "longitude", lon_info)
- _add_variable_to_file(h, "latitude", lat_info)
+ _add_variable_to_hdf4_file(h, "longitude", lon_info)
+ _add_variable_to_hdf4_file(h, "latitude", lat_info)
chlor_a_info = {
"type": SDC.FLOAT32,
@@ -93,7 +93,7 @@ def _create_seadas_chlor_a_file(full_path, mission, sensor):
"valid_range": (0.001, 100.0),
}
}
- _add_variable_to_file(h, "chlor_a", chlor_a_info)
+ _add_variable_to_hdf4_file(h, "chlor_a", chlor_a_info)
l2_flags = np.zeros((5, 5), dtype=np.int32)
l2_flags[2, 2] = -1
@@ -103,11 +103,11 @@ def _create_seadas_chlor_a_file(full_path, mission, sensor):
"dim_labels": ["Number of Scan Lines", "Number of Pixel Control Points"],
"attrs": {},
}
- _add_variable_to_file(h, "l2_flags", l2_flags_info)
+ _add_variable_to_hdf4_file(h, "l2_flags", l2_flags_info)
return [full_path]
-def _add_variable_to_file(h, var_name, var_info):
+def _add_variable_to_hdf4_file(h, var_name, var_info):
v = h.create(var_name, var_info['type'], var_info['data'].shape)
v[:] = var_info['data']
for dim_count, dimension_name in enumerate(var_info['dim_labels']):
@@ -118,6 +118,85 @@ def _add_variable_to_file(h, var_name, var_info):
setattr(v, attr_key, attr_val)
+@pytest.fixture(scope="module")
+def seadas_l2_modis_chlor_a_netcdf(tmp_path_factory):
+ """Create MODIS SEADAS NetCDF file."""
+ filename = "t1.21332.1758.seadas.nc"
+ full_path = str(tmp_path_factory.mktemp("seadas_l2") / filename)
+ return _create_seadas_chlor_a_netcdf_file(full_path, "Terra", "MODIS")
+
+
+def _create_seadas_chlor_a_netcdf_file(full_path, mission, sensor):
+ from netCDF4 import Dataset
+ nc = Dataset(full_path, "w")
+ nc.createDimension("number_of_lines", 5)
+ nc.createDimension("pixels_per_line", 5)
+ nc.instrument = sensor
+ nc.platform = mission
+ nc.time_coverage_start = "2021-11-18T17:58:53.191Z"
+ nc.time_coverage_end = "2021-11-18T18:05:51.214Z"
+
+ lon_info = {
+ "data": np.zeros((5, 5), dtype=np.float32),
+ "dim_labels": ("number_of_lines", "pixels_per_line"),
+ "attrs": {
+ "long_name": "Longitude",
+ "standard_name": "longitude",
+ "units": "degrees_east",
+ "valid_min": -180.0,
+ "valid_max": 180.0,
+ }
+ }
+ lat_info = {
+ "data": np.zeros((5, 5), np.float32),
+ "dim_labels": ("number_of_lines", "pixels_per_line"),
+ "attrs": {
+ "long_name": "Latitude",
+ "standard_name": "latitude",
+ "units": "degrees_north",
+ "valid_min": -90.0,
+ "valid_max": 90.0,
+ }
+ }
+ nav_group = nc.createGroup("navigation_data")
+ _add_variable_to_netcdf_file(nav_group, "longitude", lon_info)
+ _add_variable_to_netcdf_file(nav_group, "latitude", lat_info)
+
+ chlor_a_info = {
+ "data": np.ones((5, 5), np.float32),
+ "dim_labels": ("number_of_lines", "pixels_per_line"),
+ "attrs": {
+ "long_name": "Chlorophyll Concentration, OCI Algorithm",
+ "units": "mg m^-3",
+ "standard_name": "mass_concentration_of_chlorophyll_in_sea_water",
+ "valid_min": 0.001,
+ "valid_max": 100.0,
+ }
+ }
+ l2_flags = np.zeros((5, 5), dtype=np.int32)
+ l2_flags[2, 2] = -1
+ l2_flags_info = {
+ "data": l2_flags,
+ "dim_labels": ("number_of_lines", "pixels_per_line"),
+ "attrs": {
+ "valid_min": -2147483648,
+ "valid_max": 2147483647,
+ },
+ }
+ geophys_group = nc.createGroup("geophysical_data")
+ _add_variable_to_netcdf_file(geophys_group, "chlor_a", chlor_a_info)
+ _add_variable_to_netcdf_file(geophys_group, "l2_flags", l2_flags_info)
+ return [full_path]
+
+
+def _add_variable_to_netcdf_file(nc, var_name, var_info):
+ v = nc.createVariable(var_name, var_info["data"].dtype.str[1:], dimensions=var_info["dim_labels"],
+ fill_value=var_info.get("fill_value"))
+ v[:] = var_info['data']
+ for attr_key, attr_val in var_info['attrs'].items():
+ setattr(v, attr_key, attr_val)
+
+
class TestSEADAS:
"""Test the SEADAS L2 file reader."""
@@ -145,6 +224,7 @@ def test_scene_available_datasets(self, input_files):
(lazy_fixture("seadas_l2_modis_chlor_a"), "Aqua", {"modis"}, 10),
(lazy_fixture("seadas_l2_viirs_npp_chlor_a"), "Suomi-NPP", {"viirs"}, 16),
(lazy_fixture("seadas_l2_viirs_j01_chlor_a"), "NOAA-20", {"viirs"}, 16),
+ (lazy_fixture("seadas_l2_modis_chlor_a_netcdf"), "Terra", {"modis"}, 10),
])
@pytest.mark.parametrize("apply_quality_flags", [False, True])
def test_load_chlor_a(self, input_files, exp_plat, exp_sensor, exp_rps, apply_quality_flags):
@@ -153,6 +233,7 @@ def test_load_chlor_a(self, input_files, exp_plat, exp_sensor, exp_rps, apply_qu
scene = Scene(reader='seadas_l2', filenames=input_files, reader_kwargs=reader_kwargs)
scene.load(['chlor_a'])
data_arr = scene['chlor_a']
+ assert data_arr.dims == ("y", "x")
assert data_arr.attrs['platform_name'] == exp_plat
assert data_arr.attrs['sensor'] == exp_sensor
assert data_arr.attrs['units'] == 'mg m^-3'
diff --git a/satpy/tests/reader_tests/test_viirs_edr_active_fires.py b/satpy/tests/reader_tests/test_viirs_edr_active_fires.py
index 9341ad2e21..df94283fba 100644
--- a/satpy/tests/reader_tests/test_viirs_edr_active_fires.py
+++ b/satpy/tests/reader_tests/test_viirs_edr_active_fires.py
@@ -61,8 +61,8 @@ def get_test_content(self, filename, filename_info, filename_type):
"""Mimic reader input file content."""
file_content = {}
file_content['/attr/data_id'] = "AFMOD"
- file_content['satellite_name'] = "npp"
- file_content['sensor'] = 'VIIRS'
+ file_content['/attr/satellite_name'] = "NPP"
+ file_content['/attr/instrument_name'] = 'VIIRS'
file_content['Fire Pixels/FP_latitude'] = DEFAULT_LATLON_FILE_DATA
file_content['Fire Pixels/FP_longitude'] = DEFAULT_LATLON_FILE_DATA
@@ -87,8 +87,8 @@ def get_test_content(self, filename, filename_info, filename_type):
"""Mimic reader input file content."""
file_content = {}
file_content['/attr/data_id'] = "AFIMG"
- file_content['satellite_name'] = "npp"
- file_content['sensor'] = 'VIIRS'
+ file_content['/attr/satellite_name'] = "NPP"
+ file_content['/attr/instrument_name'] = 'VIIRS'
file_content['Fire Pixels/FP_latitude'] = DEFAULT_LATLON_FILE_DATA
file_content['Fire Pixels/FP_longitude'] = DEFAULT_LATLON_FILE_DATA
diff --git a/satpy/tests/test_modifiers.py b/satpy/tests/test_modifiers.py
index d8598a5b05..d43a1005d0 100644
--- a/satpy/tests/test_modifiers.py
+++ b/satpy/tests/test_modifiers.py
@@ -169,10 +169,10 @@ class TestNIRReflectance(unittest.TestCase):
def setUp(self):
"""Set up the test case for the NIRReflectance compositor."""
- self.get_lonlats = mock.MagicMock()
+ get_lonlats = mock.MagicMock()
self.lons, self.lats = 1, 2
- self.get_lonlats.return_value = (self.lons, self.lats)
- area = mock.MagicMock(get_lonlats=self.get_lonlats)
+ get_lonlats.return_value = (self.lons, self.lats)
+ area = mock.MagicMock(get_lonlats=get_lonlats)
self.start_time = 1
self.metadata = {'platform_name': 'Meteosat-11',
@@ -241,7 +241,9 @@ def test_no_sunz_no_co2(self, calculator, apply_modifier_info, sza):
info = {'modifiers': None}
res = comp([self.nir, self.ir_], optional_datasets=[], **info)
- self.get_lonlats.assert_called()
+ # due to copying of DataArrays, self.get_lonlats is not the same as the one that was called
+ # we must used the area from the final result DataArray
+ res.attrs["area"].get_lonlats.assert_called()
sza.assert_called_with(self.start_time, self.lons, self.lats)
self.refl_from_tbs.assert_called_with(self.da_sunz, self.nir.data, self.ir_.data, tb_ir_co2=None)
assert np.allclose(res.data, self.refl * 100).compute()
diff --git a/satpy/tests/test_scene.py b/satpy/tests/test_scene.py
index bf2ffd948c..d17d69d01a 100644
--- a/satpy/tests/test_scene.py
+++ b/satpy/tests/test_scene.py
@@ -17,6 +17,7 @@
# satpy. If not, see .
"""Unit tests for scene.py."""
+import math
import os
import random
import string
@@ -709,7 +710,7 @@ def test_storage_options_from_reader_kwargs_per_reader(self):
def _create_coarest_finest_data_array(shape, area_def, attrs=None):
data_arr = xr.DataArray(
- da.arange(shape[0] * shape[1]).reshape(shape),
+ da.arange(math.prod(shape)).reshape(shape),
attrs={
'area': area_def,
})
@@ -735,8 +736,12 @@ def _create_coarsest_finest_area_def(shape, extents):
def _create_coarsest_finest_swath_def(shape, extents, name_suffix):
from pyresample import SwathDefinition
- lons_arr = da.repeat(da.linspace(extents[0], extents[2], shape[1], dtype=np.float32)[None, :], shape[0], axis=0)
- lats_arr = da.repeat(da.linspace(extents[1], extents[3], shape[0], dtype=np.float32)[:, None], shape[1], axis=1)
+ if len(shape) == 1:
+ lons_arr = da.linspace(extents[0], extents[2], shape[0], dtype=np.float32)
+ lats_arr = da.linspace(extents[1], extents[3], shape[0], dtype=np.float32)
+ else:
+ lons_arr = da.repeat(da.linspace(extents[0], extents[2], shape[1], dtype=np.float32)[None, :], shape[0], axis=0)
+ lats_arr = da.repeat(da.linspace(extents[1], extents[3], shape[0], dtype=np.float32)[:, None], shape[1], axis=1)
lons_data_arr = xr.DataArray(lons_arr, attrs={"name": f"longitude{name_suffix}"})
lats_data_arr = xr.DataArray(lats_arr, attrs={"name": f"latitude1{name_suffix}"})
return SwathDefinition(lons_data_arr, lats_data_arr)
@@ -754,6 +759,8 @@ class TestFinestCoarsestArea:
_create_coarsest_finest_area_def((4, 10), (-1000.0, -1500.0, 1000.0, 1500.0))),
(_create_coarsest_finest_swath_def((2, 5), (1000.0, 1500.0, -1000.0, -1500.0), "1"),
_create_coarsest_finest_swath_def((4, 10), (1000.0, 1500.0, -1000.0, -1500.0), "1")),
+ (_create_coarsest_finest_swath_def((5,), (1000.0, 1500.0, -1000.0, -1500.0), "1"),
+ _create_coarsest_finest_swath_def((10,), (1000.0, 1500.0, -1000.0, -1500.0), "1")),
]
)
def test_coarsest_finest_area_different_shape(self, coarse_area, fine_area):
diff --git a/satpy/tests/test_utils.py b/satpy/tests/test_utils.py
index 354c1886bb..3f0e055765 100644
--- a/satpy/tests/test_utils.py
+++ b/satpy/tests/test_utils.py
@@ -291,8 +291,10 @@ def test_get_satpos_from_satname(self, caplog):
(lon, lat, alt) = get_satpos(data_arr, use_tle=True)
assert "Orbital parameters missing from metadata" in caplog.text
np.testing.assert_allclose(
- (lon, lat, alt),
- (119.39533705010592, -1.1491628298731498, 35803.19986408156))
+ (lon, lat, alt),
+ (119.39533705010592, -1.1491628298731498, 35803.19986408156),
+ rtol=1e-4,
+ )
def test_make_fake_scene():