Handle valid range and valid_range correctly for two types of HY2 SCAT l2b hdf5 data

.github/workflows/ci.yaml

@@ -103,7 +103,7 @@ jobs:
           git+https://github.com/dask/distributed \
           git+https://github.com/zarr-developers/zarr \
           git+https://github.com/Unidata/cftime \
-          git+https://github.com/mapbox/rasterio \
+          git+https://github.com/rasterio/rasterio \
           git+https://github.com/pydata/bottleneck \
           git+https://github.com/pydata/xarray \
           git+https://github.com/astropy/astropy;

.pre-commit-config.yaml

@@ -20,7 +20,7 @@ repos:
       - id: bandit
         args: [--ini, .bandit]
   - repo: https://github.com/pre-commit/mirrors-mypy
-    rev: 'v0.971'  # Use the sha / tag you want to point at
+    rev: 'v0.982'  # Use the sha / tag you want to point at
     hooks:
       - id: mypy
         additional_dependencies:

continuous_integration/environment.yaml

@@ -2,7 +2,7 @@ name: test-environment
 channels:
   - conda-forge
 dependencies:
-  - xarray
+  - xarray!=2022.9.0
   - dask
   - distributed
   - donfig

satpy/composites/ahi.py

@@ -1,6 +1,4 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-# Copyright (c) 2015-2021 Satpy developers
+# Copyright (c) 2022- Satpy developers
 #
 # This file is part of satpy.
 #
@@ -15,30 +13,8 @@
 #
 # You should have received a copy of the GNU General Public License along with
 # satpy.  If not, see <http://www.gnu.org/licenses/>.
-"""Composite classes for the AHI instrument."""
+"""Composite classes for AHI."""
 
-import logging
-
-from satpy.composites import GenericCompositor
-from satpy.dataset import combine_metadata
-
-LOG = logging.getLogger(__name__)
-
-
-class GreenCorrector(GenericCompositor):
-    """Corrector of the AHI green band to compensate for the deficit of chlorophyll signal."""
-
-    def __init__(self, *args, fractions=(0.85, 0.15), **kwargs):
-        """Set default keyword argument values."""
-        # XXX: Should this be 0.93 and 0.07
-        self.fractions = fractions
-        super(GreenCorrector, self).__init__(*args, **kwargs)
-
-    def __call__(self, projectables, optional_datasets=None, **attrs):
-        """Boost vegetation effect thanks to NIR (0.8µm) band."""
-        LOG.info('Boosting vegetation on green band')
-
-        projectables = self.match_data_arrays(projectables)
-        new_green = sum(fraction * value for fraction, value in zip(self.fractions, projectables))
-        new_green.attrs = combine_metadata(*projectables)
-        return super(GreenCorrector, self).__call__((new_green,), **attrs)
+# The green corrector used to be defined here, but was moved to spectral.py
+# in Satpy 0.38 because it also applies to FCI.
+from .spectral import GreenCorrector  # noqa: F401

satpy/composites/spectral.py

@@ -0,0 +1,70 @@
+# Copyright (c) 2015-2022 Satpy developers
+#
+# This file is part of satpy.
+#
+# satpy is free software: you can redistribute it and/or modify it under the
+# terms of the GNU General Public License as published by the Free Software
+# Foundation, either version 3 of the License, or (at your option) any later
+# version.
+#
+# satpy is distributed in the hope that it will be useful, but WITHOUT ANY
+# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
+# A PARTICULAR PURPOSE.  See the GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along with
+# satpy.  If not, see <http://www.gnu.org/licenses/>.
+"""Composite classes for spectral adjustments."""
+
+import logging
+
+from satpy.composites import GenericCompositor
+from satpy.dataset import combine_metadata
+
+LOG = logging.getLogger(__name__)
+
+
+class GreenCorrector(GenericCompositor):
+    """Corrector of the FCI or AHI green band.
+
+    The green band in FCI and AHI deliberately misses the chlorophyll peak
+    in order to focus on aerosol and ash rather than on vegetation.  This
+    affects true colour RGBs, because vegetation looks brown rather than green.
+    To make vegetation look greener again, this corrector allows
+    to simulate the green band as a fraction of two or more other channels.
+
+    To be used, the composite takes two or more input channels and a parameter
+    ``fractions`` that should be a list of floats with the same length as the
+    number of channels.
+
+    For example, to simulate an FCI corrected green composite, one could use
+    a combination of 93% from the green band (vis_05) and 7% from the
+    near-infrared 0.8 µm band (vis_08)::
+
+      corrected_green:
+        compositor: !!python/name:satpy.composites.ahi.GreenCorrector
+        fractions: [0.93, 0.07]
+        prerequisites:
+          - name: vis_05
+            modifiers: [sunz_corrected, rayleigh_corrected]
+          - name: vis_08
+            modifiers: [sunz_corrected, rayleigh_corrected]
+        standard_name: toa_bidirectional_reflectance
+
+    Other examples can be found in the ``fci.yaml`` and ``ahi.yaml`` composite
+    files in the satpy distribution.
+    """
+
+    def __init__(self, *args, fractions=(0.85, 0.15), **kwargs):
+        """Set default keyword argument values."""
+        # XXX: Should this be 0.93 and 0.07
+        self.fractions = fractions
+        super(GreenCorrector, self).__init__(*args, **kwargs)
+
+    def __call__(self, projectables, optional_datasets=None, **attrs):
+        """Boost vegetation effect thanks to NIR (0.8µm) band."""
+        LOG.info('Boosting vegetation on green band')
+
+        projectables = self.match_data_arrays(projectables)
+        new_green = sum(fraction * value for fraction, value in zip(self.fractions, projectables))
+        new_green.attrs = combine_metadata(*projectables)
+        return super(GreenCorrector, self).__call__((new_green,), **attrs)

satpy/enhancements/viirs.py

@@ -38,7 +38,7 @@ def water_detection(img, **kwargs):
 @exclude_alpha
 @using_map_blocks
 def _water_detection(img_data):
-    data = np.asarray(img_data)
+    data = np.asarray(img_data).copy()
     data[data == 150] = 31
     data[data == 199] = 18
     data[data >= 200] = data[data >= 200] - 100

satpy/etc/composites/ahi.yaml

@@ -16,7 +16,7 @@ modifiers:
 
 composites:
   green:
-    compositor: !!python/name:satpy.composites.ahi.GreenCorrector
+    compositor: !!python/name:satpy.composites.spectral.GreenCorrector
     # FUTURE: Set a wavelength...see what happens. Dependency finding
     #         probably wouldn't work.
     prerequisites:
@@ -31,7 +31,7 @@ composites:
   green_true_color_reproduction:
     # JMA True Color Reproduction green band
     # http://www.jma.go.jp/jma/jma-eng/satellite/introduction/TCR.html
-    compositor: !!python/name:satpy.composites.ahi.GreenCorrector
+    compositor: !!python/name:satpy.composites.spectral.GreenCorrector
     fractions: [0.6321, 0.2928, 0.0751]
     prerequisites:
     - name: B02
@@ -43,7 +43,7 @@ composites:
     standard_name: none
 
   green_nocorr:
-    compositor: !!python/name:satpy.composites.ahi.GreenCorrector
+    compositor: !!python/name:satpy.composites.spectral.GreenCorrector
     # FUTURE: Set a wavelength...see what happens. Dependency finding
     #         probably wouldn't work.
     prerequisites:

satpy/etc/composites/ami.yaml

@@ -2,7 +2,7 @@ sensor_name: visir/ami
 
 composites:
   green_raw:
-    compositor: !!python/name:satpy.composites.ahi.GreenCorrector
+    compositor: !!python/name:satpy.composites.spectral.GreenCorrector
     prerequisites:
       - name: VI005
         modifiers: [sunz_corrected]
@@ -12,7 +12,7 @@ composites:
     fractions: [0.85, 0.15]
 
   green:
-    compositor: !!python/name:satpy.composites.ahi.GreenCorrector
+    compositor: !!python/name:satpy.composites.spectral.GreenCorrector
     prerequisites:
       - name: VI005
         modifiers: [sunz_corrected, rayleigh_corrected]
@@ -22,7 +22,7 @@ composites:
     fractions: [0.85, 0.15]
 
   green_nocorr:
-    compositor: !!python/name:satpy.composites.ahi.GreenCorrector
+    compositor: !!python/name:satpy.composites.spectral.GreenCorrector
     prerequisites:
       - name: VI005
       - name: VI008

satpy/etc/composites/fci.yaml

@@ -2,6 +2,31 @@ sensor_name: visir/fci
 
 
 composites:
+  corrected_green:
+    description: >
+      The FCI green band at 0.51 µm deliberately misses the chlorophyl band, such that
+      the signal comes rather from aerosols and ash rather than vegetation.  An effect
+      is that vegetation in a true colour RGB looks rather brown than green.  Mixing in
+      some part of the NIR 0.8 channel reduced this effect.  Note that the fractions
+      currently implemented are experimental and may change in future versions of Satpy.
+    compositor: !!python/name:satpy.composites.spectral.GreenCorrector
+    fractions: [0.93, 0.07]
+    prerequisites:
+      - name: vis_05
+        modifiers: [sunz_corrected, rayleigh_corrected]
+      - name: vis_08
+        modifiers: [sunz_corrected, rayleigh_corrected]
+    standard_name: toa_bidirectional_reflectance
+
+  corrected_green_raw:
+    description: >
+      Alternative to corrected_green, but without solar zenith or rayleigh correction.
+    compositor: !!python/name:satpy.composites.spectral.GreenCorrector
+    fractions: [0.93, 0.07]
+    prerequisites:
+      - name: vis_05
+      - name: vis_08
+    standard_name: toa_bidirectional_reflectance
 
   binary_cloud_mask:
     # This will set all clear pixels to '0', all pixles with cloudy features (meteorological/dust/ash clouds) to '1' and
@@ -11,3 +36,28 @@ composites:
       - name: 'cloud_state'
     lut: [.nan, 0, 1, 1, 1, 1, 1, 1, 0, .nan]
     standard_name: binary_cloud_mask
+
+  true_color:
+    compositor: !!python/name:satpy.composites.SelfSharpenedRGB
+    description: >
+      FCI true color composite.  The green band is simulated based on a combination of
+      channels.  This simulation may change in future versions of Satpy.  See the description
+      of the corrected_green composites for details.
+    prerequisites:
+      - name: vis_06
+        modifiers: [sunz_corrected, rayleigh_corrected]
+      - name: corrected_green
+      - name: vis_04
+        modifiers: [sunz_corrected, rayleigh_corrected]
+    standard_name: true_color
+
+  true_color_raw_with_corrected_green:
+    compositor: !!python/name:satpy.composites.SelfSharpenedRGB
+    description: >
+      FCI true color without solar zenith or rayleigh corrections, but with the
+      corrected green composite.
+    prerequisites:
+      - name: vis_06
+      - name: corrected_green_raw
+      - name: vis_04
+    standard_name: true_color_raw

satpy/etc/readers/ici_l1b_nc.yaml

@@ -12,7 +12,7 @@ reader:
     name:
       required: true
     frequency_double_sideband:
-      type: !!python/name:satpy.readers.aapp_mhs_amsub_l1c.FrequencyDoubleSideBand
+      type: !!python/name:satpy.readers.pmw_channels_definitions.FrequencyDoubleSideBand
     polarization:
       enum:
         - H

satpy/etc/readers/mws_l1b_nc.yaml

@@ -493,22 +493,6 @@ datasets:
             - mws_lon
             - mws_lat
 
-# --- Land surface data ---
-    surface_type:
-        name: surface_type
-        standard_name: surface_type
-        file_type: mws_l1b_nc
-        coordinates:
-            - mws_lon
-            - mws_lat
-    terrain_elevation:
-        name: terrain_elevation
-        standard_name: terrain_elevation
-        file_type: mws_l1b_nc
-        coordinates:
-            - mws_lon
-            - mws_lat
-
 
 file_types:
     mws_l1b_nc:

satpy/etc/readers/seadas_l2.yaml

@@ -15,41 +15,53 @@ file_types:
     - '{platform_indicator:1s}1.{start_time:%y%j.%H%M}.seadas.hdf'
     file_reader: !!python/name:satpy.readers.seadas_l2.SEADASL2HDFFileHandler
     geo_resolution: 1000
+  chlora_seadas_nc:
+    file_patterns:
+      # IMAPP-style filenames:
+      - '{platform_indicator:1s}1.{start_time:%y%j.%H%M}.seadas.nc'
+    file_reader: !!python/name:satpy.readers.seadas_l2.SEADASL2NetCDFFileHandler
+    geo_resolution: 1000
   chlora_seadas_viirs:
     # SEADAS_npp_d20211118_t1728125_e1739327.hdf
     file_patterns:
       - 'SEADAS_{platform_indicator:s}_d{start_time:%Y%m%d_t%H%M%S%f}_e{end_time:%H%M%S%f}.hdf'
     file_reader: !!python/name:satpy.readers.seadas_l2.SEADASL2HDFFileHandler
     geo_resolution: 750
+  chlora_seadas_viirs_nc:
+    # SEADAS_npp_d20211118_t1728125_e1739327.nc
+    file_patterns:
+      - 'SEADAS_{platform_indicator:s}_d{start_time:%Y%m%d_t%H%M%S%f}_e{end_time:%H%M%S%f}.nc'
+    file_reader: !!python/name:satpy.readers.seadas_l2.SEADASL2NetCDFFileHandler
+    geo_resolution: 750
 
 datasets:
   longitude:
     name: longitude
-    file_type: [chlora_seadas, seadas_hdf_viirs]
-    file_key: longitude
+    file_type: [chlora_seadas_viirs_nc, chlora_seadas_nc, seadas_hdf_viirs, chlora_sedas]
+    file_key: ["navigation_data/longitude", "longitude"]
     resolution:
       1000:
-        file_type: chlora_seadas
+        file_type: [chlora_seadas_nc, chlora_seadas]
       750:
-        file_type: chlora_seadas_viirs
+        file_type: [chlora_seadas_viirs_nc, chlora_seadas_viirs]
 
   latitude:
     name: latitude
-    file_type: [chlora_seadas, seadas_hdf_viirs]
-    file_key: latitude
+    file_type: [chlora_seadas_viirs_nc, chlora_seadas_nc, seadas_hdf_viirs, chlora_sedas]
+    file_key: ["navigation_data/latitude", "latitude"]
     resolution:
       1000:
-        file_type: chlora_seadas
+        file_type: [chlora_seadas_nc, chlora_seadas]
       750:
-        file_type: chlora_seadas_viirs
+        file_type: [chlora_seadas_viirs_nc, chlora_seadas_viirs]
 
   chlor_a:
     name: chlor_a
-    file_type: [chlora_seadas, seadas_hdf_viirs]
-    file_key: chlor_a
+    file_type: [chlora_seadas_viirs_nc, chlora_seadas_nc, seadas_hdf_viirs, chlora_sedas]
+    file_key: ["geophysical_data/chlor_a", "chlor_a"]
     resolution:
       1000:
-        file_type: chlora_seadas
+        file_type: [chlora_seadas_nc, chlora_seadas]
       750:
-        file_type: chlora_seadas_viirs
+        file_type: [chlora_seadas_viirs_nc, chlora_seadas_viirs]
     coordinates: [longitude, latitude]

satpy/etc/writers/awips_tiled.yaml

@@ -129,6 +129,12 @@ templates:
           physical_element:
             raw_value: CLAVR-x Cloud Type
           units: {}
+        encoding:
+          dtype: int16
+          _Unsigned: "true"
+          scale_factor: 0.5
+          add_offset: 0.0
+          _FillValue: -128
       clavrx_cld_temp_acha:
         reader: clavrx
         name: cld_temp_acha
@@ -153,6 +159,12 @@ templates:
           units: {}
           physical_element:
             raw_value: CLAVR-x Cloud Phase
+        encoding:
+          dtype: int16
+          _Unsigned: "true"
+          scale_factor: 0.5
+          add_offset: 0.0
+          _FillValue: -128
       clavrx_cld_opd_dcomp:
         reader: clavrx
         name: cld_opd_dcomp