Merge branch 'main' into feat_default_filename_cf_writer

.github/workflows/ci.yaml

@@ -86,6 +86,8 @@ jobs:
       - name: Install unstable dependencies
         if: matrix.experimental == true
         shell: bash -l {0}
+        # We must get LD_PRELOAD for stdlibc++ or else the manylinux wheels
+        # may break the conda-forge libraries trying to use newer glibc versions
         run: |
           python -m pip install \
           --index-url https://pypi.anaconda.org/scipy-wheels-nightly/simple/ \
@@ -101,10 +103,12 @@ jobs:
           git+https://github.com/dask/distributed \
           git+https://github.com/zarr-developers/zarr \
           git+https://github.com/Unidata/cftime \
-          git+https://github.com/mapbox/rasterio \
+          git+https://github.com/rasterio/rasterio \
           git+https://github.com/pydata/bottleneck \
           git+https://github.com/pydata/xarray \
           git+https://github.com/astropy/astropy;
+          LD_PRELOAD=$(python -c "import sys; print(sys.prefix)")/lib/libstdc++.so
+          echo "LD_PRELOAD=${LD_PRELOAD}" >> $GITHUB_ENV
 
       - name: Install satpy
         shell: bash -l {0}
@@ -114,6 +118,7 @@ jobs:
       - name: Run unit tests
         shell: bash -l {0}
         run: |
+          export LD_PRELOAD=${{ env.LD_PRELOAD }};
           pytest --cov=satpy satpy/tests --cov-report=xml --cov-report=
 
       - name: Upload unittest coverage to Codecov
@@ -133,6 +138,7 @@ jobs:
       - name: Run behaviour tests
         shell: bash -l {0}
         run: |
+          export LD_PRELOAD=${{ env.LD_PRELOAD }};
           coverage run --source=satpy -m behave satpy/tests/features --tags=-download
           coverage xml
 

.pre-commit-config.yaml

@@ -20,7 +20,7 @@ repos:
       - id: bandit
         args: [--ini, .bandit]
   - repo: https://github.com/pre-commit/mirrors-mypy
-    rev: 'v0.971'  # Use the sha / tag you want to point at
+    rev: 'v0.982'  # Use the sha / tag you want to point at
     hooks:
       - id: mypy
         additional_dependencies:

continuous_integration/environment.yaml

@@ -2,7 +2,7 @@ name: test-environment
 channels:
   - conda-forge
 dependencies:
-  - xarray
+  - xarray!=2022.9.0
   - dask
   - distributed
   - donfig

doc/source/_static/main.js

@@ -1,5 +1,6 @@
 $(document).ready( function () {
     $('table.datatable').DataTable( {
-    "paging": false
+    "paging": false,
+    "dom": 'lfitp'
 } );
 } );

doc/source/index.rst

@@ -76,29 +76,6 @@ the base Satpy installation.
 
 .. include:: reader_table.rst
 
-.. note::
-
-    Status description:
-
-    Defunct
-        Most likely the reader is not functional. If it is there is a good chance of
-        bugs and/or performance problems (e.g. not ported to dask/xarray yet). Future
-        development is unclear. Users are encouraged to contribute (see section
-        :doc:`dev_guide/CONTRIBUTING` and/or get help on Slack or by opening a Github issue).
-
-    Alpha
-        This denotes early development status. Reader is functional and implements some
-        or all of the nominal features. There might be bugs. Exactness of results is
-        not be guaranteed. Use at your own risk.
-
-    Beta
-        This denotes final developement status. Reader is functional and implements all
-        nominal features. Results should be dependable but there might be bugs. Users
-        are actively encouraged to test and report bugs.
-
-    Nominal
-        This denotes a finished status. Reader is functional and most likely no new
-        features will be introduced. It has been tested and there are no known bugs.
 
 Indices and tables
 ==================

doc/source/reader_table.py

@@ -18,6 +18,8 @@
 # along with satpy.  If not, see <http://www.gnu.org/licenses/>.
 """Module for autogenerating reader table from config files."""
 
+from yaml import BaseLoader
+
 from satpy.readers import available_readers
 
 
@@ -72,7 +74,7 @@ def generate_reader_table():
     table = [rst_table_header("Satpy Readers", header=["Description", "Reader name", "Status", "fsspec support"],
                               widths=[45, 25, 30, 30])]
 
-    reader_configs = available_readers(as_dict=True)
+    reader_configs = available_readers(as_dict=True, yaml_loader=BaseLoader)
     for rc in reader_configs:
         table.append(rst_table_row([rc.get("long_name", "").rstrip("\n"), rc.get("name", ""),
                                     rc.get("status", ""), rc.get("supports_fsspec", "false")]))

satpy/composites/ahi.py

@@ -1,6 +1,4 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-# Copyright (c) 2015-2021 Satpy developers
+# Copyright (c) 2022- Satpy developers
 #
 # This file is part of satpy.
 #
@@ -15,30 +13,8 @@
 #
 # You should have received a copy of the GNU General Public License along with
 # satpy.  If not, see <http://www.gnu.org/licenses/>.
-"""Composite classes for the AHI instrument."""
+"""Composite classes for AHI."""
 
-import logging
-
-from satpy.composites import GenericCompositor
-from satpy.dataset import combine_metadata
-
-LOG = logging.getLogger(__name__)
-
-
-class GreenCorrector(GenericCompositor):
-    """Corrector of the AHI green band to compensate for the deficit of chlorophyll signal."""
-
-    def __init__(self, *args, fractions=(0.85, 0.15), **kwargs):
-        """Set default keyword argument values."""
-        # XXX: Should this be 0.93 and 0.07
-        self.fractions = fractions
-        super(GreenCorrector, self).__init__(*args, **kwargs)
-
-    def __call__(self, projectables, optional_datasets=None, **attrs):
-        """Boost vegetation effect thanks to NIR (0.8µm) band."""
-        LOG.info('Boosting vegetation on green band')
-
-        projectables = self.match_data_arrays(projectables)
-        new_green = sum(fraction * value for fraction, value in zip(self.fractions, projectables))
-        new_green.attrs = combine_metadata(*projectables)
-        return super(GreenCorrector, self).__call__((new_green,), **attrs)
+# The green corrector used to be defined here, but was moved to spectral.py
+# in Satpy 0.38 because it also applies to FCI.
+from .spectral import GreenCorrector  # noqa: F401

satpy/composites/config_loader.py

@@ -25,11 +25,7 @@
 from typing import Callable, Iterable
 
 import yaml
-
-try:
-    from yaml import UnsafeLoader
-except ImportError:
-    from yaml import Loader as UnsafeLoader  # type: ignore
+from yaml import UnsafeLoader
 
 import satpy
 from satpy import DataID, DataQuery

satpy/composites/spectral.py

@@ -0,0 +1,70 @@
+# Copyright (c) 2015-2022 Satpy developers
+#
+# This file is part of satpy.
+#
+# satpy is free software: you can redistribute it and/or modify it under the
+# terms of the GNU General Public License as published by the Free Software
+# Foundation, either version 3 of the License, or (at your option) any later
+# version.
+#
+# satpy is distributed in the hope that it will be useful, but WITHOUT ANY
+# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
+# A PARTICULAR PURPOSE.  See the GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along with
+# satpy.  If not, see <http://www.gnu.org/licenses/>.
+"""Composite classes for spectral adjustments."""
+
+import logging
+
+from satpy.composites import GenericCompositor
+from satpy.dataset import combine_metadata
+
+LOG = logging.getLogger(__name__)
+
+
+class GreenCorrector(GenericCompositor):
+    """Corrector of the FCI or AHI green band.
+
+    The green band in FCI and AHI deliberately misses the chlorophyll peak
+    in order to focus on aerosol and ash rather than on vegetation.  This
+    affects true colour RGBs, because vegetation looks brown rather than green.
+    To make vegetation look greener again, this corrector allows
+    to simulate the green band as a fraction of two or more other channels.
+
+    To be used, the composite takes two or more input channels and a parameter
+    ``fractions`` that should be a list of floats with the same length as the
+    number of channels.
+
+    For example, to simulate an FCI corrected green composite, one could use
+    a combination of 93% from the green band (vis_05) and 7% from the
+    near-infrared 0.8 µm band (vis_08)::
+
+      corrected_green:
+        compositor: !!python/name:satpy.composites.ahi.GreenCorrector
+        fractions: [0.93, 0.07]
+        prerequisites:
+          - name: vis_05
+            modifiers: [sunz_corrected, rayleigh_corrected]
+          - name: vis_08
+            modifiers: [sunz_corrected, rayleigh_corrected]
+        standard_name: toa_bidirectional_reflectance
+
+    Other examples can be found in the ``fci.yaml`` and ``ahi.yaml`` composite
+    files in the satpy distribution.
+    """
+
+    def __init__(self, *args, fractions=(0.85, 0.15), **kwargs):
+        """Set default keyword argument values."""
+        # XXX: Should this be 0.93 and 0.07
+        self.fractions = fractions
+        super(GreenCorrector, self).__init__(*args, **kwargs)
+
+    def __call__(self, projectables, optional_datasets=None, **attrs):
+        """Boost vegetation effect thanks to NIR (0.8µm) band."""
+        LOG.info('Boosting vegetation on green band')
+
+        projectables = self.match_data_arrays(projectables)
+        new_green = sum(fraction * value for fraction, value in zip(self.fractions, projectables))
+        new_green.attrs = combine_metadata(*projectables)
+        return super(GreenCorrector, self).__call__((new_green,), **attrs)

satpy/enhancements/__init__.py

@@ -277,11 +277,6 @@ def _srgb_gamma(arr):
     return da.where(arr < 0.0031308, arr * 12.92, 1.055 * arr ** 0.41666 - 0.055)
 
 
-def _lookup_delayed(luts, band_data):
-    # can't use luts.__getitem__ for some reason
-    return luts[band_data]
-
-
 def lookup(img, **kwargs):
     """Assign values to channels based on a table."""
     luts = np.array(kwargs['luts'], dtype=np.float32) / 255.0
@@ -295,11 +290,7 @@ def _lookup_table(band_data, luts=None, index=-1):
     # NaN/null values will become 0
     lut = luts[:, index] if len(luts.shape) == 2 else luts
     band_data = band_data.clip(0, lut.size - 1).astype(np.uint8)
-
-    new_delay = dask.delayed(_lookup_delayed)(lut, band_data)
-    new_data = da.from_delayed(new_delay, shape=band_data.shape,
-                               dtype=luts.dtype)
-    return new_data
+    return lut[band_data]
 
 
 def colorize(img, **kwargs):
@@ -596,6 +587,6 @@ def btemp_threshold(img, min_in, max_in, threshold, threshold_out=None, **kwargs
 @using_map_blocks
 def _bt_threshold(band_data, threshold, high_coeffs, low_coeffs):
     # expects dask array to be passed
-    return da.where(band_data >= threshold,
+    return np.where(band_data >= threshold,
                     high_coeffs.offset - high_coeffs.factor * band_data,
                     low_coeffs.offset - low_coeffs.factor * band_data)

satpy/enhancements/viirs.py

@@ -38,7 +38,7 @@ def water_detection(img, **kwargs):
 @exclude_alpha
 @using_map_blocks
 def _water_detection(img_data):
-    data = np.asarray(img_data)
+    data = np.asarray(img_data).copy()
     data[data == 150] = 31
     data[data == 199] = 18
     data[data >= 200] = data[data >= 200] - 100

satpy/etc/composites/ahi.yaml

@@ -16,7 +16,7 @@ modifiers:
 
 composites:
   green:
-    compositor: !!python/name:satpy.composites.ahi.GreenCorrector
+    compositor: !!python/name:satpy.composites.spectral.GreenCorrector
     # FUTURE: Set a wavelength...see what happens. Dependency finding
     #         probably wouldn't work.
     prerequisites:
@@ -31,7 +31,7 @@ composites:
   green_true_color_reproduction:
     # JMA True Color Reproduction green band
     # http://www.jma.go.jp/jma/jma-eng/satellite/introduction/TCR.html
-    compositor: !!python/name:satpy.composites.ahi.GreenCorrector
+    compositor: !!python/name:satpy.composites.spectral.GreenCorrector
     fractions: [0.6321, 0.2928, 0.0751]
     prerequisites:
     - name: B02
@@ -43,7 +43,7 @@ composites:
     standard_name: none
 
   green_nocorr:
-    compositor: !!python/name:satpy.composites.ahi.GreenCorrector
+    compositor: !!python/name:satpy.composites.spectral.GreenCorrector
     # FUTURE: Set a wavelength...see what happens. Dependency finding
     #         probably wouldn't work.
     prerequisites:

satpy/etc/composites/ami.yaml

@@ -2,7 +2,7 @@ sensor_name: visir/ami
 
 composites:
   green_raw:
-    compositor: !!python/name:satpy.composites.ahi.GreenCorrector
+    compositor: !!python/name:satpy.composites.spectral.GreenCorrector
     prerequisites:
       - name: VI005
         modifiers: [sunz_corrected]
@@ -12,7 +12,7 @@ composites:
     fractions: [0.85, 0.15]
 
   green:
-    compositor: !!python/name:satpy.composites.ahi.GreenCorrector
+    compositor: !!python/name:satpy.composites.spectral.GreenCorrector
     prerequisites:
       - name: VI005
         modifiers: [sunz_corrected, rayleigh_corrected]
@@ -22,7 +22,7 @@ composites:
     fractions: [0.85, 0.15]
 
   green_nocorr:
-    compositor: !!python/name:satpy.composites.ahi.GreenCorrector
+    compositor: !!python/name:satpy.composites.spectral.GreenCorrector
     prerequisites:
       - name: VI005
       - name: VI008

satpy/etc/composites/fci.yaml

@@ -2,6 +2,31 @@ sensor_name: visir/fci
 
 
 composites:
+  corrected_green:
+    description: >
+      The FCI green band at 0.51 µm deliberately misses the chlorophyl band, such that
+      the signal comes rather from aerosols and ash rather than vegetation.  An effect
+      is that vegetation in a true colour RGB looks rather brown than green.  Mixing in
+      some part of the NIR 0.8 channel reduced this effect.  Note that the fractions
+      currently implemented are experimental and may change in future versions of Satpy.
+    compositor: !!python/name:satpy.composites.spectral.GreenCorrector
+    fractions: [0.93, 0.07]
+    prerequisites:
+      - name: vis_05
+        modifiers: [sunz_corrected, rayleigh_corrected]
+      - name: vis_08
+        modifiers: [sunz_corrected, rayleigh_corrected]
+    standard_name: toa_bidirectional_reflectance
+
+  corrected_green_raw:
+    description: >
+      Alternative to corrected_green, but without solar zenith or rayleigh correction.
+    compositor: !!python/name:satpy.composites.spectral.GreenCorrector
+    fractions: [0.93, 0.07]
+    prerequisites:
+      - name: vis_05
+      - name: vis_08
+    standard_name: toa_bidirectional_reflectance
 
   binary_cloud_mask:
     # This will set all clear pixels to '0', all pixles with cloudy features (meteorological/dust/ash clouds) to '1' and
@@ -11,3 +36,28 @@ composites:
       - name: 'cloud_state'
     lut: [.nan, 0, 1, 1, 1, 1, 1, 1, 0, .nan]
     standard_name: binary_cloud_mask
+
+  true_color:
+    compositor: !!python/name:satpy.composites.SelfSharpenedRGB
+    description: >
+      FCI true color composite.  The green band is simulated based on a combination of
+      channels.  This simulation may change in future versions of Satpy.  See the description
+      of the corrected_green composites for details.
+    prerequisites:
+      - name: vis_06
+        modifiers: [sunz_corrected, rayleigh_corrected]
+      - name: corrected_green
+      - name: vis_04
+        modifiers: [sunz_corrected, rayleigh_corrected]
+    standard_name: true_color
+
+  true_color_raw_with_corrected_green:
+    compositor: !!python/name:satpy.composites.SelfSharpenedRGB
+    description: >
+      FCI true color without solar zenith or rayleigh corrections, but with the
+      corrected green composite.
+    prerequisites:
+      - name: vis_06
+      - name: corrected_green_raw
+      - name: vis_04
+    standard_name: true_color_raw