From ffe2499ab9e2f604f6622528a2c77aa2ea219cb4 Mon Sep 17 00:00:00 2001 From: howff Date: Thu, 30 Mar 2017 13:21:16 +0100 Subject: [PATCH 01/34] Start of reader for GOES HRIT --- satpy/etc/composites/goes_imager.yaml | 2 + satpy/etc/readers/hrit_goes.yaml | 108 +++++++ satpy/readers/hrit_goes.py | 400 ++++++++++++++++++++++++++ 3 files changed, 510 insertions(+) create mode 100644 satpy/etc/composites/goes_imager.yaml create mode 100644 satpy/etc/readers/hrit_goes.yaml create mode 100644 satpy/readers/hrit_goes.py diff --git a/satpy/etc/composites/goes_imager.yaml b/satpy/etc/composites/goes_imager.yaml new file mode 100644 index 0000000000..f4cce65279 --- /dev/null +++ b/satpy/etc/composites/goes_imager.yaml @@ -0,0 +1,2 @@ +# XXX arb +sensor_name: visir/goes_imager \ No newline at end of file diff --git a/satpy/etc/readers/hrit_goes.yaml b/satpy/etc/readers/hrit_goes.yaml new file mode 100644 index 0000000000..bdaacb7f49 --- /dev/null +++ b/satpy/etc/readers/hrit_goes.yaml @@ -0,0 +1,108 @@ +reader: + description: GOES HRIT Reader + name: hrit_goes + sensors: [goes_imager] + default_channels: [00_7, 03_9, 06_6, 10_7] + reader: !!python/name:satpy.readers.yaml_reader.FileYAMLReader + +# eg. +# L-000-MSG3__-GOES13______-06_6_075W-000005___-201703261200-__ +# L-000-MSG3__-GOES13______-06_6_075W-PRO______-201703261200-__ + +file_types: + HRIT_00_7: + file_reader: !!python/name:satpy.readers.hrit_goes.HRITGOESFileHandler + file_patterns: ['{rate:1s}-000-{hrit_format:_<6s}-{platform_shortname:_<6s}______-00_7_{sublon:4s}-{segment:_<9s}-{start_time:%Y%m%d%H%M}-{compressed:_<2s}'] + requires: [HRIT_PRO_00] + + HRIT_03_9: + file_reader: !!python/name:satpy.readers.hrit_goes.HRITGOESFileHandler + file_patterns: ['{rate:1s}-000-{hrit_format:_<6s}-{platform_shortname:_<6s}______-03_9_{sublon:4s}-{segment:_<9s}-{start_time:%Y%m%d%H%M}-{compressed:_<2s}'] + requires: [HRIT_PRO_03] + + HRIT_06_6: + file_reader: !!python/name:satpy.readers.hrit_goes.HRITGOESFileHandler + file_patterns: ['{rate:1s}-000-{hrit_format:_<6s}-{platform_shortname:_<6s}______-06_6_{sublon:4s}-{segment:_<9s}-{start_time:%Y%m%d%H%M}-{compressed:_<2s}'] + requires: [HRIT_PRO_06] + + HRIT_10_7: + file_reader: !!python/name:satpy.readers.hrit_goes.HRITGOESFileHandler + file_patterns: ['{rate:1s}-000-{hrit_format:_<6s}-{platform_shortname:_<6s}______-10_7_{sublon:4s}-{segment:_<9s}-{start_time:%Y%m%d%H%M}-{compressed:_<2s}'] + requires: [HRIT_PRO_10] + + HRIT_PRO_00: + file_reader: !!python/name:satpy.readers.hrit_goes.HRITGOESPrologueFileHandler + file_patterns: ['{rate:1s}-000-{hrit_format:_<6s}-{platform_shortname:_<6s}______-00_7_{sublon:4s}-PRO______-{start_time:%Y%m%d%H%M}-{compressed:_<2s}'] + + HRIT_PRO_03: + file_reader: !!python/name:satpy.readers.hrit_goes.HRITGOESPrologueFileHandler + file_patterns: ['{rate:1s}-000-{hrit_format:_<6s}-{platform_shortname:_<6s}______-03_9_{sublon:4s}-PRO______-{start_time:%Y%m%d%H%M}-{compressed:_<2s}'] + + HRIT_PRO_06: + file_reader: !!python/name:satpy.readers.hrit_goes.HRITGOESPrologueFileHandler + file_patterns: ['{rate:1s}-000-{hrit_format:_<6s}-{platform_shortname:_<6s}______-06_6_{sublon:4s}-PRO______-{start_time:%Y%m%d%H%M}-{compressed:_<2s}'] + + HRIT_PRO_10: + file_reader: !!python/name:satpy.readers.hrit_goes.HRITGOESPrologueFileHandler + file_patterns: ['{rate:1s}-000-{hrit_format:_<6s}-{platform_shortname:_<6s}______-10_7_{sublon:4s}-PRO______-{start_time:%Y%m%d%H%M}-{compressed:_<2s}'] + +datasets: + '00_7': + name: '00_7' + resolution: 3000 + wavelength: [0.55, 0.7, 0.75] + calibration: + # reflectance: + # standard_name: toa_bidirectional_reflectance + # units: "%" + radiance: + standard_name: toa_outgoing_radiance_per_unit_wavelength + units: W m-2 um-1 sr-1 + counts: + standard_name: counts + file_type: HRIT_00_7 + + '03_9': + name: '03_9' + resolution: 3000 + wavelength: [3.8, 3.9, 4.0] + calibration: + brightness_temperature: + standard_name: brightness_temperature + units: K + # radiance: + # standard_name: toa_outgoing_radiance_per_unit_wavelength + # units: W m-2 um-1 sr-1 + counts: + standard_name: counts + file_type: HRIT_03_9 + + '06_6': + name: '06_6' + resolution: 3000 + wavelength: [6.5, 6.6, 7.0] + calibration: + brightness_temperature: + standard_name: brightness_temperature + units: K + # radiance: + # standard_name: toa_outgoing_radiance_per_unit_wavelength + # units: W m-2 um-1 sr-1 + counts: + standard_name: counts + file_type: HRIT_06_6 + + '10_7': + name: '10_7' + resolution: 3000 + wavelength: [10.2, 10.7, 11.2] + calibration: + brightness_temperature: + standard_name: brightness_temperature + units: K + # radiance: + # standard_name: toa_outgoing_radiance_per_unit_wavelength + # units: W m-2 um-1 sr-1 + counts: + standard_name: counts + file_type: HRIT_10_7 diff --git a/satpy/readers/hrit_goes.py b/satpy/readers/hrit_goes.py new file mode 100644 index 0000000000..4d99f2a976 --- /dev/null +++ b/satpy/readers/hrit_goes.py @@ -0,0 +1,400 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +# Copyright (c) 2014, 2015, 2016 Adam.Dybbroe + +# Author(s): + +# Adam.Dybbroe +# Cooke, Michael.C, UK Met Office +# Martin Raspaud + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +"""HRIT format reader. + +References: + LRIT/HRIT Mission Specific Implementation, February 2012 + GVARRDL98.pdf + 05057_SPE_MSG_LRIT_HRI +""" + +import logging +from datetime import datetime, timedelta + +import numpy as np + +from pyresample import geometry +from satpy.readers.hrit_base import (HRITFileHandler, ancillary_text, + annotation_header, base_hdr_map, + image_data_function, make_time_cds_short, + time_cds_short) + + +class CalibrationError(Exception): + pass + +logger = logging.getLogger('hrit_goes') + + +# goes implementation: +key_header = np.dtype([('key_number', 'u1'), + ('seed', '>f8')]) + +segment_identification = np.dtype([('GP_SC_ID', '>i2'), + ('spectral_channel_id', '>i1'), + ('segment_sequence_number', '>u2'), + ('planned_start_segment_number', '>u2'), + ('planned_end_segment_number', '>u2'), + ('data_field_representation', '>i1')]) + +image_segment_line_quality = np.dtype([('line_number_in_grid', '>i4'), + ('line_mean_acquisition', + [('days', '>u2'), + ('milliseconds', '>u4')]), + ('line_validity', 'u1'), + ('line_radiometric_quality', 'u1'), + ('line_geometric_quality', 'u1')]) + +goms_variable_length_headers = { + image_segment_line_quality: 'image_segment_line_quality'} + +goms_text_headers = {image_data_function: 'image_data_function', + annotation_header: 'annotation_header', + ancillary_text: 'ancillary_text'} + +goes_hdr_map = base_hdr_map.copy() +goes_hdr_map.update({7: key_header, + 128: segment_identification, + 129: image_segment_line_quality + }) + + +orbit_coef = np.dtype([('StartTime', time_cds_short), + ('EndTime', time_cds_short), + ('X', '>f8', (8, )), + ('Y', '>f8', (8, )), + ('Z', '>f8', (8, )), + ('VX', '>f8', (8, )), + ('VY', '>f8', (8, )), + ('VZ', '>f8', (8, ))]) + +attitude_coef = np.dtype([('StartTime', time_cds_short), + ('EndTime', time_cds_short), + ('XofSpinAxis', '>f8', (8, )), + ('YofSpinAxis', '>f8', (8, )), + ('ZofSpinAxis', '>f8', (8, ))]) + +cuc_time = np.dtype([('coarse', 'u1', (4, )), + ('fine', 'u1', (3, ))]) + +time_cds_expanded = np.dtype([('days', '>u2'), + ('milliseconds', '>u4'), + ('microseconds', '>u2'), + ('nanoseconds', '>u2')]) + + +def make_time_cds_expanded(tcds_array): + return (datetime(1958, 1, 1) + + timedelta(days=int(tcds_array['days']), + milliseconds=int(tcds_array['milliseconds']), + microseconds=float(tcds_array['microseconds'] + + tcds_array['nanoseconds'] / 1000.))) + +satellite_status = np.dtype([("TagType", "f4"), + ("SubSatLongitude", ">f4"), # ">f4" seems better than "f4"), + ("ReferenceDistance", ">f4"), + ("ReferenceLatitude", ">f4") + ]) + + +def recarray2dict(arr): + res = {} + for dtuple in arr.dtype.descr: + key = dtuple[0] + ntype = dtuple[1] + data = arr[key] + if isinstance(ntype, list): + res[key] = recarray2dict(data) + else: + res[key] = data + + return res + + +class HRITGOESPrologueFileHandler(HRITFileHandler): + + """GOES HRIT format reader + """ + + def __init__(self, filename, filename_info, filetype_info): + """Initialize the reader.""" + super(HRITGOESPrologueFileHandler, self).__init__(filename, filename_info, + filetype_info, + (goes_hdr_map, + goms_variable_length_headers, + goms_text_headers)) + + self.prologue = {} + self.read_prologue() + + def read_prologue(self): + """Read the prologue metadata.""" + with open(self.filename, "rb") as fp_: + fp_.seek(self.mda['total_header_length']) + data = np.fromfile(fp_, dtype=prologue, count=1)[0] + + self.prologue.update(recarray2dict(data)) + + self.process_prologue() + + def process_prologue(self): + """Reprocess prologue to correct types.""" + pass + + +radiometric_processing = np.dtype([("TagType", " Date: Thu, 23 Nov 2017 11:19:59 +0000 Subject: [PATCH 02/34] fix available_composite_names in doc --- doc/source/quickstart.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/source/quickstart.rst b/doc/source/quickstart.rst index 74815a3bbd..82d0a9a95d 100644 --- a/doc/source/quickstart.rst +++ b/doc/source/quickstart.rst @@ -113,7 +113,7 @@ The easiest way to generate composites is to `load` them:: To get a list of all available composites for the current scene:: - >>> global_scene.available_composites() + >>> global_scene.available_composite_names() [u'overview_sun', u'airmass', From fedf1035290e60d95b12e02c6bf33eb947442b2a Mon Sep 17 00:00:00 2001 From: Mikhail Itkin Date: Wed, 6 Dec 2017 14:52:52 +0100 Subject: [PATCH 03/34] Add file pattern for MODIS L1B from LAADS WEB NASA's LAADS WEB pattern is slightly different --- satpy/etc/readers/hdfeos_l1b.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/satpy/etc/readers/hdfeos_l1b.yaml b/satpy/etc/readers/hdfeos_l1b.yaml index 7148be5f1e..6ecb49a972 100644 --- a/satpy/etc/readers/hdfeos_l1b.yaml +++ b/satpy/etc/readers/hdfeos_l1b.yaml @@ -422,6 +422,7 @@ file_types: file_patterns: - 'M{platform_indicator:1s}D02{resolution:1s}km_A{start_time:%y%j_%H%M%S}_{processing_time:%Y%j%H%M%S}.hdf' - 'M{platform_indicator:1s}D02{resolution:1s}km.A{start_time:%Y%j.%H%M}.{collection:03d}.{processing_time:%Y%j%H%M%S}.h5' + - 'M{platform_indicator:1s}D02{resolution:1s}KM.A{start_time:%Y%j.%H%M}.{collection:03d}.{processing_time:%Y%j%H%M%S}.hdf' - '{platform_indicator:1s}1.{start_time:%y%j.%H%M}.{resolution:s}.hdf' file_reader: !!python/name:satpy.readers.hdfeos_l1b.HDFEOSBandReader '' hdf_eos_geo: From 6fb3ce69a0b964c7a44cd00a14338d1d1167f257 Mon Sep 17 00:00:00 2001 From: Martin Raspaud Date: Thu, 7 Dec 2017 16:48:20 +0100 Subject: [PATCH 04/34] Bugfix epsl1b reader --- satpy/readers/eps_l1b.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/satpy/readers/eps_l1b.py b/satpy/readers/eps_l1b.py index 2355f2584c..b00724ed12 100644 --- a/satpy/readers/eps_l1b.py +++ b/satpy/readers/eps_l1b.py @@ -302,7 +302,7 @@ def get_dataset(self, key, info): if key.calibration == 'reflectance': array = np.ma.array( radiance_to_refl(self["SCENE_RADIANCES"][:, 1, :], - self["CH1_SOLAR_FILTERED_IRRADIANCE"])) + self["CH2_SOLAR_FILTERED_IRRADIANCE"])) else: array = np.ma.array( self["SCENE_RADIANCES"][:, 1, :]) @@ -311,7 +311,7 @@ def get_dataset(self, key, info): if key.calibration == 'reflectance': array = np.ma.array( radiance_to_refl(self["SCENE_RADIANCES"][:, 2, :], - self["CH2_SOLAR_FILTERED_IRRADIANCE"])) + self["CH3A_SOLAR_FILTERED_IRRADIANCE"])) else: array = np.ma.array(self["SCENE_RADIANCES"][:, 2, :]) From dd6813f826cc118d94c863760ca45632dc74db83 Mon Sep 17 00:00:00 2001 From: Martin Raspaud Date: Thu, 7 Dec 2017 19:59:13 +0100 Subject: [PATCH 05/34] Add github templates for issues and PRs --- .github/ISSUE_TEMPLATE.md | 18 ++++++++++++++++++ .github/PULL_REQUEST_TEMPLATE.md | 7 +++++++ 2 files changed, 25 insertions(+) create mode 100644 .github/ISSUE_TEMPLATE.md create mode 100644 .github/PULL_REQUEST_TEMPLATE.md diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md new file mode 100644 index 0000000000..18c258ad67 --- /dev/null +++ b/.github/ISSUE_TEMPLATE.md @@ -0,0 +1,18 @@ +#### Code Sample, a minimal, complete, and verifiable piece of code + +```python +# Your code here + +``` +#### Problem description + +[this should also explain **why** the current behaviour is a problem and why the +expected output is a better solution.] + +#### Expected Output + +#### Actual Result, Traceback if applicable + +#### Versions of Python, package at hand and relevant dependencies + +Thank you for reporting an issue ! diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md new file mode 100644 index 0000000000..670a685b88 --- /dev/null +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -0,0 +1,7 @@ +Please make the PR against the `develop` branch. + + - [ ] Closes #xxxx (remove if there is no corresponding issue, which should only be the case for minor changes) + - [ ] Tests added (for all bug fixes or enhancements) + - [ ] Tests passed (for all non-documentation changes) + - [ ] Passes ``git diff origin/develop **/*py | flake8 --diff`` (remove if you did not edit any Python files) + - [ ] Fully documented (remove if this change should not be visible to users, e.g., if it is an internal clean-up, or if this is part of a larger project that will be documented later) From 9e49563119b129cf32ba927315da20ab2fcdeefa Mon Sep 17 00:00:00 2001 From: Panu Lahtinen Date: Fri, 8 Dec 2017 09:41:08 +0200 Subject: [PATCH 06/34] Remove sun zenith angle correction from IR channels --- satpy/etc/readers/viirs_sdr.yaml | 7 ------- 1 file changed, 7 deletions(-) diff --git a/satpy/etc/readers/viirs_sdr.yaml b/satpy/etc/readers/viirs_sdr.yaml index af0623aeff..00e75d3570 100644 --- a/satpy/etc/readers/viirs_sdr.yaml +++ b/satpy/etc/readers/viirs_sdr.yaml @@ -93,7 +93,6 @@ datasets: I04: name: I04 wavelength: [3.580, 3.740, 3.900] - modifiers: [sunz_corrected] file_type: svi04 resolution: 371 coordinates: [i_longitude, i_latitude] @@ -107,7 +106,6 @@ datasets: I05: name: I05 wavelength: [10.500, 11.450, 12.300] - modifiers: [sunz_corrected] file_type: svi05 resolution: 371 coordinates: [i_longitude, i_latitude] @@ -275,7 +273,6 @@ datasets: M12: name: M12 wavelength: [3.610, 3.700, 3.790] - modifiers: [sunz_corrected] file_type: svm12 resolution: 742 coordinates: [m_longitude, m_latitude] @@ -289,7 +286,6 @@ datasets: M13: name: M13 wavelength: [3.973, 4.050, 4.128] - modifiers: [sunz_corrected] file_type: svm13 resolution: 742 coordinates: [m_longitude, m_latitude] @@ -303,7 +299,6 @@ datasets: M14: name: M14 wavelength: [8.400, 8.550, 8.700] - modifiers: [sunz_corrected] file_type: svm14 resolution: 742 coordinates: [m_longitude, m_latitude] @@ -317,7 +312,6 @@ datasets: M15: name: M15 wavelength: [10.263, 10.763, 11.263] - modifiers: [sunz_corrected] file_type: svm15 resolution: 742 coordinates: [m_longitude, m_latitude] @@ -331,7 +325,6 @@ datasets: M16: name: M16 wavelength: [11.538, 12.013, 12.489] - modifiers: [sunz_corrected] file_type: svm16 resolution: 742 coordinates: [m_longitude, m_latitude] From 319f3057ad53e48310409e49360a03706b7512b7 Mon Sep 17 00:00:00 2001 From: davidh-ssec Date: Mon, 11 Dec 2017 08:59:34 -0600 Subject: [PATCH 07/34] Fix SCMI writer not overwriting data from previous tiles --- satpy/writers/scmi.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/satpy/writers/scmi.py b/satpy/writers/scmi.py index 202f8f3670..626277d9ad 100644 --- a/satpy/writers/scmi.py +++ b/satpy/writers/scmi.py @@ -248,7 +248,7 @@ def __call__(self, data, fill_value=np.nan): ts = self.tile_shape tmp_tile = np.ma.zeros(ts, dtype=np.float32) tmp_tile.set_fill_value(fill_value) - tmp_tile[:] = fill_value + tmp_tile[:] = np.ma.masked if self._tile_cache: tile_infos = self._tile_cache @@ -262,6 +262,7 @@ def __call__(self, data, fill_value=np.nan): continue yield tile_info[:-2], tmp_tile + tmp_tile[:] = np.ma.masked class LetteredTileGenerator(NumberedTileGenerator): From 217cce9ae2dd93014b92a58e9e2ab26db6f83800 Mon Sep 17 00:00:00 2001 From: davidh-ssec Date: Mon, 11 Dec 2017 09:15:21 -0600 Subject: [PATCH 08/34] Remove old and unused mipp_xrit reader --- doc/source/index.rst | 3 - doc/source/satpy.readers.rst | 8 - satpy/etc/readers/mipp_h8.yaml | 236 ----------------------------- satpy/etc/readers/mipp_xrit.yaml | 177 ---------------------- satpy/readers/mipp_xrit.py | 252 ------------------------------- satpy/tests/test_readers.py | 226 --------------------------- 6 files changed, 902 deletions(-) delete mode 100644 satpy/etc/readers/mipp_h8.yaml delete mode 100644 satpy/etc/readers/mipp_xrit.yaml delete mode 100644 satpy/readers/mipp_xrit.py diff --git a/doc/source/index.rst b/doc/source/index.rst index 9e1404d84c..4b0a200083 100644 --- a/doc/source/index.rst +++ b/doc/source/index.rst @@ -39,9 +39,6 @@ distribution, we provide support for the following readers: * - GOES 16 imager data in netcdf format - `abi_l1b` - Nominal - * - GOES 11 to 15 imager data in HRIT format - - `mipp_xrit` - - Nominal * - Electro-L N2 MSU-GS data in HRIT format - `hrit_electrol` - Nominal diff --git a/doc/source/satpy.readers.rst b/doc/source/satpy.readers.rst index 9ed7ebb2db..702fa5ea91 100644 --- a/doc/source/satpy.readers.rst +++ b/doc/source/satpy.readers.rst @@ -49,14 +49,6 @@ satpy.readers.clavrx module :undoc-members: :show-inheritance: -satpy.readers.mipp_xrit module ------------------------------- - -.. automodule:: satpy.readers.mipp_xrit - :members: - :undoc-members: - :show-inheritance: - satpy.readers.viirs_l1b module ------------------------------ diff --git a/satpy/etc/readers/mipp_h8.yaml b/satpy/etc/readers/mipp_h8.yaml deleted file mode 100644 index 221b7dff56..0000000000 --- a/satpy/etc/readers/mipp_h8.yaml +++ /dev/null @@ -1,236 +0,0 @@ -reader: - description: HRIT Reader for Himawari-8 data - name: mipp_h8 - platform_name: Himawari-8 - sensors: [ahi] - default_channels: [] - reader: !!python/name:satpy.readers.mipp_xrit.xRITFile '' - -file_types: - hrit: - file_patterns: ['IMG_{domain:4s}{dataset_name}_{start_time:%Y%m%d%H%M}_{segment:03d}'] - -datasets: - B01: - name: B01 - sensor: ahi - wavelength: [0.45,0.47,0.49] - resolution: 2000 - calibration: - reflectance: - standard_name: toa_bidirectional_reflectance - units: "%" - radiance: - standard_name: toa_outgoing_radiance_per_unit_wavelength - units: W m-2 um-1 sr-1 - file_type: hrit - - B02: - name: B02 - sensor: ahi - wavelength: [0.49,0.51,0.53] - resolution: 2000 - calibration: - reflectance: - standard_name: toa_bidirectional_reflectance - units: "%" - radiance: - standard_name: toa_outgoing_radiance_per_unit_wavelength - units: W m-2 um-1 sr-1 - file_type: hrit - - B03: - name: VIS - sensor: ahi - wavelength: [0.62,0.64,0.66] - resolution: 2000 - calibration: - reflectance: - standard_name: toa_bidirectional_reflectance - units: "%" - radiance: - standard_name: toa_outgoing_radiance_per_unit_wavelength - units: W m-2 um-1 sr-1 - file_type: hrit - - B04: - name: B04 - sensor: ahi - wavelength: [0.83, 0.85, 0.87] - resolution: 2000 - calibration: - reflectance: - standard_name: toa_bidirectional_reflectance - units: "%" - radiance: - standard_name: toa_outgoing_radiance_per_unit_wavelength - units: W m-2 um-1 sr-1 - file_type: hrit - - B05: - name: B05 - sensor: ahi - wavelength: [1.5, 1.6, 1.7] - resolution: 2000 - calibration: - reflectance: - standard_name: toa_bidirectional_reflectance - units: "%" - radiance: - standard_name: toa_outgoing_radiance_per_unit_wavelength - units: W m-2 um-1 sr-1 - file_type: hrit - - B06: - name: B06 - sensor: ahi - wavelength: [2.2, 2.3, 2.4] - resolution: 2000 - calibration: - reflectance: - standard_name: toa_bidirectional_reflectance - units: "%" - radiance: - standard_name: toa_outgoing_radiance_per_unit_wavelength - units: W m-2 um-1 sr-1 - file_type: hrit - - B07: - name: IR4 - sensor: ahi - wavelength: [3.7, 3.9, 4.1] - resolution: 2000 - calibration: - brightness_temperature: - standard_name: toa_brightness_temperature - units: "K" - radiance: - standard_name: toa_outgoing_radiance_per_unit_wavelength - units: W m-2 um-1 sr-1 - file_type: hrit - - B08: - name: IR3 - sensor: ahi - wavelength: [6.0, 6.2, 6.4] - resolution: 2000 - calibration: - brightness_temperature: - standard_name: toa_brightness_temperature - units: "K" - radiance: - standard_name: toa_outgoing_radiance_per_unit_wavelength - units: W m-2 um-1 sr-1 - file_type: hrit - - B09: - name: B09 - sensor: ahi - wavelength: [6.7, 6.9, 7.1] - resolution: 2000 - calibration: - brightness_temperature: - standard_name: toa_brightness_temperature - units: "K" - radiance: - standard_name: toa_outgoing_radiance_per_unit_wavelength - units: W m-2 um-1 sr-1 - file_type: hrit - - B10: - name: B10 - sensor: ahi - wavelength: [7.1, 7.3, 7.5] - resolution: 2000 - calibration: - brightness_temperature: - standard_name: toa_brightness_temperature - units: "K" - radiance: - standard_name: toa_outgoing_radiance_per_unit_wavelength - units: W m-2 um-1 sr-1 - file_type: hrit - - B11: - name: B11 - sensor: ahi - wavelength: [8.4, 8.6, 8.8] - resolution: 2000 - calibration: - brightness_temperature: - standard_name: toa_brightness_temperature - units: "K" - radiance: - standard_name: toa_outgoing_radiance_per_unit_wavelength - units: W m-2 um-1 sr-1 - file_type: hrit - - B12: - name: B12 - sensor: ahi - wavelength: [9.4, 9.6, 9.8] - resolution: 2000 - calibration: - brightness_temperature: - standard_name: toa_brightness_temperature - units: "K" - radiance: - standard_name: toa_outgoing_radiance_per_unit_wavelength - units: W m-2 um-1 sr-1 - file_type: hrit - - B13: - name: IR1 - sensor: ahi - wavelength: [10.2, 10.4, 10.6] - resolution: 2000 - calibration: - brightness_temperature: - standard_name: toa_brightness_temperature - units: "K" - radiance: - standard_name: toa_outgoing_radiance_per_unit_wavelength - units: W m-2 um-1 sr-1 - file_type: hrit - - B14: - name: B14 - sensor: ahi - wavelength: [11.0, 11.2, 11.4] - resolution: 2000 - calibration: - brightness_temperature: - standard_name: toa_brightness_temperature - units: "K" - radiance: - standard_name: toa_outgoing_radiance_per_unit_wavelength - units: W m-2 um-1 sr-1 - file_type: hrit - - B15: - name: IR2 - sensor: ahi - wavelength: [12.2, 12.4, 12.6] - resolution: 2000 - calibration: - brightness_temperature: - standard_name: toa_brightness_temperature - units: "K" - radiance: - standard_name: toa_outgoing_radiance_per_unit_wavelength - units: W m-2 um-1 sr-1 - file_type: hrit - - B16: - name: B16 - sensor: ahi - wavelength: [13.1, 13.3, 13.5] - resolution: 2000 - calibration: - brightness_temperature: - standard_name: toa_brightness_temperature - units: "K" - radiance: - standard_name: toa_outgoing_radiance_per_unit_wavelength - units: W m-2 um-1 sr-1 - file_type: hrit diff --git a/satpy/etc/readers/mipp_xrit.yaml b/satpy/etc/readers/mipp_xrit.yaml deleted file mode 100644 index a239ed002a..0000000000 --- a/satpy/etc/readers/mipp_xrit.yaml +++ /dev/null @@ -1,177 +0,0 @@ -reader: - description: Generic HRIT/LRIT Reader - name: mipp_xrit - sensors: [seviri] - default_channels: [HRV, IR_016, IR_039, IR_087, IR_097, IR_108, IR_120, IR_134, VIS006, VIS008, WV_062, WV_073] - reader: !!python/name:satpy.readers.mipp_xrit.xRITFile '' - -file_types: - xRIT: - file_patterns: ['{rate:1s}-000-{hrit_format:_<6s}-{platform_shortname:_<12s}-{dataset_name:_<9s}-{segment:_<9s}-{start_time:%Y%m%d%H%M}-{compressed:_<2s}'] - -datasets: - HRV: - name: HRV - resolution: 1000.134348869 - wavelength: [0.5, 0.7, 0.9] - calibration: - reflectance: - standard_name: toa_bidirectional_reflectance - units: "%" - radiance: - standard_name: toa_outgoing_radiance_per_unit_wavelength - units: W m-2 um-1 sr-1 - counts: - standard_name: counts - file_type: xRIT - - IR_016: - name: IR_016 - resolution: 3000.403165817 - wavelength: [1.5, 1.64, 1.78] - calibration: - reflectance: - standard_name: reflectance - radiance: - standard_name: radiance - counts: - standard_name: counts - file_type: xRIT - - IR_039: - name: IR_039 - resolution: 3000.403165817 - wavelength: [3.48, 3.92, 4.36] - calibration: - brightness_temperature: - standard_name: brightness_temperature - radiance: - standard_name: radiance - counts: - standard_name: counts - file_type: xRIT - - IR_087: - name: IR_087 - resolution: 3000.403165817 - wavelength: [8.3, 8.7, 9.1] - calibration: - brightness_temperature: - standard_name: brightness_temperature - radiance: - standard_name: radiance - counts: - standard_name: counts - file_type: xRIT - - IR_097: - name: IR_097 - resolution: 3000.403165817 - wavelength: [9.38, 9.66, 9.94] - calibration: - brightness_temperature: - standard_name: brightness_temperature - radiance: - standard_name: radiance - counts: - standard_name: counts - file_type: xRIT - - IR_108: - name: IR_108 - resolution: 3000.403165817 - wavelength: [9.8, 10.8, 11.8] - calibration: - brightness_temperature: - standard_name: brightness_temperature - radiance: - standard_name: radiance - counts: - standard_name: counts - file_type: xRIT - - IR_120: - name: IR_120 - resolution: 3000.403165817 - wavelength: [11.0, 12.0, 13.0] - calibration: - brightness_temperature: - standard_name: brightness_temperature - radiance: - standard_name: radiance - counts: - standard_name: counts - file_type: xRIT - - IR_134: - name: IR_134 - resolution: 3000.403165817 - wavelength: [12.4, 13.4, 14.4] - calibration: - brightness_temperature: - standard_name: brightness_temperature - radiance: - standard_name: radiance - counts: - standard_name: counts - file_type: xRIT - - VIS006: - name: VIS006 - resolution: 3000.403165817 - wavelength: [0.56, 0.635, 0.71] - calibration: - reflectance: - standard_name: toa_bidirectional_reflectance - units: "%" - radiance: - standard_name: toa_outgoing_radiance_per_unit_wavelength - units: W m-2 um-1 sr-1 - counts: - standard_name: counts - file_type: xRIT - - VIS008: - name: VIS008 - resolution: 3000.403165817 - wavelength: [0.74, 0.81, 0.88] - calibration: - reflectance: - standard_name: toa_bidirectional_reflectance - units: "%" - radiance: - standard_name: toa_outgoing_radiance_per_unit_wavelength - units: W m-2 um-1 sr-1 - counts: - standard_name: counts - file_type: xRIT - - WV_062: - name: WV_062 - resolution: 3000.403165817 - wavelength: [5.35, 6.25, 7.15] - calibration: - brightness_temperature: - standard_name: toa_brightness_temperature - units: "K" - radiance: - standard_name: toa_outgoing_radiance_per_unit_wavelength - units: W m-2 um-1 sr-1 - counts: - standard_name: counts - file_type: xRIT - - WV_073: - name: WV_073 - resolution: 3000.403165817 - wavelength: [6.85, 7.35, 7.85] - calibration: - brightness_temperature: - standard_name: toa_brightness_temperature - units: "K" - radiance: - standard_name: toa_outgoing_radiance_per_unit_wavelength - units: W m-2 um-1 sr-1 - counts: - standard_name: counts - file_type: xRIT diff --git a/satpy/readers/mipp_xrit.py b/satpy/readers/mipp_xrit.py deleted file mode 100644 index da47bda37d..0000000000 --- a/satpy/readers/mipp_xrit.py +++ /dev/null @@ -1,252 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -# Copyright (c) 2010-2016. - -# Author(s): - -# Martin Raspaud -# Esben S. Nielsen -# Panu Lahtinen -# Adam Dybbroe - -# This file is part of satpy. - -# satpy is free software: you can redistribute it and/or modify it under the -# terms of the GNU General Public License as published by the Free Software -# Foundation, either version 3 of the License, or (at your option) any later -# version. - -# satpy is distributed in the hope that it will be useful, but WITHOUT ANY -# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR -# A PARTICULAR PURPOSE. See the GNU General Public License for more details. - -# You should have received a copy of the GNU General Public License along with -# satpy. If not, see . -"""Interface to Eumetcast level 1.5 HRIT/LRIT format. Uses the MIPP reader. -""" -import logging -import os -from fnmatch import fnmatch - -from mipp import CalibrationError, ReaderError, xrit -from satpy.dataset import Dataset -from satpy.readers import DatasetDict -from satpy.readers.helper_functions import area_defs_to_extent -from satpy.readers.yaml_reader import AbstractYAMLReader -from trollsift.parser import globify, parse - -LOGGER = logging.getLogger(__name__) - -try: - # Work around for on demand import of pyresample. pyresample depends - # on scipy.spatial which memory leaks on multiple imports - IS_PYRESAMPLE_LOADED = False - from pyresample import geometry - IS_PYRESAMPLE_LOADED = True -except ImportError: - LOGGER.warning("pyresample missing. Can only work in satellite projection") - - -class xRITFile(AbstractYAMLReader): - '''Class for reading XRIT data. - ''' - - def __init__(self, config_files, - start_time=None, - end_time=None, - area=None): - super(xRITFile, self).__init__(config_files, - start_time=start_time, - end_time=end_time, - area=area) - self.info['filenames'] = [] - self.file_patterns = [] - for file_type in self.config['file_types'].values(): - self.file_patterns.extend(file_type['file_patterns']) - - @property - def start_time(self): - return self._start_time - - @property - def end_time(self): - return self._end_time - - def load(self, dataset_keys, area=None, start_time=None, end_time=None): - image_files = [] - pattern = self.file_patterns[0] - prologue_file = None - epilogue_file = None - for filename in self.info['filenames']: - try: - file_info = parse(pattern, os.path.basename(filename)) - except ValueError: - continue - if file_info["segment"] == "EPI": - epilogue_file = filename - elif file_info["segment"] == "PRO": - prologue_file = filename - else: - image_files.append(filename) - - start_times = set() - datasets = DatasetDict() - area_converted_to_extent = False - area_extent = None - for ds in dataset_keys: - - channel_files = [] - for filename in image_files: - file_info = parse(pattern, os.path.basename(filename)) - if file_info["dataset_name"] == ds.name: - channel_files.append(filename) - start_times.add(file_info['start_time']) - - if not channel_files: - continue - kwargs = {} - if 'platform_name' in self.info: - kwargs['platform_name'] = self.info['platform_name'] - # Convert area definitions to maximal area_extent - if not area_converted_to_extent and area is not None: - metadata = xrit.sat.load_files(prologue_file, - channel_files, - epilogue_file, - only_metadata=True, - **kwargs) - # otherwise use the default value (MSG3 extent at - # lon0=0.0), that is, do not pass default_extent=area_extent - area_extent = area_defs_to_extent( - [area], metadata.proj4_params) - area_converted_to_extent = True - - try: - calibrate = 1 - if ds.calibration == 'counts': - calibrate = 0 - elif ds.calibration == 'radiance': - calibrate = 2 - image = xrit.sat.load_files(prologue_file, - channel_files, - epilogue_file, - mask=True, - calibrate=calibrate, - **kwargs) - if area_extent: - metadata, data = image(area_extent) - else: - metadata, data = image() - except CalibrationError: - LOGGER.warning( - "Loading non calibrated data since calibration failed.") - image = xrit.sat.load_files(prologue_file, - channel_files, - epilogue_file, - mask=True, - calibrate=False, - **kwargs) - if area_extent: - metadata, data = image(area_extent) - else: - metadata, data = image() - - except ReaderError as err: - # if dataset can't be found, go on with next dataset - LOGGER.error(str(err)) - continue - if len(metadata.instruments) != 1: - sensor = None - else: - sensor = metadata.instruments[0] - - units = {'ALBEDO(%)': '%', - 'KELVIN': 'K'} - - standard_names = {'1': 'counts', - 'W m-2 sr-1 m-1': - 'toa_outgoing_radiance_per_unit_wavelength', - '%': 'toa_bidirectional_reflectance', - 'K': - 'toa_brightness_temperature'} - - unit = units.get(metadata.calibration_unit, - metadata.calibration_unit) - projectable = Dataset( - data, - name=ds.name, - units=unit, - standard_name=standard_names[unit], - sensor=sensor, - start_time=min(start_times), - id=ds) - - # Build an area on the fly from the mipp metadata - proj_params = getattr(metadata, "proj4_params").split(" ") - proj_dict = {} - for param in proj_params: - key, val = param.split("=") - proj_dict[key] = val - - if IS_PYRESAMPLE_LOADED: - # Build area_def on-the-fly - projectable.info["area"] = geometry.AreaDefinition( - str(metadata.area_extent) + str(data.shape), - "On-the-fly area", proj_dict["proj"], proj_dict, - data.shape[1], data.shape[0], metadata.area_extent) - else: - LOGGER.info("Could not build area, pyresample missing...") - - datasets[ds] = projectable - - return datasets - - def select_files(self, - base_dir=None, - filenames=None, - sensor=None): - file_set, info_filenames = super(xRITFile, self).select_files( - base_dir, filenames, sensor) - - # for pattern in self.file_patterns: - # for filename in filenames: - # parse(pattern, os.path.basename(filename)) - - matching_filenames = [] - - # Organize filenames in to file types and create file handlers - remaining_filenames = set(self.info['filenames']) - start_times = [] - end_times = [] - for filetype, filetype_info in self.config['file_types'].items(): - patterns = filetype_info['file_patterns'] - for pattern in patterns: - used_filenames = set() - for filename in remaining_filenames: - if fnmatch(os.path.basename(filename), globify(pattern)): - # we know how to use this file (even if we may not use - # it later) - used_filenames.add(filename) - filename_info = parse(pattern, - os.path.basename(filename)) - # Only add this file handler if it is within the time - # we want - file_start = filename_info['start_time'] - file_end = filename_info.get('end_time', file_start) - if self._start_time and file_start < self._start_time: - continue - if self._end_time and file_end > self._end_time: - continue - - start_times.append(file_start) - end_times.append(file_end) - matching_filenames.append(filename) - # TODO: Area filtering - - remaining_filenames -= used_filenames - - if matching_filenames: - # Assign the start time and end time - self._start_time = min(start_times) - self._end_time = max(end_times) - self.info['filenames'] = matching_filenames - return file_set, info_filenames diff --git a/satpy/tests/test_readers.py b/satpy/tests/test_readers.py index f7275ae840..9be5103b07 100644 --- a/satpy/tests/test_readers.py +++ b/satpy/tests/test_readers.py @@ -181,237 +181,11 @@ def test_reader_name_base_dir(self): os.remove(fn) -class TestReaders(unittest.TestCase): - '''Class for testing satpy.satin''' - - # def test_lonlat_to_geo_extent(self): - # '''Test conversion of longitudes and latitudes to area extent.''' - # - # # MSG3 proj4 string from - # # xrit.sat.load(..., only_metadata=True).proj4_params - # proj4_str = 'proj=geos lon_0=0.00 lat_0=0.00 ' \ - # 'a=6378169.00 b=6356583.80 h=35785831.00' - # - # # MSG3 maximum extent - # max_extent=(-5567248.07, -5570248.48, - # 5570248.48, 5567248.07) - # - # # Few area extents in longitudes/latitudes - # area_extents_ll = [[-68.328121068060341, # left longitude - # 18.363816196771392, # down latitude - # 74.770372053870972, # right longitude - # 75.66494585661934], # up latitude - # # all corners outside Earth's disc - # [1e30, 1e30, 1e30, 1e30] - # ] - # - # # And corresponding correct values in GEO projection - # geo_extents = [[-5010596.02, 1741593.72, 5570248.48, 5567248.07], - # [-5567248.07, -5570248.48, 5570248.48, 5567248.07]] - # - # for i in range(len(area_extents_ll)): - # res = satpy.satin.mipp_xrit.lonlat_to_geo_extent(area_extents_ll[i], - # proj4_str, - # max_extent=\ - # max_extent) - # for j in range(len(res)): - # self.assertAlmostEqual(res[j], geo_extents[i][j], 2) - - ## FIXME replace the following with tests on reader.select_files - # - # @mock.patch("glob.glob") - # def test_find_sensors_readers_single_sensor_no_files(self, glob_mock, - # **mock_objs): - # from satpy.scene import Scene - # from satpy.readers import ReaderFinder - # glob_mock.return_value = ["valid", "no_found_files", "not_valid"] - # - # def fake_read_config(config_file): - # if config_file in ["valid", "no_found_files"]: - # return {"name": "fake_reader", - # "sensor": ["foo"], - # "config_files": config_file} - # else: - # raise ValueError("Fake ValueError") - # - # def fake_get_filenames(reader_info): - # if reader_info["config_files"] == "valid": - # return ["file1", "file2"] - # return [] - # - # with mock.patch.multiple("satpy.readers.ReaderFinder", - # read_reader_config=mock.DEFAULT, - # get_filenames=mock.DEFAULT, - # load_reader=mock.DEFAULT) as mock_objs: - # mock_objs["read_reader_config"].side_effect = fake_read_config - # mock_objs["get_filenames"].side_effect = fake_get_filenames - # - # scn = Scene() - # finder = ReaderFinder(scn) - # finder._find_sensors_readers("foo", None) - - # def test_get_filenames_with_start_time_provided(self): - # from satpy.scene import Scene - # from satpy.readers import ReaderFinder - # from datetime import datetime - # scn = Scene() - # finder = ReaderFinder(scn) - # reader_info = {"file_patterns": ["foo"], - # "start_time": datetime(2015, 6, 24, 0, 0)} - # - # with mock.patch("satpy.readers.glob.iglob") as mock_iglob: - # mock_iglob.return_value = ["file1", "file2", "file3", "file4", - # "file5"] - # with mock.patch("satpy.readers.Parser") as mock_parser: - # mock_parser.return_value.parse.side_effect = [ - # {"start_time": datetime(2015, 6, 23, 23, 57), # file1 - # "end_time": datetime(2015, 6, 23, 23, 59)}, - # {"start_time": datetime(2015, 6, 23, 23, 59), # file2 - # "end_time": datetime(2015, 6, 24, 0, 1)}, - # {"start_time": datetime(2015, 6, 24, 0, 1), # file3 - # "end_time": datetime(2015, 6, 24, 0, 3)}, - # {"start_time": datetime(2015, 6, 24, 0, 3), # file4 - # "end_time": datetime(2015, 6, 24, 0, 5)}, - # {"start_time": datetime(2015, 6, 24, 0, 5), # file5 - # "end_time": datetime(2015, 6, 24, 0, 7)}, - # ] - # self.assertEqual(finder.get_filenames(reader_info), ["file2"]) - # - - # def test_get_filenames_with_start_time_and_end_time(self): - # from satpy.scene import Scene - # from satpy.readers import ReaderFinder - # from datetime import datetime - # scn = Scene() - # finder = ReaderFinder(scn) - # reader_info = {"file_patterns": ["foo"], - # "start_time": datetime(2015, 6, 24, 0, 0), - # "end_time": datetime(2015, 6, 24, 0, 6)} - # with mock.patch("satpy.readers.glob.iglob") as mock_iglob: - # mock_iglob.return_value = ["file1", "file2", "file3", "file4", "file5"] - # with mock.patch("satpy.readers.Parser") as mock_parser: - # mock_parser.return_value.parse.side_effect = [{"start_time": datetime(2015, 6, 23, 23, 57), # file1 - # "end_time": datetime(2015, 6, 23, 23, 59)}, - # {"start_time": datetime(2015, 6, 23, 23, 59), # file2 - # "end_time": datetime(2015, 6, 24, 0, 1)}, - # {"start_time": datetime(2015, 6, 24, 0, 1), # file3 - # "end_time": datetime(2015, 6, 24, 0, 3)}, - # {"start_time": datetime(2015, 6, 24, 0, 3), # file4 - # "end_time": datetime(2015, 6, 24, 0, 5)}, - # {"start_time": datetime(2015, 6, 24, 0, 5), # file5 - # "end_time": datetime(2015, 6, 24, 0, 7)}, - # ] - # self.assertEqual(finder.get_filenames(reader_info), ["file2", "file3", "file4", "file5"]) - # - # def test_get_filenames_with_start_time_and_npp_style_end_time(self): - # from satpy.scene import Scene - # from satpy.readers import ReaderFinder - # from datetime import datetime - # scn = Scene() - # finder = ReaderFinder(scn) - # reader_info = {"file_patterns": ["foo"], - # "start_time": datetime(2015, 6, 24, 0, 0), - # "end_time": datetime(2015, 6, 24, 0, 6)} - # with mock.patch("satpy.readers.glob.iglob") as mock_iglob: - # mock_iglob.return_value = ["file1", "file2", "file3", "file4", "file5"] - # with mock.patch("satpy.readers.Parser") as mock_parser: - # mock_parser.return_value.parse.side_effect = [{"start_time": datetime(2015, 6, 23, 23, 57), # file1 - # "end_time": datetime(1950, 1, 1, 23, 59)}, - # {"start_time": datetime(2015, 6, 23, 23, 59), # file2 - # "end_time": datetime(1950, 1, 1, 0, 1)}, - # {"start_time": datetime(2015, 6, 24, 0, 1), # file3 - # "end_time": datetime(1950, 1, 1, 0, 3)}, - # {"start_time": datetime(2015, 6, 24, 0, 3), # file4 - # "end_time": datetime(1950, 1, 1, 0, 5)}, - # {"start_time": datetime(2015, 6, 24, 0, 5), # file5 - # "end_time": datetime(1950, 1, 1, 0, 7)}, - # ] - # self.assertEqual(finder.get_filenames(reader_info), ["file2", "file3", "file4", "file5"]) - # - # def test_get_filenames_with_start_time(self): - # from satpy.scene import Scene - # from satpy.readers import ReaderFinder - # from datetime import datetime - # scn = Scene() - # finder = ReaderFinder(scn) - # reader_info = {"file_patterns": ["foo"], - # "start_time": datetime(2015, 6, 24, 0, 0), - # "end_time": datetime(2015, 6, 24, 0, 6)} - # with mock.patch("satpy.readers.glob.iglob") as mock_iglob: - # mock_iglob.return_value = ["file1", "file2", "file3", "file4", "file5"] - # with mock.patch("satpy.readers.Parser") as mock_parser: - # mock_parser.return_value.parse.side_effect = [{"start_time": datetime(2015, 6, 23, 23, 57)}, # file1 - # {"start_time": datetime(2015, 6, 23, 23, 59)}, # file2 - # {"start_time": datetime(2015, 6, 24, 0, 1)}, # file3 - # {"start_time": datetime(2015, 6, 24, 0, 3)}, # file4 - # {"start_time": datetime(2015, 6, 24, 0, 5)}, # file5 - # ] - # self.assertEqual(finder.get_filenames(reader_info), ["file3", "file4", "file5"]) - # - # def test_get_filenames_with_only_start_times_wrong(self): - # from satpy.scene import Scene - # from satpy.readers import ReaderFinder - # from datetime import datetime - # scn = Scene() - # finder = ReaderFinder(scn) - # reader_info = {"file_patterns": ["foo"], - # "start_time": datetime(2015, 6, 24, 0, 0)} - # with mock.patch("satpy.readers.glob.iglob") as mock_iglob: - # mock_iglob.return_value = ["file1", "file2", "file3", "file4", "file5"] - # with mock.patch("satpy.readers.Parser") as mock_parser: - # mock_parser.return_value.parse.side_effect = [{"start_time": datetime(2015, 6, 23, 23, 57)}, # file1 - # {"start_time": datetime(2015, 6, 23, 23, 59)}, # file2 - # {"start_time": datetime(2015, 6, 24, 0, 1)}, # file3 - # {"start_time": datetime(2015, 6, 24, 0, 3)}, # file4 - # {"start_time": datetime(2015, 6, 24, 0, 5)}, # file5 - # ] - # self.assertEqual(finder.get_filenames(reader_info), []) - # - # def test_get_filenames_with_only_start_times_right(self): - # from satpy.scene import Scene - # from satpy.readers import ReaderFinder - # from datetime import datetime - # scn = Scene() - # finder = ReaderFinder(scn) - # reader_info = {"file_patterns": ["foo"], - # "start_time": datetime(2015, 6, 24, 0, 1)} - # with mock.patch("satpy.readers.glob.iglob") as mock_iglob: - # mock_iglob.return_value = ["file1", "file2", "file3", "file4", "file5"] - # with mock.patch("satpy.readers.Parser") as mock_parser: - # mock_parser.return_value.parse.side_effect = [{"start_time": datetime(2015, 6, 23, 23, 57)}, # file1 - # {"start_time": datetime(2015, 6, 23, 23, 59)}, # file2 - # {"start_time": datetime(2015, 6, 24, 0, 1)}, # file3 - # {"start_time": datetime(2015, 6, 24, 0, 3)}, # file4 - # {"start_time": datetime(2015, 6, 24, 0, 5)}, # file5 - # ] - # self.assertEqual(finder.get_filenames(reader_info), ["file3"]) - # - # def test_get_filenames_to_error(self): - # from satpy.scene import Scene - # from satpy.readers import ReaderFinder - # from datetime import datetime - # scn = Scene(start_time="bla") - # finder = ReaderFinder(scn) - # reader_info = {"file_patterns": ["foo"], - # "start_time": None} - # with mock.patch("satpy.readers.glob.iglob") as mock_iglob: - # mock_iglob.return_value = ["file1", "file2", "file3", "file4", "file5"] - # with mock.patch("satpy.readers.Parser") as mock_parser: - # mock_parser.return_value.parse.side_effect = [{"start_time": datetime(2015, 6, 23, 23, 57)}, # file1 - # {"start_time": datetime(2015, 6, 23, 23, 59)}, # file2 - # {"start_time": datetime(2015, 6, 24, 0, 1)}, # file3 - # {"start_time": datetime(2015, 6, 24, 0, 3)}, # file4 - # {"start_time": datetime(2015, 6, 24, 0, 5)}, # file5 - # ] - # self.assertRaises(ValueError, finder.get_filenames, reader_info) - - def suite(): """The test suite for test_scene. """ loader = unittest.TestLoader() mysuite = unittest.TestSuite() - mysuite.addTest(loader.loadTestsFromTestCase(TestReaders)) mysuite.addTest(loader.loadTestsFromTestCase(TestDatasetDict)) mysuite.addTest(loader.loadTestsFromTestCase(TestReaderFinder)) From 16efe1c672130fb88218c818e3191ac57865f43b Mon Sep 17 00:00:00 2001 From: davidh-ssec Date: Mon, 11 Dec 2017 09:19:42 -0600 Subject: [PATCH 09/34] Remove unused legacy .cfg files --- satpy/etc/GDSMetop-A.cfg | 51 ------------------ satpy/etc/MSG3.cfg | 85 ----------------------------- satpy/etc/geo_image.cfg | 2 - satpy/etc/himawari-8.cfg | 114 --------------------------------------- 4 files changed, 252 deletions(-) delete mode 100644 satpy/etc/GDSMetop-A.cfg delete mode 100644 satpy/etc/MSG3.cfg delete mode 100644 satpy/etc/geo_image.cfg delete mode 100644 satpy/etc/himawari-8.cfg diff --git a/satpy/etc/GDSMetop-A.cfg b/satpy/etc/GDSMetop-A.cfg deleted file mode 100644 index aeee9c6fc3..0000000000 --- a/satpy/etc/GDSMetop-A.cfg +++ /dev/null @@ -1,51 +0,0 @@ -[satellite] -variant = GDS -instruments = ('avhrr/3',) - -[avhrr/3-1] -frequency = (0.58, 0.63, 0.68) -resolution = 1090 -name = '1' -size = (2048,) - -[avhrr/3-2] -frequency = (0.725, 0.8625, 1.0) -resolution = 1090 -name = '2' -size = (2048,) - -[avhrr/3-3] -frequency = (1.58, 1.61, 1.64) -resolution = 1090 -name = '3A' -size = (2048,) - -[avhrr/3-4] -frequency = (3.55, 3.74, 3.93) -resolution = 1090 -name = '3B' -size = (2048,) - -[avhrr/3-5] -frequency = (10.3, 10.8, 11.3) -resolution = 1090 -name = '4' -size = (2048,) - -[avhrr/3-6] -frequency = (11.5, 12.0, 12.5) -resolution = 1090 -name = '5' -size = (2048,) - -[avhrr/3-level2] -format=eps_l1b -filename=AVHR_xxx_1B_M02_%Y%m%d%H%M%SZ* -dir=/path/to/satellite/data/ - -[avhrr/3-level3] -format=h5_pps_l2.PPSReader -cloud_product_filename=S_NWC_%(product)s_metopb_%(orbit)s_*.h5 -cloud_product_dir=/data/proj/safutv/data/polar_out/direct_readout/ -cloud_product_geofilename=S_NWC_avhrr_metopb_%(orbit)s_*.h5 -cloud_product_geodir=/san1/pps/import/PPS_data/remapped diff --git a/satpy/etc/MSG3.cfg b/satpy/etc/MSG3.cfg deleted file mode 100644 index c0f3bfcb7d..0000000000 --- a/satpy/etc/MSG3.cfg +++ /dev/null @@ -1,85 +0,0 @@ -[satellite] -projection = 'geos(0.0)' -instruments = ('seviri',) -proj4_params = 'proj=geos lon_0=0.00 lat_0=0.00 a=6378169.00 b=6356583.80 h=35785831.00' -satname = Meteosat -number = '10' - -[seviri-level1] -filename_epi = 'H-000-MSG?__-MSG?________-_________-%(segment)s-%Y%m%d%H%M-__' -format = 'mipp/xrit/MSG' -filename_pro = 'H-000-MSG?__-MSG?________-_________-%(segment)s-%Y%m%d%H%M-__' -filename = 'H-000-MSG?__-MSG?________-%(channel)s-%(segment)s-%Y%m%d%H%M-__' -dir = '/local_disk/data/satellite/met10' - -[seviri-1] -frequency = (0.56, 0.635, 0.71) -resolution = 3000.403165817 -name = 'VIS006' -size = (3712, 3712) - -[seviri-2] -frequency = (0.74, 0.81, 0.88) -resolution = 3000.403165817 -name = 'VIS008' -size = (3712, 3712) - -[seviri-3] -frequency = (1.5, 1.64, 1.78) -resolution = 3000.403165817 -name = 'IR_016' -size = (3712, 3712) - -[seviri-4] -frequency = (3.48, 3.92, 4.36) -resolution = 3000.403165817 -name = 'IR_039' -size = (3712, 3712) - -[seviri-5] -frequency = (5.35, 6.25, 7.15) -resolution = 3000.403165817 -name = 'WV_062' -size = (3712, 3712) - -[seviri-6] -frequency = (6.85, 7.35, 7.85) -resolution = 3000.403165817 -name = 'WV_073' -size = (3712, 3712) - -[seviri-7] -frequency = (8.3, 8.7, 9.1) -resolution = 3000.403165817 -name = 'IR_087' -size = (3712, 3712) - -[seviri-8] -frequency = (9.38, 9.66, 9.94) -resolution = 3000.403165817 -name = 'IR_097' -size = (3712, 3712) - -[seviri-9] -frequency = (9.8, 10.8, 11.8) -resolution = 3000.403165817 -name = 'IR_108' -size = (3712, 3712) - -[seviri-10] -frequency = (11.0, 12.0, 13.0) -resolution = 3000.403165817 -name = 'IR_120' -size = (3712, 3712) - -[seviri-11] -frequency = (12.4, 13.4, 14.4) -resolution = 3000.403165817 -name = 'IR_134' -size = (3712, 3712) - -[seviri-12] -frequency = (0.5, 0.7, 0.9) -resolution = 1000.134348869 -name = 'HRV' -size = (11136, 11136) diff --git a/satpy/etc/geo_image.cfg b/satpy/etc/geo_image.cfg deleted file mode 100644 index f9e4504be6..0000000000 --- a/satpy/etc/geo_image.cfg +++ /dev/null @@ -1,2 +0,0 @@ -[coasts] -coast_file=world_map.ascii \ No newline at end of file diff --git a/satpy/etc/himawari-8.cfg b/satpy/etc/himawari-8.cfg deleted file mode 100644 index c9dfd212fb..0000000000 --- a/satpy/etc/himawari-8.cfg +++ /dev/null @@ -1,114 +0,0 @@ -[satellite] -satname = 'himawari' -number = '8' -instruments = ('ahi',) -projection = 'geos(140.7)' - -[ahi-level2] -format = 'mipp_xrit' - -[ahi-level1] -format = 'mipp/xrit/H8' -dir = '/home/a001673/Downloads' -dir = '/home/a001673/data/satellite/Himawari-8/ahi/lvl1.5/2015/08/04/JMA-HRIT/' -filename = 'IMG_DK01%(channel)s_%Y%m%d%H%M_%(segment)s' - -[ahi-1] -name = 'B01' -frequency = (0.430, 0.455, 0.480) -#resolution = 1000.0 -#size = (11000, 11000) -resolution = 2000.0 -size = (5500, 5500) - -[ahi-2] -name = 'B02' -frequency = (0.50, 0.51, 0.52) -resolution = 2000.0 -size = (5500, 5500) - -[ahi-3] -name = 'VIS' -frequency = (0.630, 0.645, 0.660) -resolution = 2000.0 -size = (5500, 5500) - -[ahi-4] -name = 'B04' -frequency = (0.850, 0.860, 0.870) -resolution = 2000.0 -size = (5500, 5500) - - -[ahi-5] -name = 'B05' -frequency = (1.6, 1.61, 1.62) -resolution = 2000.0 -size = (5500, 5500) - - -[ahi-6] -name = 'B06' -frequency = (2.16, 2.26, 2.36) -resolution = 2000.0 -size = (5500, 5500) - -[ahi-7] -name = 'IR4' -frequency = (3.74, 3.85, 3.96) -resolution = 2000.0 -size = (5500, 5500) - -[ahi-8] -name = 'B08' -frequency = (6.06, 6.25, 6.43) -resolution = 2000.0 -size = (5500, 5500) - -[ahi-9] -name = 'IR3' -frequency = (6.89, 6.95, 7.01) -resolution = 2000.0 -size = (5500, 5500) - -[ahi-10] -name = 'B10' -frequency = (7.23, 7.35, 7.49) -resolution = 2000.0 -size = (5500, 5500) - -[ahi-11] -name = 'B11' -frequency = (8.44, 8.6, 8.74) -resolution = 2000.0 -size = (5500, 5500) - -[ahi-12] -name = 'B12' -frequency = (9.54, 9.63, 9.72) -resolution = 2000.0 -size = (5500, 5500) - -[ahi-13] -name = 'IR1' -frequency = (10.3, 10.45, 10.6) -resolution = 2000.0 -size = (5500, 5500) - -[ahi-14] -name = 'B14' -frequency = (11.1, 11.2, 11.3) -resolution = 2000.0 -size = (5500, 5500) - -[ahi-15] -name = 'IR2' -frequency = (12.2, 12.35, 12.5) -resolution = 2000.0 -size = (5500, 5500) - -[ahi-16] -name = 'B16' -frequency = (13.2, 13.3, 13.4) -resolution = 2000.0 -size = (5500, 5500) From de4c6aceefab81ace1a906ce6a0259add5605910 Mon Sep 17 00:00:00 2001 From: davidh-ssec Date: Mon, 11 Dec 2017 09:40:52 -0600 Subject: [PATCH 10/34] =?UTF-8?q?Bump=20version:=200.7.4=20=E2=86=92=200.7?= =?UTF-8?q?.5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .bumpversion.cfg | 2 +- satpy/version.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index dd6ba7d62d..ec30fed7b0 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 0.7.4 +current_version = 0.7.5 commit = True tag = True diff --git a/satpy/version.py b/satpy/version.py index 586206b400..315f780937 100644 --- a/satpy/version.py +++ b/satpy/version.py @@ -23,4 +23,4 @@ """Version file. """ -__version__ = "0.7.4" +__version__ = "0.7.5" From f54de07fd978e928c3e254916ebb726206b1de3f Mon Sep 17 00:00:00 2001 From: davidh-ssec Date: Mon, 11 Dec 2017 09:40:58 -0600 Subject: [PATCH 11/34] update changelog --- changelog.rst | 167 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 167 insertions(+) diff --git a/changelog.rst b/changelog.rst index e2b159f592..4da845358e 100644 --- a/changelog.rst +++ b/changelog.rst @@ -2,6 +2,173 @@ Changelog ========= +v0.7.5 (2017-12-11) +------------------- +- Update changelog. [davidh-ssec] +- Bump version: 0.7.4 → 0.7.5. [davidh-ssec] +- Remove unused legacy .cfg files. [davidh-ssec] +- Merge branch 'master' into develop. [davidh-ssec] +- Merge pull request #118 from mitkin/master. [Martin Raspaud] + + Add file pattern for MODIS L1B from LAADS WEB +- Add file pattern for MODIS L1B from LAADS WEB. [Mikhail Itkin] + + NASA's LAADS WEB pattern is slightly different + +- Remove old and unused mipp_xrit reader. [davidh-ssec] +- Fix SCMI writer not overwriting data from previous tiles. [davidh- + ssec] +- Merge pull request #121 from pytroll/fix-ir-modifiers. [Martin + Raspaud] + + Remove VIIRS SDR IR modifiers +- Remove sun zenith angle correction from IR channels. [Panu Lahtinen] +- Add github templates for issues and PRs. [Martin Raspaud] +- Bugfix epsl1b reader. [Martin Raspaud] +- Merge pull request #107 from pytroll/fix-nwcsaf-proj4. [David Hoese] + + Convert NWC SAF MSG projection string to meters +- Merge branch 'fix-nwcsaf-proj4' of https://github.com/pytroll/satpy + into fix-nwcsaf-proj4. [Panu Lahtinen] +- Merge branch 'fix-nwcsaf-proj4' of https://github.com/pytroll/satpy + into fix-nwcsaf-proj4. [Panu Lahtinen] +- Read attributes "flag_meanings", "flag_values" and "long_name" [Panu + Lahtinen] +- Configure more datasets. [Panu Lahtinen] +- Fix also area extents. [Panu Lahtinen] +- Add unit tests for utils.proj_units_to_meters() [Panu Lahtinen] +- Move proj_units_to_meters() to satpy.utils. [Panu Lahtinen] +- Convert projection parameters from kilometers to meters. [Panu + Lahtinen] +- Read attributes "flag_meanings", "flag_values" and "long_name" [Panu + Lahtinen] +- Configure more datasets. [Panu Lahtinen] +- Fix also area extents. [Panu Lahtinen] +- Add unit tests for utils.proj_units_to_meters() [Panu Lahtinen] +- Move proj_units_to_meters() to satpy.utils. [Panu Lahtinen] +- Convert projection parameters from kilometers to meters. [Panu + Lahtinen] +- Move proj_units_to_meters() to satpy.utils. [Panu Lahtinen] +- Convert projection parameters from kilometers to meters. [Panu + Lahtinen] +- Read attributes "flag_meanings", "flag_values" and "long_name" [Panu + Lahtinen] +- Configure more datasets. [Panu Lahtinen] +- Fix also area extents. [Panu Lahtinen] +- Add unit tests for utils.proj_units_to_meters() [Panu Lahtinen] +- Move proj_units_to_meters() to satpy.utils. [Panu Lahtinen] +- Convert projection parameters from kilometers to meters. [Panu + Lahtinen] +- Merge pull request #111 from eysteinn/sentinel1-reproject. [David + Hoese] + + Fixed area information to safe_sar_c reader to allow for resampling +- Added coordinates to sar_c.yaml to allow for reprojection. [Eysteinn] +- Merge pull request #108 from TAlonglong/feature-decorate. [Martin + Raspaud] + + Feature decorate +- __init__.py docstring in a few add pydecorate features. [Trygve + Aspenes] +- Satpy/writers/__init__.py implement more general way of handling + pydecorate calls from satpy save_dataset. Instead of logo and text + separate, use decorate. This needs to be a list to keep the order of + alignment available in pydecorate. Since the argument to add_decorate + needs to be a mapping it may look like this: + decorate={'decorate':[{'logo':{...}},{'text':{...}},...]} [Trygve + Aspenes] +- Merge branch 'develop' into develop-fork. [Trygve Aspenes] +- Satpy/writers/__init__.py added add_text function. This is meant to be + used when calling save_dataset to add text to an image using + pydecorate. eg save_dataset(...., text_overlay={'text': 'THIS IS THE + TEXT TO BE ADDED', 'align':{'top_bottom':'bottom', + 'left_right':'right'}, + 'font':'/usr/share/fonts/truetype/msttcorefonts/Arial.ttf', + 'font_size':25, 'height':30, 'bg':'black', 'bg_opacity':255, + 'line':'white'}). Not all options available as style in pydecorate are + implemented. This is left TODO. This PR is dependent on + https://github.com/pytroll/pydecorate/pull/3 to be completed. [Trygve + Aspenes] +- Adding to more options to add_overlay. This to better control which + levels of coast(GSHHS) and borders (WDB_II) are put on the plot. + [Trygve Aspenes] +- Merge pull request #88 from pytroll/feature-3d-enhancement. [Panu + Lahtinen] + + Add 3D enhancement, fix BWCompositor +- Merge branch 'feature-3d-enhancement' of + https://github.com/pytroll/satpy into feature-3d-enhancement. [Panu + Lahtinen] +- Add example of composite with 3D effect. [Panu Lahtinen] +- Fix BWCompositor to handle info correctly. [Panu Lahtinen] +- Add 3D effect enhancement. [Panu Lahtinen] +- Remove rebase comments. [Panu Lahtinen] +- Add example of composite with 3D effect. [Panu Lahtinen] +- Fix BWCompositor to handle info correctly. [Panu Lahtinen] +- Add 3D effect enhancement. [Panu Lahtinen] +- Merge pull request #87 from pytroll/feature-IASI-L2-reader. [Panu + Lahtinen] + + Add IASI L2 reader +- Merge branch 'feature-IASI-L2-reader' of + https://github.com/pytroll/satpy into feature-IASI-L2-reader. [Panu + Lahtinen] +- Merge branch 'feature-IASI-L2-reader' of + https://github.com/pytroll/satpy into feature-IASI-L2-reader. [Panu + Lahtinen] +- Fix unit of time. [Panu Lahtinen] +- Remove un-needed '' from the reader init line. [Panu Lahtinen] +- Merge branch 'develop' into feature-IASI-L2-reader. [Panu Lahtinen] +- Add mapping from M03 to Metop-C. [Panu Lahtinen] +- Add subsatellite resolution to datasets. [Panu Lahtinen] +- Fix typos, make read_dataset() and read_geo() functions instead of + methods. [Panu Lahtinen] +- Add initial version of IASI L2 reader. [Panu Lahtinen] +- Fix unit of time. [Panu Lahtinen] +- Remove un-needed '' from the reader init line. [Panu Lahtinen] +- Add mapping from M03 to Metop-C. [Panu Lahtinen] +- Add subsatellite resolution to datasets. [Panu Lahtinen] +- Fix typos, make read_dataset() and read_geo() functions instead of + methods. [Panu Lahtinen] +- Add initial version of IASI L2 reader. [Panu Lahtinen] +- Fix unit of time. [Panu Lahtinen] +- Remove un-needed '' from the reader init line. [Panu Lahtinen] +- Add mapping from M03 to Metop-C. [Panu Lahtinen] +- Add subsatellite resolution to datasets. [Panu Lahtinen] +- Fix typos, make read_dataset() and read_geo() functions instead of + methods. [Panu Lahtinen] +- Add initial version of IASI L2 reader. [Panu Lahtinen] +- Merge pull request #96 from eysteinn/create_colormap. [David Hoese] + + Create colormap +- Make colorizing/palettizing more flexible. [Eysteinn] +- Merge pull request #4 from pytroll/develop. [Eysteinn Sigurðsson] + + Develop +- Merge pull request #3 from pytroll/develop. [Eysteinn Sigurðsson] + + Develop +- Merge pull request #109 from pytroll/bugfix-scmi. [David Hoese] + + Fix SCMI writer and add more tiled grids +- Fix SCMI writer writing masked geolocation to netcdf files. [davidh- + ssec] +- Add additional GOES SCMI grids. [davidh-ssec] +- Allow adding overlay for L and LA images. [Martin Raspaud] +- Merge pull request #101 from pytroll/bugfix-scmi3. [David Hoese] + + Fix python 3 compatibility in scmi writer +- Add more SCMI writer tests for expected failures. [davidh-ssec] +- Fix python 3 compatibility in scmi writer. [davidh-ssec] + + Includes fix for X/Y coordinate precision which affects GOES-16 data + +- Merge pull request #105 from howff/doc-fix. [Martin Raspaud] + + fix available_composite_names in doc +- Fix available_composite_names in doc. [Andrew Brooks] + + v0.7.4 (2017-11-13) ------------------- - Update changelog. [davidh-ssec] From 1741856b43a57be8c847ecc15845a95e7897a990 Mon Sep 17 00:00:00 2001 From: Eysteinn Date: Mon, 11 Dec 2017 16:22:37 +0000 Subject: [PATCH 12/34] Add reader for ScatSat1 Level 2B wind speed data --- satpy/etc/readers/scatsat1_l2b.yaml | 42 +++++++++++++++++ satpy/readers/scatsat1_l2b.py | 73 +++++++++++++++++++++++++++++ 2 files changed, 115 insertions(+) create mode 100644 satpy/etc/readers/scatsat1_l2b.yaml create mode 100644 satpy/readers/scatsat1_l2b.py diff --git a/satpy/etc/readers/scatsat1_l2b.yaml b/satpy/etc/readers/scatsat1_l2b.yaml new file mode 100644 index 0000000000..afe9bcf144 --- /dev/null +++ b/satpy/etc/readers/scatsat1_l2b.yaml @@ -0,0 +1,42 @@ +reader: + description: Generic Eumetsat Scatsat-1 L2B Wind field Reader + name: scatsat1_l2b + reader: !!python/name:satpy.readers.yaml_reader.FileYAMLReader '' + sensors: [scatterometer] + default_datasets: + +datasets: + longitude: + name: longitude + resolution: 25000 + file_type: scatsat + standard_name: longitude + units: degree + + latitude: + name: latitude + resolution: 25000 + file_type: scatsat + standard_name: latitude + units: degree + + wind_speed: + name: wind_speed + sensor: Scatterometer + resolution: 25000 + coordinates: [longitude, latitude] + file_type: scatsat + standard_name: wind_speed + + wind_direction: + name: wind_direction + resolution: 25000 + coordinates: [longitude, latitude] + file_type: scatsat + standard_name: wind_direction + + +file_types: + scatsat: + file_reader: !!python/name:satpy.readers.scatsat1_l2b.SCATSAT1L2BFileHandler '' + file_patterns: ['S1L2B{start_date:%Y%j}_{start_orbit}_{end_orbit}_{direction}_{cell_spacing}_{prod_date}T{prod_time}_{version}.h5'] diff --git a/satpy/readers/scatsat1_l2b.py b/satpy/readers/scatsat1_l2b.py new file mode 100644 index 0000000000..ff4af051a2 --- /dev/null +++ b/satpy/readers/scatsat1_l2b.py @@ -0,0 +1,73 @@ +# -*- coding: utf-8 -*- +# Copyright (c) 2017. +# +# Author(s): + +# Eysteinn Már Sigurðsson + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +""" ScatSat-1 L2B Reader, distributed by Eumetsat in HDF5 format +""" + +from datetime import datetime, timedelta +import h5py + +from satpy.dataset import Dataset +from satpy.readers.file_handlers import BaseFileHandler + + +class SCATSAT1L2BFileHandler(BaseFileHandler): + + def __init__(self, filename, filename_info, filetype_info): + super(SCATSAT1L2BFileHandler, self).__init__(filename, filename_info, + filetype_info) + self.h5f = h5py.File(self.filename, "r") + h5data=self.h5f['science_data'] + + self.filename_info['start_time'] = datetime.strptime(h5data.attrs['Range Beginning Date'],'%Y-%jT%H:%M:%S.%f') + self.lons = None + self.lats = None + + self.wind_speed_scale = float(h5data.attrs['Wind Speed Selection Scale']) + self.wind_direction_scale = float(h5data.attrs['Wind Direction Selection Scale']) + self.latitude_scale = float(h5data.attrs['Latitude Scale']) + self.longitude_scale = float(h5data.attrs['Longitude Scale']) + + def get_dataset(self, key, info): + h5data = self.h5f['science_data'] + stdname = info.get('standard_name') + + if stdname in ['latitude', 'longitude']: + + if self.lons is None or self.lats is None: + self.lons = h5data['Longitude'][:]*self.longitude_scale + self.lats = h5data['Latitude'][:]*self.latitude_scale + + if info['standard_name'] == 'longitude': + return Dataset(self.lons, id=key, **info) + else: + return Dataset(self.lats, id=key, **info) + + if stdname in ['wind_speed']: + windspeed=h5data['Wind_speed_selection'][:,:]*self.wind_speed_scale + return Dataset(windspeed, id=key, **info) + + if stdname in ['wind_direction']: + wind_direction = h5data['Wind_direction_selection'][:,:]*self.wind_direction_scale + return Dataset(wind_direction, id=key, **info) + + + + + From 563a4e12da2c7db011b7de9dffe7d51553c039e4 Mon Sep 17 00:00:00 2001 From: "Adam.Dybbroe" Date: Thu, 14 Dec 2017 11:14:04 +0100 Subject: [PATCH 13/34] Use the CIRA stretch for True color imagery Signed-off-by: Adam.Dybbroe --- satpy/etc/enhancements/generic.yaml | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/satpy/etc/enhancements/generic.yaml b/satpy/etc/enhancements/generic.yaml index 0c1371354a..fb0090c807 100644 --- a/satpy/etc/enhancements/generic.yaml +++ b/satpy/etc/enhancements/generic.yaml @@ -20,15 +20,13 @@ enhancements: true_color_norayleigh_default: standard_name: true_color_norayleigh operations: - - name: stretch - method: *stretchfun - kwargs: {stretch: linear} + - name: cira_stretch + method: !!python/name:satpy.enhancements.cira_stretch true_color_reducedsize_default: standard_name: true_color_reducedsize operations: - - name: stretch - method: *stretchfun - kwargs: {stretch: linear} + - name: cira_stretch + method: !!python/name:satpy.enhancements.cira_stretch overview_default: standard_name: overview operations: From b60af0698646b8e0387cec275a93d8b074fa8016 Mon Sep 17 00:00:00 2001 From: "Adam.Dybbroe" Date: Thu, 14 Dec 2017 11:51:26 +0100 Subject: [PATCH 14/34] Use marine_clean and us-standard for atm correction, and improve stretch at low sun elevation Signed-off-by: Adam.Dybbroe --- satpy/etc/composites/ahi.yaml | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/satpy/etc/composites/ahi.yaml b/satpy/etc/composites/ahi.yaml index 12d01cd9ef..442f41d005 100644 --- a/satpy/etc/composites/ahi.yaml +++ b/satpy/etc/composites/ahi.yaml @@ -20,6 +20,8 @@ modifiers: rayleigh_corrected: compositor: !!python/name:satpy.composites.PSPRayleighReflectance + atmosphere: us-standard + aerosol_type: marine_clean_aerosol prerequisites: - wavelength: 0.46 modifiers: [reducer2, sunz_corrected] @@ -42,33 +44,33 @@ composites: compositor: !!python/name:satpy.composites.RGBCompositor prerequisites: - wavelength: 0.65 - modifiers: [reducer2, sunz_corrected, rayleigh_corrected] + modifiers: [reducer2, effective_solar_pathlength_corrected, rayleigh_corrected] - wavelength: 0.51 - modifiers: [vegetation_corrected, sunz_corrected, rayleigh_corrected] + modifiers: [vegetation_corrected, effective_solar_pathlength_corrected, rayleigh_corrected] - wavelength: 0.46 - modifiers: [sunz_corrected, rayleigh_corrected] + modifiers: [effective_solar_pathlength_corrected, rayleigh_corrected] standard_name: true_color_ahi true_color_reducedsize: compositor: !!python/name:satpy.composites.RGBCompositor prerequisites: - wavelength: 0.65 - modifiers: [reducer4, sunz_corrected, rayleigh_corrected] + modifiers: [reducer4, effective_solar_pathlength_corrected, rayleigh_corrected] - wavelength: 0.51 - modifiers: [reducer2, vegetation_corrected_reduced, sunz_corrected, rayleigh_corrected] + modifiers: [reducer2, vegetation_corrected_reduced, effective_solar_pathlength_corrected, rayleigh_corrected] - wavelength: 0.46 - modifiers: [reducer2, sunz_corrected, rayleigh_corrected] + modifiers: [reducer2, effective_solar_pathlength_corrected, rayleigh_corrected] standard_name: true_color_reducedsize true_color_norayleigh: compositor: !!python/name:satpy.composites.RGBCompositor prerequisites: - wavelength: 0.65 - modifiers: [reducer2, sunz_corrected] + modifiers: [reducer2, effective_solar_pathlength_corrected] - wavelength: 0.51 - modifiers: [vegetation_corrected, sunz_corrected] + modifiers: [vegetation_corrected, effective_solar_pathlength_corrected] - wavelength: 0.46 - modifiers: [sunz_corrected] + modifiers: [effective_solar_pathlength_corrected] standard_name: true_color_norayleigh day_microphysics_eum: From 0c142bca50fb6cdec7e48b075a25b4500d5283bc Mon Sep 17 00:00:00 2001 From: Eysteinn Date: Thu, 14 Dec 2017 15:32:35 +0000 Subject: [PATCH 15/34] Read end_time info correctly --- satpy/readers/scatsat1_l2b.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/satpy/readers/scatsat1_l2b.py b/satpy/readers/scatsat1_l2b.py index ff4af051a2..2a1863419f 100644 --- a/satpy/readers/scatsat1_l2b.py +++ b/satpy/readers/scatsat1_l2b.py @@ -36,6 +36,8 @@ def __init__(self, filename, filename_info, filetype_info): h5data=self.h5f['science_data'] self.filename_info['start_time'] = datetime.strptime(h5data.attrs['Range Beginning Date'],'%Y-%jT%H:%M:%S.%f') + self.filename_info['end_time'] = datetime.strptime(h5data.attrs['Range Ending Date'],'%Y-%jT%H:%M:%S.%f') + self.lons = None self.lats = None From d0935a5fd425890226029cc566c824655dcd5cde Mon Sep 17 00:00:00 2001 From: "Adam.Dybbroe" Date: Fri, 15 Dec 2017 17:41:28 +0100 Subject: [PATCH 16/34] Add some pre-defined atm/rayleigh corrections to appply over land and sea Signed-off-by: Adam.Dybbroe --- satpy/etc/composites/olci.yaml | 39 ++++++++++++++++++++++++++--- satpy/etc/composites/visir.yaml | 44 ++++++++++++++++++++++++++++++--- 2 files changed, 77 insertions(+), 6 deletions(-) diff --git a/satpy/etc/composites/olci.yaml b/satpy/etc/composites/olci.yaml index bfc4c6ec28..9f35c62949 100644 --- a/satpy/etc/composites/olci.yaml +++ b/satpy/etc/composites/olci.yaml @@ -5,9 +5,42 @@ composites: compositor: !!python/name:satpy.composites.RGBCompositor prerequisites: - name: 'Oa08' - modifiers: [sunz_corrected, rayleigh_corrected] + modifiers: [effective_solar_pathlength_corrected, rayleigh_corrected] - name: 'Oa06' - modifiers: [sunz_corrected, rayleigh_corrected] + modifiers: [effective_solar_pathlength_corrected, rayleigh_corrected] - name: 'Oa03' - modifiers: [sunz_corrected, rayleigh_corrected] + modifiers: [effective_solar_pathlength_corrected, rayleigh_corrected] + standard_name: true_color + + true_color_desert: + compositor: !!python/name:satpy.composites.RGBCompositor + prerequisites: + - name: 'Oa08' + modifiers: [effective_solar_pathlength_corrected, rayleigh_corrected_desert] + - name: 'Oa06' + modifiers: [effective_solar_pathlength_corrected, rayleigh_corrected_desert] + - name: 'Oa03' + modifiers: [effective_solar_pathlength_corrected, rayleigh_corrected_desert] + standard_name: true_color + + true_color_marine_tropical: + compositor: !!python/name:satpy.composites.RGBCompositor + prerequisites: + - name: 'Oa08' + modifiers: [effective_solar_pathlength_corrected, rayleigh_corrected_marine_tropical] + - name: 'Oa06' + modifiers: [effective_solar_pathlength_corrected, rayleigh_corrected_marine_tropical] + - name: 'Oa03' + modifiers: [effective_solar_pathlength_corrected, rayleigh_corrected_marine_tropical] + standard_name: true_color + + true_color_raw: + compositor: !!python/name:satpy.composites.RGBCompositor + prerequisites: + - name: 'Oa08' + modifiers: [effective_solar_pathlength_corrected] + - name: 'Oa06' + modifiers: [effective_solar_pathlength_corrected] + - name: 'Oa03' + modifiers: [effective_solar_pathlength_corrected] standard_name: true_color diff --git a/satpy/etc/composites/visir.yaml b/satpy/etc/composites/visir.yaml index fe9f903fcc..24505d7071 100644 --- a/satpy/etc/composites/visir.yaml +++ b/satpy/etc/composites/visir.yaml @@ -29,9 +29,47 @@ modifiers: rayleigh_corrected: compositor: !!python/name:satpy.composites.PSPRayleighReflectance - atmosphere: midlatitude summer - #aerosol_type: marine_clean_aerosol - aerosol_type: rayleigh_only + atmosphere: us-standard + aerosol_type: marine_clean_aerosol + prerequisites: + - wavelength: 0.44 + modifiers: [sunz_corrected] + optional_prerequisites: + - satellite_azimuth_angle + - satellite_zenith_angle + - solar_azimuth_angle + - solar_zenith_angle + + rayleigh_corrected_marine_tropical: + compositor: !!python/name:satpy.composites.PSPRayleighReflectance + atmosphere: tropical + aerosol_type: marine_tropical_aerosol + prerequisites: + - wavelength: 0.44 + modifiers: [sunz_corrected] + optional_prerequisites: + - satellite_azimuth_angle + - satellite_zenith_angle + - solar_azimuth_angle + - solar_zenith_angle + + rayleigh_corrected_desert: + compositor: !!python/name:satpy.composites.PSPRayleighReflectance + atmosphere: tropical + aerosol_type: desert_aerosol + prerequisites: + - wavelength: 0.44 + modifiers: [sunz_corrected] + optional_prerequisites: + - satellite_azimuth_angle + - satellite_zenith_angle + - solar_azimuth_angle + - solar_zenith_angle + + rayleigh_corrected_land: + compositor: !!python/name:satpy.composites.PSPRayleighReflectance + atmosphere: us-standard + aerosol_type: continental_average_aerosol prerequisites: - wavelength: 0.44 modifiers: [sunz_corrected] From 2081faa1f14267d05997cbdada2dd79c1736c73f Mon Sep 17 00:00:00 2001 From: "Adam.Dybbroe" Date: Fri, 15 Dec 2017 17:47:07 +0100 Subject: [PATCH 17/34] pep8 editorial, and fixing copyright Signed-off-by: Adam.Dybbroe --- satpy/plugin_base.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/satpy/plugin_base.py b/satpy/plugin_base.py index 86a9cac40d..0018f69011 100644 --- a/satpy/plugin_base.py +++ b/satpy/plugin_base.py @@ -1,7 +1,7 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- -# Copyright (c) 2011 SMHI +# Copyright (c) 2011-2017 PyTroll # Author(s): @@ -37,6 +37,7 @@ class Plugin(object): + """The base plugin class. It is not to be used as is, it has to be inherited by other classes. """ From d78e624f17843f64005c60ebb02b55a6dd74557f Mon Sep 17 00:00:00 2001 From: "Adam.Dybbroe" Date: Fri, 15 Dec 2017 17:47:29 +0100 Subject: [PATCH 18/34] pep8 cosmetics Signed-off-by: Adam.Dybbroe --- satpy/readers/__init__.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/satpy/readers/__init__.py b/satpy/readers/__init__.py index fad27f5390..405fd684e7 100644 --- a/satpy/readers/__init__.py +++ b/satpy/readers/__init__.py @@ -46,6 +46,7 @@ class MalformedConfigError(Exception): class DatasetDict(dict): + """Special dictionary object that can handle dict operations based on dataset name, wavelength, or DatasetID. @@ -262,6 +263,7 @@ def load_reader(reader_configs, metadata=None, **reader_kwargs): class ReaderFinder(object): + """Find readers given a scene, filenames, sensors, and/or a reader_name """ From 986ae14b0a73c0bb62138fe0e3155f58f805f236 Mon Sep 17 00:00:00 2001 From: "Adam.Dybbroe" Date: Fri, 15 Dec 2017 17:48:19 +0100 Subject: [PATCH 19/34] pep8 cosmetics Signed-off-by: Adam.Dybbroe --- satpy/scene.py | 1 + 1 file changed, 1 insertion(+) diff --git a/satpy/scene.py b/satpy/scene.py index 43305a36c6..d48c0d1039 100644 --- a/satpy/scene.py +++ b/satpy/scene.py @@ -144,6 +144,7 @@ def create_reader_instances(self, start_time=self.info.get('start_time'), end_time=self.info.get('end_time'), area=self.info.get('area'), ) + return finder(reader=reader, sensor=self.info.get("sensor"), filenames=filenames, From c28c602b0711b1402b7b5351616512d17b2916ba Mon Sep 17 00:00:00 2001 From: "Adam.Dybbroe" Date: Fri, 15 Dec 2017 17:58:51 +0100 Subject: [PATCH 20/34] Use the Pyspectral atm correction as the default. Add a high-res overview RGB, use the hncc-dnb in the night-microphysics and use the effective_solar_pathlength_corrected for all true color RGBs. Signed-off-by: Adam.Dybbroe --- satpy/etc/composites/viirs.yaml | 41 ++++++++++++++++++--------------- 1 file changed, 22 insertions(+), 19 deletions(-) diff --git a/satpy/etc/composites/viirs.yaml b/satpy/etc/composites/viirs.yaml index 98e0330c30..a1bf54b90e 100644 --- a/satpy/etc/composites/viirs.yaml +++ b/satpy/etc/composites/viirs.yaml @@ -1,7 +1,7 @@ sensor_name: visir/viirs modifiers: - rayleigh_corrected: + rayleigh_corrected_crefl: compositor: !!python/name:satpy.composites.viirs.ReflectanceCorrector dem_filename: CMGDEM.hdf prerequisites: @@ -15,11 +15,6 @@ modifiers: prerequisites: - solar_zenith_angle - effective_solar_pathlength_corrected: - compositor: !!python/name:satpy.composites.EffectiveSolarPathLengthCorrector - prerequisites: - - solar_zenith_angle - composites: @@ -27,14 +22,14 @@ composites: compositor: !!python/name:satpy.composites.viirs.RatioSharpenedRGB prerequisites: - name: M05 - modifiers: [sunz_corrected, rayleigh_corrected] + modifiers: [effective_solar_pathlength_corrected, rayleigh_corrected] - name: M04 - modifiers: [sunz_corrected, rayleigh_corrected] + modifiers: [effective_solar_pathlength_corrected, rayleigh_corrected] - name: M03 - modifiers: [sunz_corrected, rayleigh_corrected] + modifiers: [effective_solar_pathlength_corrected, rayleigh_corrected] optional_prerequisites: - name: I01 - modifiers: [sunz_corrected, rayleigh_corrected] + modifiers: [effective_solar_pathlength_corrected, rayleigh_corrected] standard_name: true_color high_resolution_band: red @@ -42,20 +37,20 @@ composites: compositor: !!python/name:satpy.composites.RGBCompositor prerequisites: - name: M05 - modifiers: [sunz_corrected, rayleigh_corrected] + modifiers: [effective_solar_pathlength_corrected, rayleigh_corrected] - name: M04 - modifiers: [sunz_corrected, rayleigh_corrected] + modifiers: [effective_solar_pathlength_corrected, rayleigh_corrected] - name: M03 - modifiers: [sunz_corrected, rayleigh_corrected] + modifiers: [effective_solar_pathlength_corrected, rayleigh_corrected] standard_name: true_color natural_color: compositor: !!python/name:satpy.composites.viirs.RatioSharpenedRGB prerequisites: - name: M11 - modifiers: [sunz_corrected, rayleigh_corrected] + modifiers: [sunz_corrected] - name: M07 - modifiers: [sunz_corrected, rayleigh_corrected] + modifiers: [sunz_corrected] - name: M05 modifiers: [sunz_corrected, rayleigh_corrected] optional_prerequisites: @@ -68,11 +63,11 @@ composites: compositor: !!python/name:satpy.composites.RGBCompositor prerequisites: - name: M05 - modifiers: [sunz_corrected] + modifiers: [effective_solar_pathlength_corrected] - name: M04 - modifiers: [sunz_corrected] + modifiers: [effective_solar_pathlength_corrected] - name: M03 - modifiers: [sunz_corrected] + modifiers: [effective_solar_pathlength_corrected] standard_name: true_color night_overview: @@ -91,10 +86,18 @@ composites: - M15 standard_name: overview + hr_overview: + compositor: !!python/name:satpy.composites.RGBCompositor + prerequisites: + - I01 + - I02 + - I05 + standard_name: overview + night_microphysics: compositor: !!python/name:satpy.composites.RGBCompositor prerequisites: - - DNB + - hncc_dnb - M12 - M15 standard_name: night_microphysics From 99c2f7269a40cb17ea661cb25db419e48b57d105 Mon Sep 17 00:00:00 2001 From: "Adam.Dybbroe" Date: Fri, 15 Dec 2017 18:11:35 +0100 Subject: [PATCH 21/34] Use a more appropriate and shorter link to the MSG native format pdf doc Signed-off-by: Adam.Dybbroe --- satpy/readers/native_msg.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/satpy/readers/native_msg.py b/satpy/readers/native_msg.py index c17a063eae..ebddebcd89 100644 --- a/satpy/readers/native_msg.py +++ b/satpy/readers/native_msg.py @@ -24,8 +24,7 @@ """A reader for the EUMETSAT MSG native format -https://www.google.se/url?sa=t&rct=j&q=&esrc=s&source=web&cd=1&ved=0ahUKEwi_pcrm1vjSAhUMS5oKHc3QADgQFggcMAA&url=http%3A%2F%2Fwww.eumetsat.int%2Fwebsite%2Fwcm%2Fidc%2Fidcplg%3FIdcService%3DGET_FILE%26dDocName%3DPDF_FG15_MSG-NATIVE-FORMAT-15%26RevisionSelectionMethod%3DLatestReleased%26Rendition%3DWeb&usg=AFQjCNE9YWoECpgBhGGSkPZkHBWB37pgUA&sig2=4TXWtAj-yFA6XCaTmhFN9w - +https://www.eumetsat.int/website/wcm/idc/idcplg?IdcService=GET_FILE&dDocName=PDF_FG15_MSG-NATIVE-FORMAT-15&RevisionSelectionMethod=LatestReleased&Rendition=Web """ From 0d19ee8b7f58f4082bea5d819b488bd4b5e13896 Mon Sep 17 00:00:00 2001 From: Martin Raspaud Date: Mon, 18 Dec 2017 14:57:40 +0100 Subject: [PATCH 22/34] Bugfix hrit_jma --- satpy/etc/readers/hrit_jma.yaml | 2 +- satpy/readers/hrit_jma.py | 4 +--- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/satpy/etc/readers/hrit_jma.yaml b/satpy/etc/readers/hrit_jma.yaml index 4fe78b4376..f4621dcbdd 100644 --- a/satpy/etc/readers/hrit_jma.yaml +++ b/satpy/etc/readers/hrit_jma.yaml @@ -113,7 +113,7 @@ datasets: radiance: standard_name: toa_outgoing_radiance_per_unit_wavelength units: W m-2 um-1 sr-1 - file_type: hrit_b04 + file_type: hrit_b03 B04: name: B04 diff --git a/satpy/readers/hrit_jma.py b/satpy/readers/hrit_jma.py index 81b0b76cee..b51b3ad678 100644 --- a/satpy/readers/hrit_jma.py +++ b/satpy/readers/hrit_jma.py @@ -201,9 +201,7 @@ def get_dataset(self, key, info, out=None, out = res self.calibrate(out, key.calibration) - out.info['units'] = info['units'] - out.info['standard_name'] = info['standard_name'] - #out.info['platform_name'] = self.platform_name + out.info.update(info) out.info['platform_name'] = 'Himawari-8' out.info['sensor'] = 'ahi' From f20dce44c461445473c68d7bf3d62dd45d1fcb46 Mon Sep 17 00:00:00 2001 From: Martin Raspaud Date: Mon, 18 Dec 2017 15:49:31 +0100 Subject: [PATCH 23/34] Update PULL_REQUEST_TEMPLATE.md This hides the comments when the PR is previewed and reminds user to provide a description for the PR. --- .github/PULL_REQUEST_TEMPLATE.md | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 670a685b88..346c742658 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -1,7 +1,9 @@ -Please make the PR against the `develop` branch. + - - [ ] Closes #xxxx (remove if there is no corresponding issue, which should only be the case for minor changes) - - [ ] Tests added (for all bug fixes or enhancements) - - [ ] Tests passed (for all non-documentation changes) - - [ ] Passes ``git diff origin/develop **/*py | flake8 --diff`` (remove if you did not edit any Python files) - - [ ] Fully documented (remove if this change should not be visible to users, e.g., if it is an internal clean-up, or if this is part of a larger project that will be documented later) + + + - [ ] Closes #xxxx + - [ ] Tests added + - [ ] Tests passed + - [ ] Passes ``git diff origin/develop **/*py | flake8 --diff`` + - [ ] Fully documented From 5f59ecb99fb5b04d5d168503f04983df30996c24 Mon Sep 17 00:00:00 2001 From: Martin Raspaud Date: Tue, 19 Dec 2017 12:13:56 +0100 Subject: [PATCH 24/34] Finalize GOES LRIT reader --- satpy/etc/readers/hrit_goes.yaml | 12 +- satpy/readers/hrit_goes.py | 205 +++++++++++++++++++++++++++---- 2 files changed, 185 insertions(+), 32 deletions(-) diff --git a/satpy/etc/readers/hrit_goes.yaml b/satpy/etc/readers/hrit_goes.yaml index bdaacb7f49..0e1f26a28e 100644 --- a/satpy/etc/readers/hrit_goes.yaml +++ b/satpy/etc/readers/hrit_goes.yaml @@ -52,12 +52,12 @@ datasets: resolution: 3000 wavelength: [0.55, 0.7, 0.75] calibration: - # reflectance: - # standard_name: toa_bidirectional_reflectance - # units: "%" - radiance: - standard_name: toa_outgoing_radiance_per_unit_wavelength - units: W m-2 um-1 sr-1 + reflectance: + standard_name: toa_bidirectional_reflectance + units: "%" + # radiance: + # standard_name: toa_outgoing_radiance_per_unit_wavelength + # units: W m-2 um-1 sr-1 counts: standard_name: counts file_type: HRIT_00_7 diff --git a/satpy/readers/hrit_goes.py b/satpy/readers/hrit_goes.py index 4d99f2a976..bac8ced012 100644 --- a/satpy/readers/hrit_goes.py +++ b/satpy/readers/hrit_goes.py @@ -112,6 +112,41 @@ def make_time_cds_expanded(tcds_array): microseconds=float(tcds_array['microseconds'] + tcds_array['nanoseconds'] / 1000.))) +sgs_time = np.dtype([('century', 'u1'), + ('year', 'u1'), + ('doy1', 'u1'), + ('doy_hours', 'u1'), + ('hours_mins', 'u1'), + ('mins_secs', 'u1'), + ('secs_msecs', 'u1'), + ('msecs', 'u1')]) + +def make_sgs_time(sgs_time_array): + year = ((sgs_time_array['century'] >> 4) * 1000 + + (sgs_time_array['century'] & 15) * 100 + + (sgs_time_array['year'] >> 4) * 10 + + (sgs_time_array['year'] & 15)) + doy = ((sgs_time_array['doy1'] >> 4) * 100 + + (sgs_time_array['doy1'] & 15) * 10 + + (sgs_time_array['doy_hours'] >> 4)) + hours = ((sgs_time_array['doy_hours'] & 15) * 10 + + (sgs_time_array['hours_mins'] >> 4)) + mins = ((sgs_time_array['hours_mins'] & 15) * 10 + + (sgs_time_array['mins_secs'] >> 4)) + secs = ((sgs_time_array['mins_secs'] & 15) * 10 + + (sgs_time_array['secs_msecs'] >> 4)) + msecs = ((sgs_time_array['secs_msecs'] & 15) * 100 + + (sgs_time_array['msecs'] >> 4) * 10 + + (sgs_time_array['msecs'] &15)) + return (datetime(year, 1, 1) + + timedelta(days=doy - 1, + hours=hours, + minutes=mins, + seconds=secs, + milliseconds=msecs)) + + + satellite_status = np.dtype([("TagType", "> 24) - 64 + mant = float_val & ((1 << 24) - 1) + if mant == 0: + return 0. + res = sign * mant * 2.0**(-24 + exp * 4) + return res + + # XXX arb #prologue = np.dtype([('SatelliteStatus', satellite_status), # ('ImageAcquisition', image_acquisition, (10, )), @@ -145,17 +196,49 @@ def make_time_cds_expanded(tcds_array): # product header ("ImageProductHeaderVersion", "u1"), ("Junk2", "u1", 3), - ("ImageProductHeaderLength", "u4"), ("ImageProductVersion", "u1"), # first block-0 ("SatelliteID", "u1"), - ("Junk3", "u1", 173), # move to "word" 175 - ("SubSatLatitude", ">f4"), - ("SubSatLongitude", ">f4"), # ">f4" seems better than "f4"), - ("ReferenceDistance", ">f4"), - ("ReferenceLatitude", ">f4") + ("SPSID", "u1"), + ("IScan", "u1", 4), + ("IDSub", "u1", 16), + ("TCurr", sgs_time), + ("TCHED", sgs_time), + ("TCTRL", sgs_time), + ("TLHED", sgs_time), + ("TLTRL", sgs_time), + ("TIPFS", sgs_time), + ("TINFS", sgs_time), + ("TISPC", sgs_time), + ("TIECL", sgs_time), + ("TIBBC", sgs_time), + ("TISTR", sgs_time), + ("TLRAN", sgs_time), + ("TIIRT", sgs_time), + ("TIVIT", sgs_time), + ("TCLMT", sgs_time), + ("TIONA", sgs_time), + ("RelativeScanCount", '>u2'), + ("AbsoluteScanCount", '>u2'), + ("NorthernmostScanLine", '>u2'), + ("WesternmostPixel", '>u2'), + ("EasternmostPixel", '>u2'), + ("NorthernmostFrameLine", '>u2'), + ("SouthernmostFrameLine", '>u2'), + ("0Pixel", '>u2'), + ("0ScanLine", '>u2'), + ("0Scan", '>u2'), + ("SubSatScan", '>u2'), + ("SubSatPixel", '>u2'), + ("SubSatLatitude", gvar_float), + ("SubSatLongitude", gvar_float), # ">f4" seems better than " Date: Tue, 19 Dec 2017 15:07:34 +0100 Subject: [PATCH 25/34] Fix GOES navigation --- satpy/readers/hrit_goes.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/satpy/readers/hrit_goes.py b/satpy/readers/hrit_goes.py index bac8ced012..d89252e44b 100644 --- a/satpy/readers/hrit_goes.py +++ b/satpy/readers/hrit_goes.py @@ -492,6 +492,8 @@ def get_area_def(self, dsid): nlines = int(self.mda['number_of_lines']) ncols = int(self.mda['number_of_columns']) + loff = nlines - loff + area_extent = self.get_area_extent((nlines, ncols), (loff, coff), (lfac, cfac), From 1616d2cdbbb7e87ec403673e6f48e31403728f70 Mon Sep 17 00:00:00 2001 From: Martin Raspaud Date: Tue, 19 Dec 2017 15:10:13 +0100 Subject: [PATCH 26/34] Update documentation to add hrit_goes --- doc/source/index.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/doc/source/index.rst b/doc/source/index.rst index 4b0a200083..5141788a4b 100644 --- a/doc/source/index.rst +++ b/doc/source/index.rst @@ -39,6 +39,9 @@ distribution, we provide support for the following readers: * - GOES 16 imager data in netcdf format - `abi_l1b` - Nominal + * - GOES 11 to 15 imager data in HRIT format + - `hrit_goes` + - Nominal * - Electro-L N2 MSU-GS data in HRIT format - `hrit_electrol` - Nominal From df9dc30237ae831ee056469454acbb7853f0a7db Mon Sep 17 00:00:00 2001 From: Andrew Brooks Date: Tue, 19 Dec 2017 18:25:09 +0000 Subject: [PATCH 27/34] Added IPOPP-style MODIS-L1b filenames --- satpy/etc/readers/hdfeos_l1b.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/satpy/etc/readers/hdfeos_l1b.yaml b/satpy/etc/readers/hdfeos_l1b.yaml index 6ecb49a972..85bae431e0 100644 --- a/satpy/etc/readers/hdfeos_l1b.yaml +++ b/satpy/etc/readers/hdfeos_l1b.yaml @@ -423,11 +423,13 @@ file_types: - 'M{platform_indicator:1s}D02{resolution:1s}km_A{start_time:%y%j_%H%M%S}_{processing_time:%Y%j%H%M%S}.hdf' - 'M{platform_indicator:1s}D02{resolution:1s}km.A{start_time:%Y%j.%H%M}.{collection:03d}.{processing_time:%Y%j%H%M%S}.h5' - 'M{platform_indicator:1s}D02{resolution:1s}KM.A{start_time:%Y%j.%H%M}.{collection:03d}.{processing_time:%Y%j%H%M%S}.hdf' + - 'M{platform_indicator:1s}D02{resolution:1s}KM.{start_time:%y%j%H%M%S}.hdf' - '{platform_indicator:1s}1.{start_time:%y%j.%H%M}.{resolution:s}.hdf' file_reader: !!python/name:satpy.readers.hdfeos_l1b.HDFEOSBandReader '' hdf_eos_geo: file_patterns: - 'M{platform_indicator:1s}D03_A{start_time:%y%j_%H%M%S}_{processing_time:%Y%j%H%M%S}.hdf' - 'M{platform_indicator:1s}D03.A{start_time:%Y%j.%H%M}.{collection:03d}.{processing_time:%Y%j%H%M%S}.hdf' + - 'M{platform_indicator:1s}D03.{start_time:%y%j%H%M%S}.hdf' - '{platform_indicator:1s}1.{start_time:%y%j.%H%M}.geo.hdf' file_reader: !!python/name:satpy.readers.hdfeos_l1b.HDFEOSGeoReader '' From d11dfd971d74119115aa8c68c4028b718492080f Mon Sep 17 00:00:00 2001 From: "Adam.Dybbroe" Date: Tue, 19 Dec 2017 20:23:30 +0100 Subject: [PATCH 28/34] Replace effective_solar_pathlength_corrected witn the standard sunz-correction. VIIRS data are already sun-zenith corrected. Signed-off-by: Adam.Dybbroe --- satpy/etc/composites/viirs.yaml | 42 +++++++++++++++++++++++++-------- 1 file changed, 32 insertions(+), 10 deletions(-) diff --git a/satpy/etc/composites/viirs.yaml b/satpy/etc/composites/viirs.yaml index a1bf54b90e..bae715f8ee 100644 --- a/satpy/etc/composites/viirs.yaml +++ b/satpy/etc/composites/viirs.yaml @@ -22,14 +22,14 @@ composites: compositor: !!python/name:satpy.composites.viirs.RatioSharpenedRGB prerequisites: - name: M05 - modifiers: [effective_solar_pathlength_corrected, rayleigh_corrected] + modifiers: [sunz_corrected, rayleigh_corrected] - name: M04 - modifiers: [effective_solar_pathlength_corrected, rayleigh_corrected] + modifiers: [sunz_corrected, rayleigh_corrected] - name: M03 - modifiers: [effective_solar_pathlength_corrected, rayleigh_corrected] + modifiers: [sunz_corrected, rayleigh_corrected] optional_prerequisites: - name: I01 - modifiers: [effective_solar_pathlength_corrected, rayleigh_corrected] + modifiers: [sunz_corrected, rayleigh_corrected] standard_name: true_color high_resolution_band: red @@ -37,11 +37,33 @@ composites: compositor: !!python/name:satpy.composites.RGBCompositor prerequisites: - name: M05 - modifiers: [effective_solar_pathlength_corrected, rayleigh_corrected] + modifiers: [sunz_corrected, rayleigh_corrected] + - name: M04 + modifiers: [sunz_corrected, rayleigh_corrected] + - name: M03 + modifiers: [sunz_corrected, rayleigh_corrected] + standard_name: true_color + + true_color_lowres_crefl: + compositor: !!python/name:satpy.composites.RGBCompositor + prerequisites: + - name: M05 + modifiers: [sunz_corrected, rayleigh_corrected_crefl] + - name: M04 + modifiers: [sunz_corrected, rayleigh_corrected_crefl] + - name: M03 + modifiers: [sunz_corrected, rayleigh_corrected_crefl] + standard_name: true_color + + true_color_lowres_land: + compositor: !!python/name:satpy.composites.RGBCompositor + prerequisites: + - name: M05 + modifiers: [sunz_corrected, rayleigh_corrected_land] - name: M04 - modifiers: [effective_solar_pathlength_corrected, rayleigh_corrected] + modifiers: [sunz_corrected, rayleigh_corrected_land] - name: M03 - modifiers: [effective_solar_pathlength_corrected, rayleigh_corrected] + modifiers: [sunz_corrected, rayleigh_corrected_land] standard_name: true_color natural_color: @@ -63,11 +85,11 @@ composites: compositor: !!python/name:satpy.composites.RGBCompositor prerequisites: - name: M05 - modifiers: [effective_solar_pathlength_corrected] + modifiers: [sunz_corrected] - name: M04 - modifiers: [effective_solar_pathlength_corrected] + modifiers: [sunz_corrected] - name: M03 - modifiers: [effective_solar_pathlength_corrected] + modifiers: [sunz_corrected] standard_name: true_color night_overview: From 87ef63b0da2cd71ed7d7ea27107338427200c21a Mon Sep 17 00:00:00 2001 From: Martin Raspaud Date: Tue, 19 Dec 2017 21:16:29 +0100 Subject: [PATCH 29/34] =?UTF-8?q?Bump=20version:=200.7.5=20=E2=86=92=200.7?= =?UTF-8?q?.6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .bumpversion.cfg | 2 +- satpy/version.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index ec30fed7b0..8a491ec2cb 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 0.7.5 +current_version = 0.7.6 commit = True tag = True diff --git a/satpy/version.py b/satpy/version.py index 315f780937..d30d9e2535 100644 --- a/satpy/version.py +++ b/satpy/version.py @@ -23,4 +23,4 @@ """Version file. """ -__version__ = "0.7.5" +__version__ = "0.7.6" From 434c584f790d9e5579e060a5ecf172fedbb65eca Mon Sep 17 00:00:00 2001 From: Martin Raspaud Date: Tue, 19 Dec 2017 21:16:30 +0100 Subject: [PATCH 30/34] update changelog --- changelog.rst | 135 +++++++++++++++++++++++++------------------------- 1 file changed, 67 insertions(+), 68 deletions(-) diff --git a/changelog.rst b/changelog.rst index 4da845358e..0ace200671 100644 --- a/changelog.rst +++ b/changelog.rst @@ -2,6 +2,65 @@ Changelog ========= +v0.7.6 (2017-12-19) +------------------- +- Update changelog. [Martin Raspaud] +- Bump version: 0.7.5 → 0.7.6. [Martin Raspaud] +- Merge pull request #135 from pytroll/viirs_truecolor_config_error. + [Martin Raspaud] + + Replace effective_solar_pathlength_corrected with the standard sunz-corrected +- Replace effective_solar_pathlength_corrected witn the standard sunz- + correction. VIIRS data are already sun-zenith corrected. + [Adam.Dybbroe] +- Update documentation to add hrit_goes. [Martin Raspaud] +- Fix GOES navigation. [Martin Raspaud] +- Finalize GOES LRIT reader. [Martin Raspaud] +- Merge pull request #39 from howff/develop. [Martin Raspaud] + + Reader for GOES HRIT, WIP +- Fix available_composite_names in doc. [Andrew Brooks] +- Merge branch 'develop' of https://github.com/pytroll/satpy into + develop. [Andrew Brooks] +- Start of reader for GOES HRIT. [howff] +- Update PULL_REQUEST_TEMPLATE.md. [Martin Raspaud] + + This hides the comments when the PR is previewed and reminds user to provide a description for the PR. +- Merge pull request #122 from eysteinn/scatsat1. [Martin Raspaud] + + Add reader for ScatSat1 Level 2B wind speed data, HDF5 format +- Read end_time info correctly. [Eysteinn] +- Add reader for ScatSat1 Level 2B wind speed data. [Eysteinn] +- Merge pull request #129 from pytroll/viirs_rgbs. [Martin Raspaud] + + Use the Pyspectral atm correction as the default. +- Use the Pyspectral atm correction as the default. Add a high-res + overview RGB, use the hncc-dnb in the night-microphysics and use the + effective_solar_pathlength_corrected for all true color RGBs. + [Adam.Dybbroe] +- Merge pull request #128 from pytroll/atm_corrections. [Martin Raspaud] + + Atm corrections +- Pep8 cosmetics. [Adam.Dybbroe] +- Pep8 cosmetics. [Adam.Dybbroe] +- Pep8 editorial, and fixing copyright. [Adam.Dybbroe] +- Add some pre-defined atm/rayleigh corrections to appply over land and + sea. [Adam.Dybbroe] +- Merge pull request #131 from pytroll/bugfix-hrit-jma. [Martin Raspaud] + + Bugfix hrit_jma +- Bugfix hrit_jma. [Martin Raspaud] +- Use a more appropriate and shorter link to the MSG native format pdf + doc. [Adam.Dybbroe] +- Merge pull request #126 from pytroll/feature_ahi_stretch. [Martin + Raspaud] + + Improvemements to AHI True color imagery +- Use marine_clean and us-standard for atm correction, and improve + stretch at low sun elevation. [Adam.Dybbroe] +- Use the CIRA stretch for True color imagery. [Adam.Dybbroe] + + v0.7.5 (2017-12-11) ------------------- - Update changelog. [davidh-ssec] @@ -307,6 +366,9 @@ Other - Fix SCMI writer for lettered grids. [davidh-ssec] - Fix numbered tile counts for SCMI writer. [davidh-ssec] - Add initial SCMI writer. [davidh-ssec] + + WIP: Multiple tiles, lettered tiles, debug images + - Separate EnhancementDecisionTree in to base DecisionTree and subclass. [davidh-ssec] - Add 'goesr' as possible platform in geocat reader. [davidh-ssec] @@ -759,7 +821,6 @@ Other Conflicts: satpy/readers/__init__.py satpy/readers/hrit_msg.py - - Fix IR and VIS calibration. [Adam.Dybbroe] - Pep8 and editorial (header) updates. [Adam.Dybbroe] - Adding the native msg header record definitions. [Adam.Dybbroe] @@ -1098,11 +1159,12 @@ v0.3.1 (2017-01-16) and make the `safe_sar_c.py` reader compute coordinate arrays from a collection of GCPs provided in the measurement files. + NB: each polarization has it's set of longitudes and latitudes. + - Restore reducers to their original values. [Martin Raspaud] - Add alternatives for true color on ahi. [Martin Raspaud] Thanks balt - - Add name to the dataset attributes when writing nc files. [Martin Raspaud] - Improve documentation. [Martin Raspaud] @@ -1130,13 +1192,11 @@ v0.3.1 (2017-01-16) - Implement a mipp-free HRIT reader. [Martin Raspaud] WIP, supports only MSG, no calibration yet. - - Concatenate area_def through making new AreaDefinition. [Martin Raspaud] This makes the concatenation independent of the AreaDefinition implementation. - - Allow stacking area_def from bottom-up. [Martin Raspaud] - Fix yaml_reader testing. [Martin Raspaud] - Add support for filetype requirements. [Martin Raspaud] @@ -1203,12 +1263,10 @@ v0.3.0 (2016-12-13) Conflicts: satpy/readers/yaml_reader.py - - Merge branch 'develop' into feature-lonlat-datasets. [Martin Raspaud] Conflicts: satpy/readers/yaml_reader.py - - Pass down the calibration, polarization and resolution from main load. [Martin Raspaud] - Fix typo in sunzenith correction description. Default is 88 deg, not @@ -1227,7 +1285,6 @@ v0.3.0 (2016-12-13) the dataset at hand. - v0.2.1 (2016-12-08) ------------------- - Update changelog. [Martin Raspaud] @@ -1328,7 +1385,6 @@ Fix Prior to h5py 3.0, the h5 files open with h5py are not closed upon deletion, so we have to do it ourselves... - - Bugfix: area.id doesn't exist, use area.area_id. [Martin Raspaud] - Bugfix: return when each file has been loaded independently. [Martin Raspaud] @@ -1564,12 +1620,10 @@ Other This was triggering a `Too many open files` error since the memmap was called for every scanline. - - Fix loading for datasets with no navigation. [Martin Raspaud] - Read start and end time from filename for eps_l1b. [Martin Raspaud] This avoids opening every file just for time checks. - - Rename file handler's get_area to get_lonlats. [davidh-ssec] There is now a get_area_def and get_lonlats method on individual file handlers @@ -1628,7 +1682,6 @@ Other - Add mipp config file for MSG3. [Martin Raspaud] This is needed by mipp when the mipp_hrit reader is used. - - Remove `if True` from viirs sharp true color. [davidh-ssec] - Fix small bug in scene when dataset isn't found in a reader. [davidh- ssec] @@ -1643,11 +1696,9 @@ Other - Put AHI HSD reflectances in % [Martin Raspaud] They were between 0 and 1 by default - - Fix AHI HSD nav dtype. [Martin Raspaud] lon ssp and lat ssp where swaped - - Adjust correct standard names for seviri calibration. [Martin Raspaud] - Fix Seviri CO2 correction buggy yaml def. [Martin Raspaud] - Fix sunz corrector with different resolutions. [davidh-ssec] @@ -1705,7 +1756,6 @@ Other etc/composites/visir.yaml satpy/composites/__init__.py satpy/scene.py - - Add support for new prerequisite syntax. [Martin Raspaud] - Got VIIRS L1B True color working. [davidh-ssec] @@ -1719,7 +1769,6 @@ Other Conflicts: etc/composites/viirs.yaml satpy/readers/yaml_reader.py - - Add viirs composites. [Martin Raspaud] - Fix the area_def concatenation. [Martin Raspaud] - Mask nan in ir calibration for ahi hsd. [Martin Raspaud] @@ -1730,11 +1779,9 @@ Other This is enabled by implementing the `get_area_def` method in the file handler. - - Optimize AHI reading using inplace loading. [Martin Raspaud] Navigation is switched off for now. - - Allow area loading for the data file handlers. [Martin Raspaud] - Use a named tuple to pass both data, mask and info dict for inplace loading. [Martin Raspaud] @@ -1780,7 +1827,6 @@ Other - Avoid raising an error when no files are found. [Martin Raspaud] Instead, a warning is logged. - - Remove unused code from readers/__init__.py. [Martin Raspaud] - Cleanup style. [Martin Raspaud] - Fix unittests. [Martin Raspaud] @@ -1802,7 +1848,6 @@ Other - Add modifiers feature. [Martin Raspaud] Now modifiers can be added to the prerequisites as dictionnaries. - - Add standard_names to channels in mipp_xrit. [Martin Raspaud] - Add a NC4/CF writer. [Martin Raspaud] - Use YAML instead of CFG for composites. [Martin Raspaud] @@ -1843,7 +1888,6 @@ Other not covered by the YAMLBasedReader. Some parts of the class being still valid in this situation, we split the class to avoid code duplication, using subclassing instead. - - Add hrpt reader. [Martin Raspaud] - Change AMSR2 L1B reader config to be 2 spaces instead of 4. [davidh- ssec] @@ -1934,7 +1978,6 @@ Other The multiscene class adds the possibility to blend different datasets together, given a blend function. - - Add a test yaml-based reader for aapp1b. [Martin Raspaud] - Fix manually added datasets not being resampled. [davidh-ssec] - Merge pull request #8 from davidh-ssec/feature-ewa-resampling. [David @@ -1974,18 +2017,15 @@ Other No matching of file was done, resulting in assigning all found files to all readers. - - Fix reader_info reference in yaml base reader. [davidh-ssec] - Keep channels in the wishlist when necessary. [Martin Raspaud] Due to the creation of a DatasetID for each dataset key, the wishlist wasn't matching the actual ids of the datasets. - - Adapt reading to yaml reader way. [Martin Raspaud] Since there is more delegating of tasks to the reader, the reading has to be adapted. - - Cleanup using pep8. [Martin Raspaud] - Allow yaml files as config files. [Martin Raspaud] - Add the dependency tree based reading. [Martin Raspaud] @@ -1996,7 +2036,6 @@ Other The multiscene class adds the possibility to blend different datasets together, given a blend function. - - Add a test yaml-based reader for aapp1b. [Martin Raspaud] - Fix netcdf dimension use to work with older versions of netcdf-python library. [davidh-ssec] @@ -2095,7 +2134,6 @@ Other In the case of true color with crefl corrected channels for example, the true color needs to depend on 3 corrected channels, which in turn can now be composites. - - Add Scene import to __init__ for convience. [davidh-ssec] - Add composites to 'available_datasets' [davidh-ssec] @@ -2142,7 +2180,6 @@ Other In order to merge or keep metadata for Dataset during arithmetic operations we need to implement the numeric type methods. - - Cleanup unused arguments in base reader. [davidh-ssec] Also makes _load_navigation by renaming it to load_navigation to resolve some quantifiedcode code checks. @@ -2187,12 +2224,10 @@ Other 3d array masks were not precomputed correctly, so we now make a workaround. A better solution is yet to be found. - - Fix kd3 precomputation for AreaDefinitions. [Martin Raspaud] The lons and lats attributes aren't defined by default in AreaDefs, so we now make sure to call the get_lonlats method. - - Set default format for dataset saving to geotiff. [Martin Raspaud] - Move `save_datasets` logic from Scene to base Writer. [davidh-ssec] - Fix bug in resample when geolocation is 2D. [davidh-ssec] @@ -2227,12 +2262,10 @@ Other the dict `keys` method return views in py3. We now convert to list for consistency. - - Add a test case for resample caching. [Martin Raspaud] - Revert resample cache changes. [Martin Raspaud] They didn't seem necessary in the way resampling is called. - - Rename to satpy. [Martin Raspaud] - Remove the world_map.ascii file. [Martin Raspaud] - Allow compressed files to be checked by hrit reader. [Martin Raspaud] @@ -2259,35 +2292,29 @@ Other We add the .bumpversion.cfg and .gitchangelog.rc for easy version bumping and changelog updates. - - Remove v from version string. [Martin Raspaud] - Add str and repr methods for composites. [Martin Raspaud] This add simple repl and str methods for compositors. - - Restructure the documentation for mpop2. [Martin Raspaud] This is an attempt to reorganize the documentation to prepare for mpop2. Old stuff has been take away, and a fresh quickstart and api are now provided. - - Improve the ReaderFinder ImportError message to include original error. [Martin Raspaud] To make the ImportError more useful in ReaderFinder, the original error string is now provided. - - Fix save_dataset to allow both empty filename and writer. [Martin Raspaud] When saving a dataset without a filename and writer, save_dataset would crash. Instead, we are now putting writer to "simple_image" in that case. - - Rename projectable when assigning it through setitem. [Martin Raspaud] When a new dataset is added to a scene, it's name should match the string key provided by the user. - - Remove references to deprecated mpop.projector. [Martin Raspaud] - Allow resample to receive strings as area identifiers. [Martin Raspaud] @@ -2297,13 +2324,11 @@ Other area by name, than to get the area definition object from the file. This patch allows the `resample` projectable method to work with string ids also. - - Add a dataset to whishlish when added with setitem. [Martin Raspaud] When adding a dataset to a scene via the datasetdict.__setitem__ method, it is likely that the user case about this dataset. As such, it should be added to the wishlist in order not to get removed accidently. - - Move composite loading out of Scene to mpop.composites. [Martin Raspaud] @@ -2312,7 +2337,6 @@ Other place to have it is the mpop.composites modules. As a conterpart, we now provide the `available_composites` method to the Scene to be able to figure out what we have possibility to generate. - - Fix the travis file to allow python 2.6 to fail. [Martin Raspaud] - Allow travis to fail on python 2.6. [Martin Raspaud] - Install importlib for travis tests on python 2.6. [Martin Raspaud] @@ -2322,34 +2346,28 @@ Other Three scenarios were added, testing showing a dataset, saving a dataset, and bulk saving datasets (`save_datasets`). - - Fix loading behaviour tests. [Martin Raspaud] A little cleanup, and using builtin functions for getting the dataset_names - - Fix DatasetDict's setitem to allow empty md in value. [Martin Raspaud] Sometimes a dataset/projectable doesn't have any info attached to it, eg because the dataset is synthetic. In these cases, setitem would crash. This is now fixed, and if a string is provided as a key in setitem it is used as a name if no better name is already there. - - Simplify dataset saving to disk. [Martin Raspaud] saving datasets can now be done one by one. If a writer is not provided, it is guessed from the filename extension. - - Add a show method to the Scene class. [Martin Raspaud] That allows the user to interactively vizualize the data - - Add a default areas.def file. [Martin Raspaud] - Fix the manifest file to include the config files. [Martin Raspaud] - Add missing config files to setup.py. [Martin Raspaud] - Fix setup.py to add cfg files. [Martin Raspaud] This is in order to make mpop work out of the box after a pip install. - - Add a behaviour test to find out the available dataset. [Martin Raspaud] - Prevent crashing when a load requirement is not available. [Martin @@ -2357,11 +2375,9 @@ Other When requiring a band which isn't available, mpop would crash. This is now fixed and replaced by a warning in the log. - - Use behave to do higher level tests. [Martin Raspaud] Two small scenarios for testing the loading of the data are implemented now. - - Fix import error in scene. [davidh-ssec] A small refactor was done and then undone to move DatasetDict and DatasetID. This little import change wasn't properly cleaned up. @@ -2383,7 +2399,6 @@ Other config changing on the fly, but also more resilience for multiple sensor cases, like one sensor is loaded after another, and the composites wouldn't get updated. - - Fix the name issue in sensor-specific composite requests. [Martin Raspaud] @@ -2391,7 +2406,6 @@ Other should be empty or None, making it not read the sensor config file at all. In turn that meant that generic composites were used instead of sensor- specific ones. - - Got metadata requests working for composites. [davidh-ssec] - Use DatasetID in composite requirements instead of names and wavelengths only. [davidh-ssec] @@ -2475,7 +2489,6 @@ Other Raspaud] The class ReaderFinder was created for this purpose. - - Cleanup. [Martin Raspaud] - Fix overview and natural composites. [Martin Raspaud] - Make read and load argument lists consistent. [Martin Raspaud] @@ -2495,7 +2508,6 @@ Other "natural", "true_color"]) BandIDs are now used internally as key for the scene's projectables dict. - - Add file keys to metop's getitem. [Martin Raspaud] - Rename metop calibration functions. [Martin Raspaud] - Add file keys for start and end times for metop. [Martin Raspaud] @@ -2569,7 +2581,6 @@ Other - we can now provide "nearest" or "kdtree" instead of a resampler class. - The precompute/dump kwarg is now a directory where to save the proj info, defaulting to '.' if precompute=True. - - Switch to containers in travis. [Martin Raspaud] - Fix repo in .travis. [Martin Raspaud] - Add OrderedDict for python < 2.7. [Martin Raspaud] @@ -2580,7 +2591,6 @@ Other - OO architecture allowing other resampling methods to be implemented. - resampling is divided between pre- and actual computation. - hashing of areas is implemented, resampler-specific. - - Fixed bad patch on new scene test. [davidh-ssec] - First try at more scene tests. [davidh-ssec] - Move image generation methods to Dataset and move enh. application to @@ -2610,7 +2620,6 @@ Other mpop/satellites/__init__.py mpop/satin/helper_functions.py mpop/satin/mipp_xrit.py - - Add algorithm version in output cloud products. [Martin Raspaud] - Minor PEP8 tweaks. [Panu Lahtinen] - Script to generate external calibration files for AVHRR instruments. @@ -2705,7 +2714,6 @@ Other * Product dependencies * loading from viirs * generating images - - WIP: successfully loaded the first viirs granule with newscene! [Martin Raspaud] - Rewriting scene. [Martin Raspaud] @@ -2739,7 +2747,6 @@ Other - Fallback to pre-launch if not available. - load(..., pre_launch_coeffs=True) to force using pre-launch coeffs) - - Correct npp name in h5 files. [Martin Raspaud] - Add the pps v2014 h5 reader. [Martin Raspaud] - Use h5py for lonlat reading also. [Martin Raspaud] @@ -2820,7 +2827,6 @@ Other - Fix name matching in hdfeos_l1b. [Martin Raspaud] The full name didn't work with fnmatch, take basename instead. - - Allows hdfeos_l1b to read a batch of files. [Martin Raspaud] - Add delitem, and code cleanup. [Martin Raspaud] - Merge branch 'pre-master' of github.com:mraspaud/mpop into pre-master. @@ -2851,7 +2857,6 @@ Other - Allow loading a file directly for aapp1b and eps_l1b. [Martin Raspaud] Just run global_data.load(..., filename="/path/to/myfile.1b") - - Merge branch 'pre-master' of github.com:mraspaud/mpop into pre-master. [Martin Raspaud] - Viirs_sdr can now load depending on an area. [Martin Raspaud] @@ -2905,8 +2910,8 @@ Other - Merge pull request #10 from pnuu/pre-master. [Martin Raspaud] Fixed failed merging. Thanks Pnuu. -- Fixed failed merging (removed "<<<<<<< HEAD" and ">>>>>>> - upstream/pre-master" lines) [Panu Lahtinen] +- Fixed failed merging (removed "<<<<<<< HEAD" and ">>>>>>> upstream + /pre-master" lines) [Panu Lahtinen] - Merge branch 'pre-master' of https://github.com/mraspaud/mpop into pre-master. [Adam Dybbroe] - Merge branch 'pre-master' of https://github.com/mraspaud/mpop into @@ -3073,7 +3078,6 @@ Other Conflicts: mpop/imageo/geo_image.py - - Night_color (should had beed called night_overview) is the same as cloudtop. [Lars Orum Rasmussen] - Bug fix from Bocheng. [Lars Orum Rasmussen] @@ -3116,7 +3120,6 @@ Other channels 3a and 3b each time, one of them being entirely masked. This of course created some problem further down. Fixed by setting the not loadable channel to None. - - Merge branch 'unstable' of /data/proj/SAF/GIT/mpop into unstable. [Martin Raspaud] - Bugfix in npp.cfg template. [Adam Dybbroe] @@ -3139,7 +3142,6 @@ Other an allclose. This was inefficient, and the programming team decided that it was the user's task to know before projection if the source and target area were the same. In other words, the user should be at least a little smart. - - Remove dummy test to boost projection performance. [Martin Raspaud] Mpop was checking in 2 different places if the source and target areas were @@ -3148,7 +3150,6 @@ Other an allclose. This was inefficient, and the programming team decided that it was the user's task to know before projection if the source and target area were the same. In other words, the user should be at least a little smart. - - Update channel list for modis lvl2. [Martin Raspaud] - Bump up version number: 1.0.0. [Martin Raspaud] - Merge branch 'pre-master' into unstable. [Martin Raspaud] @@ -3329,7 +3330,6 @@ Other This fixes a bug in windows that prevents running strftime on string that contain mapping keys conversion specifiers. - - Catch the error if there is no file to load from. [Martin Raspaud] - Add a proper logger in hdfeos reader. [Martin Raspaud] - Get resolution from filename for eos data. [Martin Raspaud] @@ -3343,7 +3343,6 @@ Other - Fix the loading of BT for VIIRS M13 channel. [Martin Raspaud] Has no scale and offset - - Merge branch 'pre-master' of github.com:mraspaud/mpop into pre-master. [Lars Orum Rasmussen] - Refactor the unsigned netcdf packing code. [Martin Raspaud] @@ -3353,7 +3352,6 @@ Other - Replace auto mask and scale from netcdf4. [Martin Raspaud] Eats up too much memory. - - Merge branch 'pre-master' of github.com:mraspaud/mpop into pre-master. [Lars Orum Rasmussen] - Feature: Added template for electro-l satellite. [Martin Raspaud] @@ -4172,3 +4170,4 @@ Other - New rebase. [Martin Raspaud] + From 3d70e88d0ef519b47e3ac13c2bbb8e90f612fdf1 Mon Sep 17 00:00:00 2001 From: Andrew Brooks Date: Tue, 19 Dec 2017 21:21:32 +0000 Subject: [PATCH 31/34] Update doc re. IMAPP and IPOPP --- doc/source/index.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/source/index.rst b/doc/source/index.rst index 5141788a4b..d7c89c2d4f 100644 --- a/doc/source/index.rst +++ b/doc/source/index.rst @@ -66,7 +66,7 @@ distribution, we provide support for the following readers: * - Callipso Caliop data in EOS-hdf4 format - `hdf4_caliopv3` - Nominal - * - Terra and Aqua MODIS data in EOS-hdf4 format + * - Terra and Aqua MODIS data in EOS-hdf4 level-1 format as produced by IMAPP and IPOPP or downloaded from LAADS - `hdfeos_l1b` - Nominal * - NWCSAF MSG 2016 products in netCDF4 format From 0c1426e4507b82a3539226e48a26448e0b32b741 Mon Sep 17 00:00:00 2001 From: "Adam.Dybbroe" Date: Tue, 19 Dec 2017 23:07:43 +0100 Subject: [PATCH 32/34] When opening/reading a file fails, be verbose telling which file it is that fails Signed-off-by: Adam.Dybbroe --- satpy/readers/hdf5_utils.py | 17 +++++++++++++---- satpy/readers/netcdf_utils.py | 9 +++++++-- 2 files changed, 20 insertions(+), 6 deletions(-) diff --git a/satpy/readers/hdf5_utils.py b/satpy/readers/hdf5_utils.py index 1c2f99594b..d5ff599272 100644 --- a/satpy/readers/hdf5_utils.py +++ b/satpy/readers/hdf5_utils.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- -# Copyright (c) 2016. +# Copyright (c) 2016-2017. # Author(s): @@ -36,13 +36,21 @@ class HDF5FileHandler(BaseFileHandler): + """Small class for inspecting a HDF5 file and retrieve its metadata/header data. """ def __init__(self, filename, filename_info, filetype_info): - super(HDF5FileHandler, self).__init__(filename, filename_info, filetype_info) + super(HDF5FileHandler, self).__init__( + filename, filename_info, filetype_info) self.file_content = {} - file_handle = h5py.File(self.filename, 'r') + try: + file_handle = h5py.File(self.filename, 'r') + except IOError: + LOG.exception( + 'Failed reading file %s. Possibly corrupted file', self.filename) + raise + file_handle.visititems(self.collect_metadata) self._collect_attrs('', file_handle.attrs) file_handle.close() @@ -70,7 +78,8 @@ def collect_metadata(self, name, obj): def __getitem__(self, key): val = self.file_content[key] if isinstance(val, h5py.Dataset): - # these datasets are closed and inaccessible when the file is closed, need to reopen + # these datasets are closed and inaccessible when the file is + # closed, need to reopen return h5py.File(self.filename, 'r')[key].value return val diff --git a/satpy/readers/netcdf_utils.py b/satpy/readers/netcdf_utils.py index bda42be5f9..8e7ee1b4d2 100644 --- a/satpy/readers/netcdf_utils.py +++ b/satpy/readers/netcdf_utils.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- -# Copyright (c) 2016. +# Copyright (c) 2016-2017. # Author(s): @@ -67,7 +67,12 @@ def __init__(self, filename, filename_info, filetype_info, auto_maskandscale=Fal super(NetCDF4FileHandler, self).__init__( filename, filename_info, filetype_info) self.file_content = {} - file_handle = netCDF4.Dataset(self.filename, 'r') + try: + file_handle = netCDF4.Dataset(self.filename, 'r') + except IOError: + LOG.exception( + 'Failed reading file %s. Possibly corrupted file', self.filename) + raise self.auto_maskandscale = auto_maskandscale if hasattr(file_handle, "set_auto_maskandscale"): From d1a0746195509e9987b58399309c3e7204ec7be9 Mon Sep 17 00:00:00 2001 From: Martin Raspaud Date: Wed, 20 Dec 2017 11:20:36 +0100 Subject: [PATCH 33/34] Fix lon/lat caching in hdfeos_l1b for different resolutions Fixes #132 --- satpy/readers/hdfeos_l1b.py | 79 +++++++++++-------------------------- 1 file changed, 24 insertions(+), 55 deletions(-) diff --git a/satpy/readers/hdfeos_l1b.py b/satpy/readers/hdfeos_l1b.py index 429f9f7066..9a269e8b60 100644 --- a/satpy/readers/hdfeos_l1b.py +++ b/satpy/readers/hdfeos_l1b.py @@ -130,14 +130,19 @@ def __init__(self, filename, filename_info, filetype_info): else: self.resolution = 5000 self.cache = {} - self.cache['lons'] = None - self.cache['lats'] = None + self.cache[250] = {} + self.cache[250]['lons'] = None + self.cache[250]['lats'] = None - def get_area_def(self, *args, **kwargs): - raise NotImplementedError + self.cache[500] = {} + self.cache[500]['lons'] = None + self.cache[500]['lats'] = None - def get_dataset(self, key, info, out=None, xslice=None, yslice=None): + self.cache[1000] = {} + self.cache[1000]['lons'] = None + self.cache[1000]['lats'] = None + def get_dataset(self, key, info, out=None, xslice=None, yslice=None): if key.name in ['solar_zenith_angle', 'solar_azimuth_angle', 'satellite_zenith_angle', 'satellite_azimuth_angle']: @@ -157,7 +162,8 @@ def get_dataset(self, key, info, out=None, xslice=None, yslice=None): if key.name not in ['longitude', 'latitude']: return - if self.cache['lons'] is None or self.cache['lats'] is None: + if (self.cache[key.resolution]['lons'] is None or + self.cache[key.resolution]['lats'] is None): lons_id = DatasetID('longitude', resolution=key.resolution) @@ -166,14 +172,21 @@ def get_dataset(self, key, info, out=None, xslice=None, yslice=None): lons, lats = self.load( [lons_id, lats_id], interpolate=False, raw=True) - from geotiepoints.geointerpolator import GeoInterpolator - self.cache['lons'], self.cache['lats'] = self._interpolate( - [lons, lats], self.resolution, lons_id.resolution, GeoInterpolator) + if key.resolution != self.resolution: + from geotiepoints.geointerpolator import GeoInterpolator + lons, lats = self._interpolate([lons, lats], + self.resolution, + lons_id.resolution, + GeoInterpolator) + lons = np.ma.masked_invalid(np.ascontiguousarray(lons)) + lats = np.ma.masked_invalid(np.ascontiguousarray(lats)) + self.cache[key.resolution]['lons'] = lons + self.cache[key.resolution]['lats'] = lats if key.name == 'latitude': - return Dataset(self.cache['lats'], id=key, **info) + return Dataset(self.cache[key.resolution]['lats'], id=key, **info) else: - return Dataset(self.cache['lons'], id=key, **info) + return Dataset(self.cache[key.resolution]['lons'], id=key, **info) def load(self, keys, interpolate=True, raw=False): projectables = [] @@ -248,50 +261,6 @@ def _interpolate(data, coarse_resolution, resolution, interpolator=None): satint.fill_borders("y", "x") return satint.interpolate() - def get_lonlat(self, resolution, cores=1): - """Read lat and lon. - """ - if resolution in self.areas: - return self.areas[resolution] - logger.debug("generating lon, lat at %d", resolution) - if self.geofile is not None: - coarse_resolution = 1000 - filename = self.geofile - else: - coarse_resolution = 5000 - logger.info("Using 5km geolocation and interpolating") - filename = (self.datafiles.get(1000) or - self.datafiles.get(500) or - self.datafiles.get(250)) - - logger.debug("Loading geolocation from file: " + str(filename) - + " at resolution " + str(coarse_resolution)) - - data = SD(str(filename)) - lat = data.select("Latitude") - fill_value = lat.attributes()["_FillValue"] - lat = np.ma.masked_equal(lat.get(), fill_value) - lon = data.select("Longitude") - fill_value = lon.attributes()["_FillValue"] - lon = np.ma.masked_equal(lon.get(), fill_value) - - if resolution == coarse_resolution: - self.areas[resolution] = lon, lat - return lon, lat - - from geotiepoints import modis5kmto1km, modis1kmto500m, modis1kmto250m - logger.debug("Interpolating from " + str(coarse_resolution) - + " to " + str(resolution)) - if coarse_resolution == 5000: - lon, lat = modis5kmto1km(lon, lat) - if resolution == 500: - lon, lat = modis1kmto500m(lon, lat, cores) - if resolution == 250: - lon, lat = modis1kmto250m(lon, lat, cores) - - self.areas[resolution] = lon, lat - return lon, lat - class HDFEOSBandReader(HDFEOSFileReader): From ed7c0189b3e7e550bd0abc80df6e5154e18c9c26 Mon Sep 17 00:00:00 2001 From: Martin Raspaud Date: Wed, 20 Dec 2017 11:21:03 +0100 Subject: [PATCH 34/34] Clean up style --- satpy/enhancements/__init__.py | 13 ++++++------- satpy/readers/hdfeos_l1b.py | 14 +++++--------- 2 files changed, 11 insertions(+), 16 deletions(-) diff --git a/satpy/enhancements/__init__.py b/satpy/enhancements/__init__.py index 0778a7d5be..fb65af74b4 100644 --- a/satpy/enhancements/__init__.py +++ b/satpy/enhancements/__init__.py @@ -44,19 +44,21 @@ def invert(img, *args): def cira_stretch(img, **kwargs): - """Logarithmic stretch adapted to human vision.""" + """Logarithmic stretch adapted to human vision. + + Applicable only for visible channels. + """ LOG.debug("Applying the cira-stretch") for chn in img.channels: chn /= 100 np.log10(chn.data, out=chn.data) - # chn[:] = np.ma.masked_invalid(chn) chn -= np.log10(0.0223) chn /= 1.0 - np.log10(0.0223) chn /= 0.75 def lookup(img, **kwargs): - """Assigns values to channels based on a table""" + """Assign values to channels based on a table.""" luts = np.array(kwargs['luts'], dtype=np.float32) / 255.0 for idx, ch in enumerate(img.channels): @@ -67,14 +69,12 @@ def lookup(img, **kwargs): def colorize(img, **kwargs): """Colorize the given image.""" - full_cmap = _merge_colormaps(kwargs) img.colorize(full_cmap) def palettize(img, **kwargs): """Palettize the given image (no color interpolation).""" - full_cmap = _merge_colormaps(kwargs) img.palettize(full_cmap) @@ -95,8 +95,7 @@ def _merge_colormaps(kwargs): def create_colormap(palette): - """Create colormap of the given numpy file, color vector or colormap template.""" - + """Create colormap of the given numpy file, color vector or colormap.""" from trollimage.colormap import Colormap fname = palette.get('filename', None) if fname: diff --git a/satpy/readers/hdfeos_l1b.py b/satpy/readers/hdfeos_l1b.py index 9a269e8b60..05b5e64f0e 100644 --- a/satpy/readers/hdfeos_l1b.py +++ b/satpy/readers/hdfeos_l1b.py @@ -34,21 +34,13 @@ w=c&wchp=dGLzVlz-zSkWz&md5=bac5bc7a4f08007722ae793954f1dd63&ie=/sdarticle.pdf """ -import glob -import hashlib import logging -import math -import multiprocessing -import os.path from datetime import datetime -from fnmatch import fnmatch import numpy as np from pyhdf.error import HDF4Error from pyhdf.SD import SD -from pyresample import geometry -from satpy.config import CONFIG_PATH from satpy.dataset import Dataset, DatasetID from satpy.readers.file_handlers import BaseFileHandler @@ -143,6 +135,7 @@ def __init__(self, filename, filename_info, filetype_info): self.cache[1000]['lats'] = None def get_dataset(self, key, info, out=None, xslice=None, yslice=None): + """Get the dataset designated by *key*.""" if key.name in ['solar_zenith_angle', 'solar_azimuth_angle', 'satellite_zenith_angle', 'satellite_azimuth_angle']: @@ -189,6 +182,7 @@ def get_dataset(self, key, info, out=None, xslice=None, yslice=None): return Dataset(self.cache[key.resolution]['lons'], id=key, **info) def load(self, keys, interpolate=True, raw=False): + """Load the data.""" projectables = [] for key in keys: dataset = self.sd.select(key.name.capitalize()) @@ -200,7 +194,9 @@ def load(self, keys, interpolate=True, raw=False): data = np.ma.masked_equal(dataset.get(), fill_value) * scale_factor # TODO: interpolate if needed - if key.resolution is not None and key.resolution < self.resolution and interpolate: + if (key.resolution is not None and + key.resolution < self.resolution and + interpolate): data = self._interpolate(data, self.resolution, key.resolution) if raw: projectables.append(data)