diff --git a/satpy/dataset.py b/satpy/dataset.py index f4c042d630..677b447458 100644 --- a/satpy/dataset.py +++ b/satpy/dataset.py @@ -24,9 +24,11 @@ """Dataset objects. """ +import sys import logging import numbers from collections import namedtuple +from datetime import datetime import numpy as np @@ -46,20 +48,54 @@ def id(self): return DatasetID.from_dict(self.attrs) -def combine_metadata(*metadata_objects): +def average_datetimes(dt_list): + """Average a series of datetime objects. + + .. note:: + + This function assumes all datetime objects are naive and in the same + time zone (UTC). + + Args: + dt_list (iterable): Datetime objects to average + + Returns: Average datetime as a datetime object + + """ + if sys.version_info < (3, 3): + # timestamp added in python 3.3 + import time + + def timestamp_func(dt): + return time.mktime(dt.timetuple()) + else: + timestamp_func = datetime.timestamp + + total = [timestamp_func(dt) for dt in dt_list] + return datetime.fromtimestamp(sum(total) / len(total)) + + +def combine_metadata(*metadata_objects, **kwargs): """Combine the metadata of two or more Datasets. + If any keys are not equal or do not exist in all provided dictionaries + then they are not included in the returned dictionary. + By default any keys with the word 'time' in them and consisting + of datetime objects will be averaged. This is to handle cases where + data were observed at almost the same time but not exactly. + Args: *metadata_objects: MetadataObject or dict objects to combine + average_times (bool): Average any keys with 'time' in the name Returns: - the combined metadata + dict: the combined metadata """ + average_times = kwargs.get('average_times', True) # python 2 compatibility (no kwarg after *args) shared_keys = None info_dicts = [] - # grab all of the dictionary objects provided and make a set of the shared - # keys + # grab all of the dictionary objects provided and make a set of the shared keys for metadata_object in metadata_objects: if isinstance(metadata_object, dict): metadata_dict = metadata_object @@ -82,6 +118,8 @@ def combine_metadata(*metadata_objects): if any_arrays: if all(np.all(val == values[0]) for val in values[1:]): shared_info[k] = values[0] + elif 'time' in k and isinstance(values[0], datetime) and average_times: + shared_info[k] = average_datetimes(values) elif all(val == values[0] for val in values[1:]): shared_info[k] = values[0] diff --git a/satpy/readers/ahi_hsd.py b/satpy/readers/ahi_hsd.py index ac8b22383c..929b83c589 100644 --- a/satpy/readers/ahi_hsd.py +++ b/satpy/readers/ahi_hsd.py @@ -1,31 +1,41 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- - +# # Copyright (c) 2014-2018 PyTroll developers - +# # Author(s): - +# # Adam.Dybbroe # Cooke, Michael.C, UK Met Office # Martin Raspaud - +# # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. - +# # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. - +# # You should have received a copy of the GNU General Public License # along with this program. If not, see . -"""Advanced Himawari Imager (AHI) standard format data reader +"""Advanced Himawari Imager (AHI) standard format data reader. The HSD format +that this reader reads are described at the URL below: http://www.data.jma.go.jp/mscweb/en/himawari89/space_segment/spsg_ahi.html +Time Information +**************** + +AHI observations use the idea of a "scheduled" time and an "observation time. +The "scheduled" time is when the instrument was told to record the data, +usually at a specific and consistent interval. The "observation" time is when +the data was actually observed. Scheduled time can be accessed from the +`scheduled_time` metadata key and observation time from the `start_time` key. + """ import logging @@ -44,11 +54,6 @@ "6", "7", "8", "9", "10", "11", "12", "13", "14", "15", "16") - -class CalibrationError(ValueError): - pass - - logger = logging.getLogger('ahi_hsd') # Basic information block: @@ -215,9 +220,7 @@ class CalibrationError(ValueError): class AHIHSDFileHandler(BaseFileHandler): - - """AHI standard format reader - """ + """AHI standard format reader.""" def __init__(self, filename, filename_info, filetype_info): """Initialize the reader.""" @@ -250,18 +253,19 @@ def __init__(self, filename, filename_info, filetype_info): self.platform_name = np2str(self.basic_info['satellite']) self.sensor = 'ahi' - def get_shape(self, dsid, ds_info): - return int(self.data_info['number_of_lines']), int(self.data_info['number_of_columns']) - @property def start_time(self): - return (datetime(1858, 11, 17) + - timedelta(days=float(self.basic_info['observation_start_time']))) + return datetime(1858, 11, 17) + timedelta(days=float(self.basic_info['observation_start_time'])) @property def end_time(self): - return (datetime(1858, 11, 17) + - timedelta(days=float(self.basic_info['observation_end_time']))) + return datetime(1858, 11, 17) + timedelta(days=float(self.basic_info['observation_end_time'])) + + @property + def scheduled_time(self): + """Time this band was scheduled to be recorded.""" + timeline = "{:04d}".format(self.basic_info['observation_timeline'][0]) + return self.start_time.replace(hour=int(timeline[:2]), minute=int(timeline[2:4]), second=0, microsecond=0) def get_dataset(self, key, info): return self.read_band(key, info) @@ -311,10 +315,6 @@ def get_area_def(self, dsid): self.area = area return area - def get_lonlats(self, key, info, lon_out, lat_out): - logger.debug('Computing area for %s', str(key)) - lon_out[:], lat_out[:] = self.area.get_lonlats() - def geo_mask(self): """Masking the space pixels from geometry info.""" cfac = np.uint32(self.proj_info['CFAC']) @@ -346,7 +346,7 @@ def ellipse(line, col): return ellipse(lines_idx[:, None], cols_idx[None, :]) def read_band(self, key, info): - """Read the data""" + """Read the data.""" tic = datetime.now() header = {} with open(self.filename, "rb") as fp_: @@ -428,19 +428,17 @@ def read_band(self, key, info): dtype=' - +# +# Copyright (c) 2015-2018 SatPy Developers +# # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. - +# # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. - +# # You should have received a copy of the GNU General Public License # along with this program. If not, see . - -"""test projectable objects. +"""Test objects and functions in the dataset module. """ -import unittest +import sys +from datetime import datetime + +if sys.version_info < (2, 7): + import unittest2 as unittest +else: + import unittest class TestDatasetID(unittest.TestCase): + """Test DatasetID object creation and other methods.""" def test_basic_init(self): """Test basic ways of creating a DatasetID.""" @@ -57,11 +59,45 @@ def test_compare_no_wl(self): self.assertTrue(d2 < d1) +class TestCombineMetadata(unittest.TestCase): + """Test how metadata is combined.""" + + def test_average_datetimes(self): + """Test the average_datetimes helper function.""" + from satpy.dataset import average_datetimes + dts = ( + datetime(2018, 2, 1, 11, 58, 0), + datetime(2018, 2, 1, 11, 59, 0), + datetime(2018, 2, 1, 12, 0, 0), + datetime(2018, 2, 1, 12, 1, 0), + datetime(2018, 2, 1, 12, 2, 0), + ) + ret = average_datetimes(dts) + self.assertEqual(dts[2], ret) + + def test_combine_times(self): + """Test the combine_metadata with times.""" + from satpy.dataset import combine_metadata + dts = ( + {'start_time': datetime(2018, 2, 1, 11, 58, 0)}, + {'start_time': datetime(2018, 2, 1, 11, 59, 0)}, + {'start_time': datetime(2018, 2, 1, 12, 0, 0)}, + {'start_time': datetime(2018, 2, 1, 12, 1, 0)}, + {'start_time': datetime(2018, 2, 1, 12, 2, 0)}, + ) + ret = combine_metadata(*dts) + self.assertEqual(dts[2]['start_time'], ret['start_time']) + ret = combine_metadata(*dts, average_times=False) + # times are not equal so don't include it in the final result + self.assertNotIn('start_time', ret) + + def suite(): """The test suite for test_projector. """ loader = unittest.TestLoader() my_suite = unittest.TestSuite() my_suite.addTest(loader.loadTestsFromTestCase(TestDatasetID)) + my_suite.addTest(loader.loadTestsFromTestCase(TestCombineMetadata)) return my_suite