From b41c18920b99bac7cfa243384465395e1092d53c Mon Sep 17 00:00:00 2001 From: "david.bentley" Date: Wed, 8 Dec 2021 17:30:56 +0000 Subject: [PATCH 1/4] alter calculation of offset --- iris_grib/__init__.py | 6 ++---- iris_grib/message.py | 6 ++---- 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/iris_grib/__init__.py b/iris_grib/__init__.py index 118886dd..43cbb23e 100644 --- a/iris_grib/__init__.py +++ b/iris_grib/__init__.py @@ -150,6 +150,7 @@ def __init__(self, grib_message, grib_fh=None): offset = grib_fh.tell() message_length = eccodes.codes_get_long( grib_message, 'totalLength') + offset = gribapi.grib_get_message_offset(grib_message) # Initialise the key-extension dictionary. # NOTE: this attribute *must* exist, or the the __getattr__ overload @@ -170,11 +171,8 @@ def __init__(self, grib_message, grib_fh=None): if deferred: # Wrap the reference to the data payload within the data proxy # in order to support deferred data loading. - # The byte offset requires to be reset back to the first byte - # of this message. The file pointer offset is always at the end - # of the current message due to the grib-api reading the message. proxy = GribDataProxy(shape, np.array([0.]).dtype, grib_fh.name, - offset - message_length) + offset) self._data = as_lazy_data(proxy) else: self.data = _message_values(grib_message, shape) diff --git a/iris_grib/message.py b/iris_grib/message.py index 41d7db1c..ea56b261 100644 --- a/iris_grib/message.py +++ b/iris_grib/message.py @@ -59,12 +59,10 @@ def messages_from_filename(filename): file_ref = _OpenFileRef(grib_fh) while True: - offset = grib_fh.tell() - grib_id = eccodes.codes_new_from_file( - grib_fh, eccodes.CODES_PRODUCT_GRIB - ) + grib_id = gribapi.grib_new_from_file(grib_fh) if grib_id is None: break + offset = gribapi.grib_get_message_offset(grib_id) raw_message = _RawGribMessage(grib_id) recreate_raw = _MessageLocation(filename, offset) yield GribMessage(raw_message, recreate_raw, file_ref=file_ref) From 0c185d6e3f12f3c8d6689562050554dd3cc91568 Mon Sep 17 00:00:00 2001 From: "david.bentley" Date: Fri, 10 Dec 2021 09:49:03 +0000 Subject: [PATCH 2/4] update test to mock grib_get_message_offset --- iris_grib/__init__.py | 5 ----- iris_grib/tests/unit/test_GribWrapper.py | 17 ++++++++++++----- 2 files changed, 12 insertions(+), 10 deletions(-) diff --git a/iris_grib/__init__.py b/iris_grib/__init__.py index 43cbb23e..12f49f78 100644 --- a/iris_grib/__init__.py +++ b/iris_grib/__init__.py @@ -145,11 +145,6 @@ def __init__(self, grib_message, grib_fh=None): # Store the file pointer and message length from the current # grib message before it's changed by calls to the grib-api. if deferred: - # Note that, the grib-api has already read this message and - # advanced the file pointer to the end of the message. - offset = grib_fh.tell() - message_length = eccodes.codes_get_long( - grib_message, 'totalLength') offset = gribapi.grib_get_message_offset(grib_message) # Initialise the key-extension dictionary. diff --git a/iris_grib/tests/unit/test_GribWrapper.py b/iris_grib/tests/unit/test_GribWrapper.py index 48f703b7..c394b705 100644 --- a/iris_grib/tests/unit/test_GribWrapper.py +++ b/iris_grib/tests/unit/test_GribWrapper.py @@ -21,6 +21,7 @@ from iris_grib import GribWrapper, _load_generate +_offset = 0 _message_length = 1000 @@ -50,6 +51,10 @@ def _mock_codes_get_native_type(grib_message, key): return result +def _mock_codes_get_message_offset(grib_message, key): + return _offset + + class Test_edition(tests.IrisGribTest): def setUp(self): self.patch('iris_grib.GribWrapper._confirm_in_scope') @@ -58,7 +63,8 @@ def setUp(self): self.patch('eccodes.codes_get_string', _mock_codes_get_string) self.patch('eccodes.codes_get_native_type', _mock_codes_get_native_type) - self.tell = mock.Mock(side_effect=[_message_length]) + self.patch('eccodes.codes_get_message_offset', + _mock_codes_get_message_offset) def test_not_edition_1(self): def func(grib_message, key): @@ -71,7 +77,7 @@ def func(grib_message, key): def test_edition_1(self): grib_message = 'regular_ll' - grib_fh = mock.Mock(tell=self.tell) + grib_fh = mock.Mock() wrapper = GribWrapper(grib_message, grib_fh) self.assertEqual(wrapper.grib_message, grib_message) @@ -99,9 +105,10 @@ def setUp(self): self.patch('eccodes.codes_get_string', _mock_codes_get_string) self.patch('eccodes.codes_get_native_type', _mock_codes_get_native_type) - tell_tale = np.arange(1, 5) * _message_length - self.expected = tell_tale - _message_length - self.grib_fh = mock.Mock(tell=mock.Mock(side_effect=tell_tale)) + self.patch('eccodes.codes_get_message_offset', + _mock_codes_get_message_offset) + self.expected = np.atleast_1d(_offset) + self.grib_fh = mock.Mock() self.dtype = np.float64 self.path = self.grib_fh.name self.lookup = _mock_codes_get_long From 075ce91b6f0a216ff4d5689587255c19aecb4b11 Mon Sep 17 00:00:00 2001 From: "david.bentley" Date: Wed, 5 Jul 2023 11:31:39 +0100 Subject: [PATCH 3/4] swap gribapi for eccodes --- iris_grib/__init__.py | 2 +- iris_grib/message.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/iris_grib/__init__.py b/iris_grib/__init__.py index 12f49f78..c22f59aa 100644 --- a/iris_grib/__init__.py +++ b/iris_grib/__init__.py @@ -145,7 +145,7 @@ def __init__(self, grib_message, grib_fh=None): # Store the file pointer and message length from the current # grib message before it's changed by calls to the grib-api. if deferred: - offset = gribapi.grib_get_message_offset(grib_message) + offset = eccodes.codes_get_message_offset(grib_message) # Initialise the key-extension dictionary. # NOTE: this attribute *must* exist, or the the __getattr__ overload diff --git a/iris_grib/message.py b/iris_grib/message.py index ea56b261..b34958f9 100644 --- a/iris_grib/message.py +++ b/iris_grib/message.py @@ -59,10 +59,10 @@ def messages_from_filename(filename): file_ref = _OpenFileRef(grib_fh) while True: - grib_id = gribapi.grib_new_from_file(grib_fh) + grib_id = eccodes.codes_new_from_file(grib_fh) if grib_id is None: break - offset = gribapi.grib_get_message_offset(grib_id) + offset = eccodes.codes_get_message_offset(grib_id) raw_message = _RawGribMessage(grib_id) recreate_raw = _MessageLocation(filename, offset) yield GribMessage(raw_message, recreate_raw, file_ref=file_ref) From fa65b988f5e514b1adeb61e24014d4531c6fbc6a Mon Sep 17 00:00:00 2001 From: "david.bentley" Date: Wed, 5 Jul 2023 11:44:12 +0100 Subject: [PATCH 4/4] correct arguments to fix failing tests --- iris_grib/message.py | 3 ++- iris_grib/tests/unit/test_GribWrapper.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/iris_grib/message.py b/iris_grib/message.py index b34958f9..4d673ed9 100644 --- a/iris_grib/message.py +++ b/iris_grib/message.py @@ -59,7 +59,8 @@ def messages_from_filename(filename): file_ref = _OpenFileRef(grib_fh) while True: - grib_id = eccodes.codes_new_from_file(grib_fh) + grib_id = eccodes.codes_new_from_file(grib_fh, + eccodes.CODES_PRODUCT_GRIB) if grib_id is None: break offset = eccodes.codes_get_message_offset(grib_id) diff --git a/iris_grib/tests/unit/test_GribWrapper.py b/iris_grib/tests/unit/test_GribWrapper.py index c394b705..d3d5964e 100644 --- a/iris_grib/tests/unit/test_GribWrapper.py +++ b/iris_grib/tests/unit/test_GribWrapper.py @@ -51,7 +51,7 @@ def _mock_codes_get_native_type(grib_message, key): return result -def _mock_codes_get_message_offset(grib_message, key): +def _mock_codes_get_message_offset(grib_message): return _offset