From 5401d28011fb22543daea6928f376c13eb957c28 Mon Sep 17 00:00:00 2001 From: Pouria Khalaj Date: Fri, 26 Jul 2024 13:39:28 +0200 Subject: [PATCH 01/22] Add `readers.utils.fromfile()` for remote reading This function uses `readers.utils.generic_open()` and `np.frombuffer()` to achieve this. --- satpy/readers/utils.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/satpy/readers/utils.py b/satpy/readers/utils.py index 170cc5abcc..378f16de0b 100644 --- a/satpy/readers/utils.py +++ b/satpy/readers/utils.py @@ -361,6 +361,27 @@ def generic_open(filename, *args, **kwargs): fp.close() +def fromfile(filename, dtype, count=1, offset=0): + """Reads the numpy array from a (remote or local) file using a buffer. + + Note: + This function relies on the :func:`generic_open` context manager to read a file remotely. + + Args: + filename: Either the name of the file to read or a :class:`satpy.readers.FSFile` object. + dtype: The data type of the numpy array + count (Optional, default ``1``): Number of items to read + offset (Optional, default ``0``): Starting point for reading the buffer from + + Returns: + The content of the filename as a numpy array with the given data type. + """ + with generic_open(filename, mode="rb") as istream: + istream.seek(offset) + content = np.frombuffer(istream.read(dtype.itemsize * count), dtype=dtype, count=count) + return content + + def bbox(img): """Find the bounding box around nonzero elements in the given array. From 7142a682a1d75f27fdc6cb79cfc4bb80382bc44a Mon Sep 17 00:00:00 2001 From: Pouria Khalaj Date: Fri, 26 Jul 2024 13:51:02 +0200 Subject: [PATCH 02/22] Update `AUTHORS.md` --- AUTHORS.md | 1 + 1 file changed, 1 insertion(+) diff --git a/AUTHORS.md b/AUTHORS.md index 461ce3ca1e..e764b96627 100644 --- a/AUTHORS.md +++ b/AUTHORS.md @@ -43,6 +43,7 @@ The following people have made contributions to this project: - [Jactry Zeng](https://github.com/jactry) - [Johannes Johansson (JohannesSMHI)](https://github.com/JohannesSMHI) - [Sauli Joro (sjoro)](https://github.com/sjoro) +- [Pouria Khalaj](https://github.com/pkhalaj) - [Janne Kotro (jkotro)](https://github.com/jkotro) - [Ralph Kuehn (ralphk11)](https://github.com/ralphk11) - [Panu Lahtinen (pnuu)](https://github.com/pnuu) From 8051d50703c366989a99618fd350d600e502ba40 Mon Sep 17 00:00:00 2001 From: Pouria Khalaj Date: Fri, 26 Jul 2024 13:54:25 +0200 Subject: [PATCH 03/22] Reformat `readers.utils` to make it PEP8 compliant --- satpy/readers/utils.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/satpy/readers/utils.py b/satpy/readers/utils.py index 378f16de0b..e45e78aef1 100644 --- a/satpy/readers/utils.py +++ b/satpy/readers/utils.py @@ -98,8 +98,8 @@ def get_geostationary_angle_extent(geos_area): h = float(h) / 1000 + req # compute some constants - aeq = 1 - req**2 / (h ** 2) - ap_ = 1 - rp**2 / (h ** 2) + aeq = 1 - req ** 2 / (h ** 2) + ap_ = 1 - rp ** 2 / (h ** 2) # generate points around the north hemisphere in satellite projection # make it a bit smaller so that we stay inside the valid area @@ -142,15 +142,15 @@ def _lonlat_from_geos_angle(x, y, geos_area): b__ = (a / float(b)) ** 2 sd = np.sqrt((h__ * np.cos(x) * np.cos(y)) ** 2 - - (np.cos(y)**2 + b__ * np.sin(y)**2) * - (h__**2 - (float(a) / 1000)**2)) + (np.cos(y) ** 2 + b__ * np.sin(y) ** 2) * + (h__ ** 2 - (float(a) / 1000) ** 2)) # sd = 0 - sn = (h__ * np.cos(x) * np.cos(y) - sd) / (np.cos(y)**2 + b__ * np.sin(y)**2) + sn = (h__ * np.cos(x) * np.cos(y) - sd) / (np.cos(y) ** 2 + b__ * np.sin(y) ** 2) s1 = h__ - sn * np.cos(x) * np.cos(y) s2 = sn * np.sin(x) * np.cos(y) s3 = -sn * np.sin(y) - sxy = np.sqrt(s1**2 + s2**2) + sxy = np.sqrt(s1 ** 2 + s2 ** 2) lons = np.rad2deg(np.arctan2(s2, s1)) + lon_0 lats = np.rad2deg(-np.arctan2(b__ * s3, sxy)) @@ -256,7 +256,7 @@ def _unzip_with_pbzip(filename, tmpfilepath, fdn): if n_thr: runner = [pbzip, "-dc", - "-p"+str(n_thr), + "-p" + str(n_thr), filename] else: runner = [pbzip, @@ -416,7 +416,7 @@ def get_earth_radius(lon, lat, a, b): latlong = pyproj.CRS.from_dict({"proj": "latlong", "a": a, "b": b, "units": "m"}) transformer = pyproj.Transformer.from_crs(latlong, geocent) x, y, z = transformer.transform(lon, lat, 0.0) - return np.sqrt(x**2 + y**2 + z**2) + return np.sqrt(x ** 2 + y ** 2 + z ** 2) def reduce_mda(mda, max_size=100): From 16f26425f8a7d4460d9689f51b13264b46b31f8e Mon Sep 17 00:00:00 2001 From: Pouria Khalaj Date: Fri, 26 Jul 2024 14:14:39 +0200 Subject: [PATCH 04/22] Adapt `readers.seviri_l1b_native` for remote reading In particular, the following functions/methods have been modified: - `has_archive_header()` now uses `readers.utils.generic_open()` instead of `open()`. - `read_header()` now uses `readers.utils.fromfile()` instead of `np.formfile()`. - `NativeMSGFileHandler._read_trailer()` now uses `readers.utils.fromfile()` instead of `np.formfile()`. - `NativeMSGFileHandler._get_memmap()` has been renamed to `NativeMSGFileHandler._get_array()` and now uses `readers.utils.fromfile()` instead of `np.memmap()`. --- satpy/readers/seviri_l1b_native.py | 26 ++++++++++---------------- 1 file changed, 10 insertions(+), 16 deletions(-) diff --git a/satpy/readers/seviri_l1b_native.py b/satpy/readers/seviri_l1b_native.py index 976cb7c338..d5f3ba2692 100644 --- a/satpy/readers/seviri_l1b_native.py +++ b/satpy/readers/seviri_l1b_native.py @@ -138,7 +138,7 @@ get_native_header, native_trailer, ) -from satpy.readers.utils import reduce_mda +from satpy.readers.utils import fromfile, generic_open, reduce_mda from satpy.utils import get_legacy_chunk_size logger = logging.getLogger("native_msg") @@ -193,7 +193,7 @@ def __init__(self, filename, filename_info, filetype_info, # Available channels are known only after the header has been read self.header_type = get_native_header(has_archive_header(self.filename)) self._read_header() - self.dask_array = da.from_array(self._get_memmap(), chunks=(CHUNK_SIZE,)) + self.dask_array = da.from_array(self._get_array(), chunks=(CHUNK_SIZE,)) self._read_trailer() self.image_boundaries = ImageBoundaries(self.header, self.trailer, self.mda) @@ -276,15 +276,11 @@ def get_lrec(cols): return np.dtype(drec) - def _get_memmap(self): - """Get the memory map for the SEVIRI data.""" - with open(self.filename) as fp: - data_dtype = self._get_data_dtype() - hdr_size = self.header_type.itemsize - - return np.memmap(fp, dtype=data_dtype, - shape=(self.mda["number_of_lines"],), - offset=hdr_size, mode="r") + def _get_array(self): + """Get the numpy array for the SEVIRI data.""" + data_dtype = self._get_data_dtype() + hdr_size = self.header_type.itemsize + return fromfile(self.filename, dtype=data_dtype, offset=hdr_size, count=self.mda["number_of_lines"]) def _read_header(self): """Read the header info.""" @@ -387,9 +383,7 @@ def _read_trailer(self): data_size = (self._get_data_dtype().itemsize * self.mda["number_of_lines"]) - with open(self.filename) as fp: - fp.seek(hdr_size + data_size) - data = np.fromfile(fp, dtype=native_trailer, count=1) + data = fromfile(self.filename, dtype=native_trailer, count=1, offset=hdr_size + data_size) self.trailer.update(recarray2dict(data)) @@ -888,12 +882,12 @@ def get_available_channels(header): def has_archive_header(filename): """Check whether the file includes an ASCII archive header.""" - with open(filename, mode="rb") as istream: + with generic_open(filename, mode="rb") as istream: return istream.read(36) == ASCII_STARTSWITH def read_header(filename): """Read SEVIRI L1.5 native header.""" dtype = get_native_header(has_archive_header(filename)) - hdr = np.fromfile(filename, dtype=dtype, count=1) + hdr = fromfile(filename, dtype=dtype, count=1) return recarray2dict(hdr) From 1f106903afae1a86aac3ac8e5e66636d752dc28f Mon Sep 17 00:00:00 2001 From: Pouria Khalaj Date: Fri, 26 Jul 2024 14:24:24 +0200 Subject: [PATCH 05/22] Update `test_seviri_l1b_native` with `_get_array` Reason: since `NativeMSGFileHandler._get_memmap()` has been renamed to `NativeMSGFileHandler._get_array()`. --- .../tests/reader_tests/test_seviri_l1b_native.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/satpy/tests/reader_tests/test_seviri_l1b_native.py b/satpy/tests/reader_tests/test_seviri_l1b_native.py index 8f4e46e2fb..c8501cb6a7 100644 --- a/satpy/tests/reader_tests/test_seviri_l1b_native.py +++ b/satpy/tests/reader_tests/test_seviri_l1b_native.py @@ -634,7 +634,7 @@ def prepare_area_definitions(test_dict): with mock.patch("satpy.readers.seviri_l1b_native.np.fromfile") as fromfile, \ mock.patch("satpy.readers.seviri_l1b_native.recarray2dict") as recarray2dict, \ - mock.patch("satpy.readers.seviri_l1b_native.NativeMSGFileHandler._get_memmap") as _get_memmap, \ + mock.patch("satpy.readers.seviri_l1b_native.NativeMSGFileHandler._get_array") as _get_array, \ mock.patch("satpy.readers.seviri_l1b_native.NativeMSGFileHandler._read_trailer"), \ mock.patch( "satpy.readers.seviri_l1b_native.has_archive_header" @@ -642,7 +642,7 @@ def prepare_area_definitions(test_dict): has_archive_header.return_value = True fromfile.return_value = header recarray2dict.side_effect = (lambda x: x) - _get_memmap.return_value = np.arange(3) + _get_array.return_value = np.arange(3) fh = NativeMSGFileHandler(filename=None, filename_info={}, filetype_info=None) fh.fill_disk = fill_disk fh.header = header @@ -718,7 +718,7 @@ def prepare_is_roi(test_dict): with mock.patch("satpy.readers.seviri_l1b_native.np.fromfile") as fromfile, \ mock.patch("satpy.readers.seviri_l1b_native.recarray2dict") as recarray2dict, \ - mock.patch("satpy.readers.seviri_l1b_native.NativeMSGFileHandler._get_memmap") as _get_memmap, \ + mock.patch("satpy.readers.seviri_l1b_native.NativeMSGFileHandler._get_array") as _get_array, \ mock.patch("satpy.readers.seviri_l1b_native.NativeMSGFileHandler._read_trailer"), \ mock.patch( "satpy.readers.seviri_l1b_native.has_archive_header" @@ -726,7 +726,7 @@ def prepare_is_roi(test_dict): has_archive_header.return_value = True fromfile.return_value = header recarray2dict.side_effect = (lambda x: x) - _get_memmap.return_value = np.arange(3) + _get_array.return_value = np.arange(3) fh = NativeMSGFileHandler(filename=None, filename_info={}, filetype_info=None) fh.header = header fh.trailer = trailer @@ -1168,12 +1168,12 @@ def test_header_type(file_content, exp_header_size): header.pop("15_SECONDARY_PRODUCT_HEADER") with mock.patch("satpy.readers.seviri_l1b_native.np.fromfile") as fromfile, \ mock.patch("satpy.readers.seviri_l1b_native.recarray2dict") as recarray2dict, \ - mock.patch("satpy.readers.seviri_l1b_native.NativeMSGFileHandler._get_memmap") as _get_memmap, \ + mock.patch("satpy.readers.seviri_l1b_native.NativeMSGFileHandler._get_array") as _get_array, \ mock.patch("satpy.readers.seviri_l1b_native.NativeMSGFileHandler._read_trailer"), \ mock.patch("builtins.open", mock.mock_open(read_data=file_content)): fromfile.return_value = header recarray2dict.side_effect = (lambda x: x) - _get_memmap.return_value = np.arange(3) + _get_array.return_value = np.arange(3) fh = NativeMSGFileHandler(filename=None, filename_info={}, filetype_info=None) assert fh.header_type.itemsize == exp_header_size assert "15_SECONDARY_PRODUCT_HEADER" in fh.header @@ -1198,11 +1198,11 @@ def test_header_warning(): with mock.patch("satpy.readers.seviri_l1b_native.np.fromfile") as fromfile, \ mock.patch("satpy.readers.seviri_l1b_native.recarray2dict") as recarray2dict, \ - mock.patch("satpy.readers.seviri_l1b_native.NativeMSGFileHandler._get_memmap") as _get_memmap, \ + mock.patch("satpy.readers.seviri_l1b_native.NativeMSGFileHandler._get_array") as _get_array, \ mock.patch("satpy.readers.seviri_l1b_native.NativeMSGFileHandler._read_trailer"), \ mock.patch("builtins.open", mock.mock_open(read_data=ASCII_STARTSWITH)): recarray2dict.side_effect = (lambda x: x) - _get_memmap.return_value = np.arange(3) + _get_array.return_value = np.arange(3) exp_warning = "The quality flag for this file indicates not OK. Use this data with caution!" From 7d516d82a94f0a6f74902b83ab9956405240686d Mon Sep 17 00:00:00 2001 From: Pouria Khalaj Date: Fri, 26 Jul 2024 14:49:00 +0200 Subject: [PATCH 06/22] Update mock.patch args in `test_seviri_l1b_native` to match changes in `seviri_l1b_native` --- .../tests/reader_tests/test_seviri_l1b_native.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/satpy/tests/reader_tests/test_seviri_l1b_native.py b/satpy/tests/reader_tests/test_seviri_l1b_native.py index c8501cb6a7..38a4e16e77 100644 --- a/satpy/tests/reader_tests/test_seviri_l1b_native.py +++ b/satpy/tests/reader_tests/test_seviri_l1b_native.py @@ -632,7 +632,7 @@ def prepare_area_definitions(test_dict): trailer = create_test_trailer(is_rapid_scan) expected_area_def = test_dict["expected_area_def"] - with mock.patch("satpy.readers.seviri_l1b_native.np.fromfile") as fromfile, \ + with mock.patch("satpy.readers.seviri_l1b_native.fromfile") as fromfile, \ mock.patch("satpy.readers.seviri_l1b_native.recarray2dict") as recarray2dict, \ mock.patch("satpy.readers.seviri_l1b_native.NativeMSGFileHandler._get_array") as _get_array, \ mock.patch("satpy.readers.seviri_l1b_native.NativeMSGFileHandler._read_trailer"), \ @@ -716,7 +716,7 @@ def prepare_is_roi(test_dict): trailer = create_test_trailer(is_rapid_scan) expected = test_dict["is_roi"] - with mock.patch("satpy.readers.seviri_l1b_native.np.fromfile") as fromfile, \ + with mock.patch("satpy.readers.seviri_l1b_native.fromfile") as fromfile, \ mock.patch("satpy.readers.seviri_l1b_native.recarray2dict") as recarray2dict, \ mock.patch("satpy.readers.seviri_l1b_native.NativeMSGFileHandler._get_array") as _get_array, \ mock.patch("satpy.readers.seviri_l1b_native.NativeMSGFileHandler._read_trailer"), \ @@ -1166,11 +1166,11 @@ def test_header_type(file_content, exp_header_size): ) if file_content == b"foobar": header.pop("15_SECONDARY_PRODUCT_HEADER") - with mock.patch("satpy.readers.seviri_l1b_native.np.fromfile") as fromfile, \ + with mock.patch("satpy.readers.seviri_l1b_native.fromfile") as fromfile, \ mock.patch("satpy.readers.seviri_l1b_native.recarray2dict") as recarray2dict, \ mock.patch("satpy.readers.seviri_l1b_native.NativeMSGFileHandler._get_array") as _get_array, \ mock.patch("satpy.readers.seviri_l1b_native.NativeMSGFileHandler._read_trailer"), \ - mock.patch("builtins.open", mock.mock_open(read_data=file_content)): + mock.patch("satpy.readers.seviri_l1b_native.generic_open", mock.mock_open(read_data=file_content)): fromfile.return_value = header recarray2dict.side_effect = (lambda x: x) _get_array.return_value = np.arange(3) @@ -1196,11 +1196,11 @@ def test_header_warning(): good_qual="NOK" ) - with mock.patch("satpy.readers.seviri_l1b_native.np.fromfile") as fromfile, \ + with mock.patch("satpy.readers.seviri_l1b_native.fromfile") as fromfile, \ mock.patch("satpy.readers.seviri_l1b_native.recarray2dict") as recarray2dict, \ mock.patch("satpy.readers.seviri_l1b_native.NativeMSGFileHandler._get_array") as _get_array, \ mock.patch("satpy.readers.seviri_l1b_native.NativeMSGFileHandler._read_trailer"), \ - mock.patch("builtins.open", mock.mock_open(read_data=ASCII_STARTSWITH)): + mock.patch("satpy.readers.seviri_l1b_native.generic_open", mock.mock_open(read_data=ASCII_STARTSWITH)): recarray2dict.side_effect = (lambda x: x) _get_array.return_value = np.arange(3) @@ -1233,7 +1233,7 @@ def test_header_warning(): ) def test_has_archive_header(starts_with, expected): """Test if the file includes an ASCII archive header.""" - with mock.patch("builtins.open", mock.mock_open(read_data=starts_with)): + with mock.patch("satpy.readers.seviri_l1b_native.generic_open", mock.mock_open(read_data=starts_with)): actual = has_archive_header("filename") assert actual == expected @@ -1248,7 +1248,7 @@ def test_read_header(): dtypes = np.dtype([(k, t) for k, t in zip(keys, types)]) hdr_data = np.array([values], dtype=dtypes) - with mock.patch("satpy.readers.seviri_l1b_native.np.fromfile") as fromfile: + with mock.patch("satpy.readers.seviri_l1b_native.fromfile") as fromfile: fromfile.return_value = hdr_data actual = recarray2dict(hdr_data) assert actual == expected From d01a519c4220df9ec5079a42f38e4e179afb256e Mon Sep 17 00:00:00 2001 From: Pouria Khalaj Date: Sun, 28 Jul 2024 17:46:25 +0200 Subject: [PATCH 07/22] Update `test_seviri_l1b_native` with tests for remote reading This includes generating an actual file on disk and attempt to read it. --- .../reader_tests/test_seviri_l1b_native.py | 122 ++++++++++++++++++ 1 file changed, 122 insertions(+) diff --git a/satpy/tests/reader_tests/test_seviri_l1b_native.py b/satpy/tests/reader_tests/test_seviri_l1b_native.py index 38a4e16e77..4a41162483 100644 --- a/satpy/tests/reader_tests/test_seviri_l1b_native.py +++ b/satpy/tests/reader_tests/test_seviri_l1b_native.py @@ -39,6 +39,8 @@ get_available_channels, has_archive_header, ) +from satpy.readers.seviri_l1b_native_hdr import Msg15NativeHeaderRecord +from satpy.scene import Scene from satpy.tests.reader_tests.test_seviri_base import ORBIT_POLYNOMIALS, ORBIT_POLYNOMIALS_INVALID from satpy.tests.reader_tests.test_seviri_l1b_calibration import TestFileHandlerCalibrationBase from satpy.tests.utils import assert_attrs_equal, make_dataid @@ -1252,3 +1254,123 @@ def test_read_header(): fromfile.return_value = hdr_data actual = recarray2dict(hdr_data) assert actual == expected + + +def generate_seviri_native_null_header(): + """Generates the header of the seviri native format which is filled with zeros, hence, the term null!""" + header_type = Msg15NativeHeaderRecord().get(True) + null_header = np.zeros(header_type.shape, dtype=header_type).reshape(1, ) + return header_type, null_header + + +def scene_from_physical_seviri_nat_file(filename): + """Generates a Scene object from the given seviri native file.""" + return Scene([filename], reader="seviri_l1b_native", reader_kwargs={"fill_disk": True}) + + +def amend_seviri_native_null_header(hdr_null_numpy): + """Amends the given null header so that the ``seviri_l1b_native`` reader can properly parse it. + + This is achieved by setting values for the bare minimum number of header fields so that the reader can make sense of + the given header. This function relies on a number of auxiliary functions all of which are enclosed in the body of + the present function. + + Note: + The naming scheme of the auxiliary functions is as follows: ``_amend_____...``, where + corresponds to keys in the header when it is represented as a dictionary, i.e. when calling ``recarray2dict()`` + on the given header array. + + For example, ``_amend_15_DATA_HEADER__SatelliteStatus__SatelliteDefinition__SatelliteId()`` corresponds to an + auxiliary function which manipulates the following entry: + ``hdr_null_numpy_as_dict["15_DATA_HEADER"]["SatelliteStatus"]["SatelliteDefinition"]["SatelliteId"]`` + """ + + def _amend_15_MAIN_PRODUCT_HEADER(): + hdr_null_numpy[0][0][0] = (b"FormatName : ", b"NATIVE\n") + + def _amend_15_SECONDARY_PRODUCT_HEADER(): + hdr_null_numpy[0][1][9] = (b"SelectedBandIDs", b"XXXXXXXXXXXX") + hdr_null_numpy[0][1][10] = (b"SouthLineSelectedRectangle", b"1") + hdr_null_numpy[0][1][11] = (b"NorthLineSelectedRectangle", b"3712") + hdr_null_numpy[0][1][12] = (b"EastColumnSelectedRectangle", b"1") + hdr_null_numpy[0][1][13] = (b"WestColumnSelectedRectangle", b"3712") + hdr_null_numpy[0][1][14] = (b"NumberLinesVISIR", b"3712") + hdr_null_numpy[0][1][15] = (b"NumberColumnsVISIR", b"3712") + hdr_null_numpy[0][1][16] = (b"NumberLinesHRV", b"11136") + hdr_null_numpy[0][1][17] = (b"NumberColumnsHRV", b"11136") + + def _amend_GP_PK_SH1__PacketTime(): + hdr_null_numpy[0][3][5] = (23158, 27921912) + + def _amend_15_DATA_HEADER__SatelliteStatus__SatelliteDefinition__SatelliteId(): + hdr_null_numpy[0][4][1][0][0] = 324 + + def _amend_15_DATA_HEADER__GeometricProcessing__EarthModel(): + hdr_null_numpy[0][4][6][1] = (2, 6378.169, 6356.5838, 6356.5838) + + def _amend_15_DATA_HEADER__ImageAcquisition__PlannedAcquisitionTime(): + hdr_null_numpy[0][4][2][0] = ( + (23158, 27911177, 286, 223), + (23158, 28663675, 401, 687), + (23158, 28810078, 157, 663) + ) + + # Apply all the header amendments + _amend_15_MAIN_PRODUCT_HEADER() + _amend_15_SECONDARY_PRODUCT_HEADER() + _amend_GP_PK_SH1__PacketTime() + _amend_15_DATA_HEADER__SatelliteStatus__SatelliteDefinition__SatelliteId() + _amend_15_DATA_HEADER__GeometricProcessing__EarthModel() + _amend_15_DATA_HEADER__ImageAcquisition__PlannedAcquisitionTime() + + +@pytest.fixture() +def tmp_seviri_nat_filename(tmp_path): + """Creates a fully-qualified filename for a seviri native format file.""" + tmp_filename = "MSG4-SEVI-MSG15-0100-NA-20210528075743.722000000Z-NA" + return tmp_path / f"{tmp_filename}.nat" + + +def append_data_and_trailer_content_to_seviri_native_header(filename, hdr_null_numpy): + """Generates the data and trailer part (null content) of the file and appends them to the null header. + + The data and trailer are also null and appending them to the header results in a complete seviri nat file. + """ + # size of different parts of the seviri native file in bytes + size = dict(header_with_archive=450400, data=270344960, trailer=380363) + + zero_bytes = bytearray(size["data"] + size["trailer"]) + bytes_data = bytes(zero_bytes) + + hdr_null_numpy.tofile(filename) + with open(filename, "ab") as f: + f.write(bytes_data) + + +@pytest.fixture() +def physical_seviri_native_file(tmp_seviri_nat_filename): + """Creates a physical seviri native file on disk.""" + hdr_null_type, hdr_null = generate_seviri_native_null_header() + amend_seviri_native_null_header(hdr_null) + append_data_and_trailer_content_to_seviri_native_header(tmp_seviri_nat_filename, hdr_null) + + return dict(header_type=hdr_null_type, header=hdr_null, filename=tmp_seviri_nat_filename) + + +def test_read_physical_seviri_nat_file(physical_seviri_native_file): + """Tests that the physical seviri native file has been read successfully. + + Note: + The purpose of this function is not to fully test the properties of the scene. It only provides a test for + reading a physical file from disk. + """ + scene = scene_from_physical_seviri_nat_file(physical_seviri_native_file["filename"]) + + assert physical_seviri_native_file["header_type"] == physical_seviri_native_file["header"].dtype + assert scene.sensor_names == {"seviri"} + assert len(scene.available_dataset_ids()) == 36 + assert set(scene.available_dataset_names()) == set(CHANNEL_INDEX_LIST) + + scene.load(["VIS006"]) + assert scene["VIS006"].shape == (3712, 3712) + assert isinstance(scene["VIS006"], xr.core.dataarray.DataArray) From 0e5f3e78131e4fbdeb3c9403f761aa5b70533867 Mon Sep 17 00:00:00 2001 From: Pouria Khalaj Date: Mon, 29 Jul 2024 10:23:51 +0200 Subject: [PATCH 08/22] Parametrize `test_read_physical_seviri_nat_file` to test zip files as well as plain files --- .../reader_tests/test_seviri_l1b_native.py | 33 ++++++++++++++----- 1 file changed, 24 insertions(+), 9 deletions(-) diff --git a/satpy/tests/reader_tests/test_seviri_l1b_native.py b/satpy/tests/reader_tests/test_seviri_l1b_native.py index 4a41162483..aab1e95568 100644 --- a/satpy/tests/reader_tests/test_seviri_l1b_native.py +++ b/satpy/tests/reader_tests/test_seviri_l1b_native.py @@ -23,12 +23,14 @@ import os import unittest import warnings +import zipfile from unittest import mock import dask.array as da import numpy as np import pytest import xarray as xr +from pytest_lazy_fixtures import lf from satpy.readers.eum_base import recarray2dict, time_cds_short from satpy.readers.seviri_l1b_native import ( @@ -1328,7 +1330,15 @@ def _amend_15_DATA_HEADER__ImageAcquisition__PlannedAcquisitionTime(): def tmp_seviri_nat_filename(tmp_path): """Creates a fully-qualified filename for a seviri native format file.""" tmp_filename = "MSG4-SEVI-MSG15-0100-NA-20210528075743.722000000Z-NA" - return tmp_path / f"{tmp_filename}.nat" + return dict(path=tmp_path, filename=tmp_filename, full_path=tmp_path / f"{tmp_filename}.nat") + + +def compress_seviri_native_file(path, seviri_native_filename): + """Compresses the given seviri native file into a zip file.""" + zip_full_path = path / f"{seviri_native_filename}.zip" + with zipfile.ZipFile(zip_full_path, mode="w") as archive: + archive.write(path / f"{seviri_native_filename}.nat", f"{seviri_native_filename}.nat") + return f"zip://*.nat::{zip_full_path}" def append_data_and_trailer_content_to_seviri_native_header(filename, hdr_null_numpy): @@ -1347,26 +1357,31 @@ def append_data_and_trailer_content_to_seviri_native_header(filename, hdr_null_n f.write(bytes_data) -@pytest.fixture() -def physical_seviri_native_file(tmp_seviri_nat_filename): +def physical_seviri_native_file(seviri_nat_full_file_path): """Creates a physical seviri native file on disk.""" hdr_null_type, hdr_null = generate_seviri_native_null_header() amend_seviri_native_null_header(hdr_null) - append_data_and_trailer_content_to_seviri_native_header(tmp_seviri_nat_filename, hdr_null) + append_data_and_trailer_content_to_seviri_native_header(seviri_nat_full_file_path, hdr_null) - return dict(header_type=hdr_null_type, header=hdr_null, filename=tmp_seviri_nat_filename) + return dict(header_type=hdr_null_type, header=hdr_null) -def test_read_physical_seviri_nat_file(physical_seviri_native_file): - """Tests that the physical seviri native file has been read successfully. +@pytest.mark.parametrize(("treat_native_file", "args"), [ + (lambda path, filename: path / f"{filename}.nat", lf("tmp_seviri_nat_filename")), + (compress_seviri_native_file, lf("tmp_seviri_nat_filename")) +]) +def test_read_physical_seviri_nat_file(tmp_seviri_nat_filename, treat_native_file, args): + """Tests that the physical seviri native file can be read successfully, in case of both a plain and a zip file. Note: The purpose of this function is not to fully test the properties of the scene. It only provides a test for reading a physical file from disk. """ - scene = scene_from_physical_seviri_nat_file(physical_seviri_native_file["filename"]) + native_file = physical_seviri_native_file(tmp_seviri_nat_filename["full_path"]) + full_path = treat_native_file(args["path"], args["filename"]) + scene = scene_from_physical_seviri_nat_file(full_path) - assert physical_seviri_native_file["header_type"] == physical_seviri_native_file["header"].dtype + assert native_file["header_type"] == native_file["header"].dtype assert scene.sensor_names == {"seviri"} assert len(scene.available_dataset_ids()) == 36 assert set(scene.available_dataset_names()) == set(CHANNEL_INDEX_LIST) From 15d17d60db50c27ba5131e9c44dd34b0a8048be5 Mon Sep 17 00:00:00 2001 From: Pouria Khalaj Date: Mon, 29 Jul 2024 10:26:13 +0200 Subject: [PATCH 09/22] Reformat `test_seviri_l1b_native` to make it PEP8 compliant --- satpy/tests/reader_tests/test_seviri_l1b_native.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/satpy/tests/reader_tests/test_seviri_l1b_native.py b/satpy/tests/reader_tests/test_seviri_l1b_native.py index aab1e95568..ecb2b42122 100644 --- a/satpy/tests/reader_tests/test_seviri_l1b_native.py +++ b/satpy/tests/reader_tests/test_seviri_l1b_native.py @@ -1004,12 +1004,12 @@ def test_get_dataset(self, file_handler): def test_time(self, file_handler): """Test start/end nominal/observation time handling.""" assert dt.datetime(2006, 1, 1, 12, 15, 9, 304888) == file_handler.observation_start_time - assert dt.datetime(2006, 1, 1, 12, 15,) == file_handler.start_time + assert dt.datetime(2006, 1, 1, 12, 15, ) == file_handler.start_time assert file_handler.start_time == file_handler.nominal_start_time assert dt.datetime(2006, 1, 1, 12, 27, 9, 304888) == file_handler.observation_end_time assert file_handler.end_time == file_handler.nominal_end_time - assert dt.datetime(2006, 1, 1, 12, 30,) == file_handler.end_time + assert dt.datetime(2006, 1, 1, 12, 30, ) == file_handler.end_time def test_repeat_cycle_duration(self, file_handler): """Test repeat cycle handling for FD or ReduscedScan.""" From df4a6dbd861cd7a7b9d4e4eff94734b9525f86ed Mon Sep 17 00:00:00 2001 From: Pouria Khalaj Date: Mon, 29 Jul 2024 10:42:27 +0200 Subject: [PATCH 10/22] Address @mraspaud comments on PR #2863 --- AUTHORS.md | 1 - .../reader_tests/test_seviri_l1b_native.py | 108 +++++++++--------- 2 files changed, 54 insertions(+), 55 deletions(-) diff --git a/AUTHORS.md b/AUTHORS.md index e764b96627..a8f2cb8e4b 100644 --- a/AUTHORS.md +++ b/AUTHORS.md @@ -95,5 +95,4 @@ The following people have made contributions to this project: - [Will Sharpe (wjsharpe)](https://github.com/wjsharpe) - [Sara Hörnquist (shornqui)](https://github.com/shornqui) - [Antonio Valentino](https://github.com/avalentino) -- [Pouria Khalaj](https://github.com/pkhalaj) - [Clément (ludwigvonkoopa)](https://github.com/ludwigVonKoopa) diff --git a/satpy/tests/reader_tests/test_seviri_l1b_native.py b/satpy/tests/reader_tests/test_seviri_l1b_native.py index ecb2b42122..32b6c4fb32 100644 --- a/satpy/tests/reader_tests/test_seviri_l1b_native.py +++ b/satpy/tests/reader_tests/test_seviri_l1b_native.py @@ -1258,11 +1258,44 @@ def test_read_header(): assert actual == expected -def generate_seviri_native_null_header(): - """Generates the header of the seviri native format which is filled with zeros, hence, the term null!""" - header_type = Msg15NativeHeaderRecord().get(True) - null_header = np.zeros(header_type.shape, dtype=header_type).reshape(1, ) - return header_type, null_header +@pytest.fixture() +def tmp_seviri_nat_filename(tmp_path): + """Creates a fully-qualified filename for a seviri native format file.""" + tmp_filename = "MSG4-SEVI-MSG15-0100-NA-20210528075743.722000000Z-NA" + return dict(path=tmp_path, filename=tmp_filename, full_path=tmp_path / f"{tmp_filename}.nat") + + +def compress_seviri_native_file(path, seviri_native_filename): + """Compresses the given seviri native file into a zip file.""" + zip_full_path = path / f"{seviri_native_filename}.zip" + with zipfile.ZipFile(zip_full_path, mode="w") as archive: + archive.write(path / f"{seviri_native_filename}.nat", f"{seviri_native_filename}.nat") + return f"zip://*.nat::{zip_full_path}" + + +@pytest.mark.parametrize(("treat_native_file", "args"), [ + (lambda path, filename: path / f"{filename}.nat", lf("tmp_seviri_nat_filename")), + (compress_seviri_native_file, lf("tmp_seviri_nat_filename")) +]) +def test_read_physical_seviri_nat_file(tmp_seviri_nat_filename, treat_native_file, args): + """Tests that the physical seviri native file can be read successfully, in case of both a plain and a zip file. + + Note: + The purpose of this function is not to fully test the properties of the scene. It only provides a test for + reading a physical file from disk. + """ + native_file = physical_seviri_native_file(tmp_seviri_nat_filename["full_path"]) + full_path = treat_native_file(args["path"], args["filename"]) + scene = scene_from_physical_seviri_nat_file(full_path) + + assert native_file["header_type"] == native_file["header"].dtype + assert scene.sensor_names == {"seviri"} + assert len(scene.available_dataset_ids()) == 36 + assert set(scene.available_dataset_names()) == set(CHANNEL_INDEX_LIST) + + scene.load(["VIS006"]) + assert scene["VIS006"].shape == (3712, 3712) + assert isinstance(scene["VIS006"], xr.core.dataarray.DataArray) def scene_from_physical_seviri_nat_file(filename): @@ -1270,6 +1303,22 @@ def scene_from_physical_seviri_nat_file(filename): return Scene([filename], reader="seviri_l1b_native", reader_kwargs={"fill_disk": True}) +def physical_seviri_native_file(seviri_nat_full_file_path): + """Creates a physical seviri native file on disk.""" + header_type, header_null = generate_seviri_native_null_header() + amend_seviri_native_null_header(header_null) + append_data_and_trailer_content_to_seviri_native_header(seviri_nat_full_file_path, header_null) + + return dict(header_type=header_type, header=header_null) + + +def generate_seviri_native_null_header(): + """Generates the header of the seviri native format which is filled with zeros, hence, the term null!""" + header_type = Msg15NativeHeaderRecord().get(True) + null_header = np.zeros(header_type.shape, dtype=header_type).reshape(1, ) + return header_type, null_header + + def amend_seviri_native_null_header(hdr_null_numpy): """Amends the given null header so that the ``seviri_l1b_native`` reader can properly parse it. @@ -1326,21 +1375,6 @@ def _amend_15_DATA_HEADER__ImageAcquisition__PlannedAcquisitionTime(): _amend_15_DATA_HEADER__ImageAcquisition__PlannedAcquisitionTime() -@pytest.fixture() -def tmp_seviri_nat_filename(tmp_path): - """Creates a fully-qualified filename for a seviri native format file.""" - tmp_filename = "MSG4-SEVI-MSG15-0100-NA-20210528075743.722000000Z-NA" - return dict(path=tmp_path, filename=tmp_filename, full_path=tmp_path / f"{tmp_filename}.nat") - - -def compress_seviri_native_file(path, seviri_native_filename): - """Compresses the given seviri native file into a zip file.""" - zip_full_path = path / f"{seviri_native_filename}.zip" - with zipfile.ZipFile(zip_full_path, mode="w") as archive: - archive.write(path / f"{seviri_native_filename}.nat", f"{seviri_native_filename}.nat") - return f"zip://*.nat::{zip_full_path}" - - def append_data_and_trailer_content_to_seviri_native_header(filename, hdr_null_numpy): """Generates the data and trailer part (null content) of the file and appends them to the null header. @@ -1355,37 +1389,3 @@ def append_data_and_trailer_content_to_seviri_native_header(filename, hdr_null_n hdr_null_numpy.tofile(filename) with open(filename, "ab") as f: f.write(bytes_data) - - -def physical_seviri_native_file(seviri_nat_full_file_path): - """Creates a physical seviri native file on disk.""" - hdr_null_type, hdr_null = generate_seviri_native_null_header() - amend_seviri_native_null_header(hdr_null) - append_data_and_trailer_content_to_seviri_native_header(seviri_nat_full_file_path, hdr_null) - - return dict(header_type=hdr_null_type, header=hdr_null) - - -@pytest.mark.parametrize(("treat_native_file", "args"), [ - (lambda path, filename: path / f"{filename}.nat", lf("tmp_seviri_nat_filename")), - (compress_seviri_native_file, lf("tmp_seviri_nat_filename")) -]) -def test_read_physical_seviri_nat_file(tmp_seviri_nat_filename, treat_native_file, args): - """Tests that the physical seviri native file can be read successfully, in case of both a plain and a zip file. - - Note: - The purpose of this function is not to fully test the properties of the scene. It only provides a test for - reading a physical file from disk. - """ - native_file = physical_seviri_native_file(tmp_seviri_nat_filename["full_path"]) - full_path = treat_native_file(args["path"], args["filename"]) - scene = scene_from_physical_seviri_nat_file(full_path) - - assert native_file["header_type"] == native_file["header"].dtype - assert scene.sensor_names == {"seviri"} - assert len(scene.available_dataset_ids()) == 36 - assert set(scene.available_dataset_names()) == set(CHANNEL_INDEX_LIST) - - scene.load(["VIS006"]) - assert scene["VIS006"].shape == (3712, 3712) - assert isinstance(scene["VIS006"], xr.core.dataarray.DataArray) From 43b846b25cbaf8bc516df082e8f9e1ff8be5a8dd Mon Sep 17 00:00:00 2001 From: Pouria Khalaj Date: Mon, 29 Jul 2024 11:32:22 +0200 Subject: [PATCH 11/22] Simplify fixtures and parameters of `test_read_physical_seviri_nat_file()` --- .../reader_tests/test_seviri_l1b_native.py | 42 ++++++++++--------- 1 file changed, 23 insertions(+), 19 deletions(-) diff --git a/satpy/tests/reader_tests/test_seviri_l1b_native.py b/satpy/tests/reader_tests/test_seviri_l1b_native.py index 32b6c4fb32..b3117b91d9 100644 --- a/satpy/tests/reader_tests/test_seviri_l1b_native.py +++ b/satpy/tests/reader_tests/test_seviri_l1b_native.py @@ -1258,37 +1258,44 @@ def test_read_header(): assert actual == expected -@pytest.fixture() -def tmp_seviri_nat_filename(tmp_path): +@pytest.fixture(scope="session") +def session_tmp_path(tmp_path_factory): + """Generates a single temp path to use for the entire session.""" + return tmp_path_factory.mktemp("data") + + +@pytest.fixture(scope="session") +def tmp_seviri_nat_filename(session_tmp_path): """Creates a fully-qualified filename for a seviri native format file.""" - tmp_filename = "MSG4-SEVI-MSG15-0100-NA-20210528075743.722000000Z-NA" - return dict(path=tmp_path, filename=tmp_filename, full_path=tmp_path / f"{tmp_filename}.nat") + full_file_path = session_tmp_path / "MSG4-SEVI-MSG15-0100-NA-20210528075743.722000000Z-N.nat" + create_physical_seviri_native_file(full_file_path) + return full_file_path -def compress_seviri_native_file(path, seviri_native_filename): +@pytest.fixture(scope="session") +def compress_seviri_native_file(tmp_seviri_nat_filename, session_tmp_path): """Compresses the given seviri native file into a zip file.""" - zip_full_path = path / f"{seviri_native_filename}.zip" + zip_full_path = session_tmp_path / "test_seviri_native.zip" with zipfile.ZipFile(zip_full_path, mode="w") as archive: - archive.write(path / f"{seviri_native_filename}.nat", f"{seviri_native_filename}.nat") + archive.write(tmp_seviri_nat_filename, os.path.basename(tmp_seviri_nat_filename)) return f"zip://*.nat::{zip_full_path}" -@pytest.mark.parametrize(("treat_native_file", "args"), [ - (lambda path, filename: path / f"{filename}.nat", lf("tmp_seviri_nat_filename")), - (compress_seviri_native_file, lf("tmp_seviri_nat_filename")) +@pytest.mark.slow() +@pytest.mark.order("last") +@pytest.mark.parametrize(("full_path"), [ + lf("tmp_seviri_nat_filename"), + lf("compress_seviri_native_file") ]) -def test_read_physical_seviri_nat_file(tmp_seviri_nat_filename, treat_native_file, args): +def test_read_physical_seviri_nat_file(full_path): """Tests that the physical seviri native file can be read successfully, in case of both a plain and a zip file. Note: The purpose of this function is not to fully test the properties of the scene. It only provides a test for reading a physical file from disk. """ - native_file = physical_seviri_native_file(tmp_seviri_nat_filename["full_path"]) - full_path = treat_native_file(args["path"], args["filename"]) scene = scene_from_physical_seviri_nat_file(full_path) - assert native_file["header_type"] == native_file["header"].dtype assert scene.sensor_names == {"seviri"} assert len(scene.available_dataset_ids()) == 36 assert set(scene.available_dataset_names()) == set(CHANNEL_INDEX_LIST) @@ -1303,17 +1310,15 @@ def scene_from_physical_seviri_nat_file(filename): return Scene([filename], reader="seviri_l1b_native", reader_kwargs={"fill_disk": True}) -def physical_seviri_native_file(seviri_nat_full_file_path): +def create_physical_seviri_native_file(seviri_nat_full_file_path): """Creates a physical seviri native file on disk.""" header_type, header_null = generate_seviri_native_null_header() amend_seviri_native_null_header(header_null) append_data_and_trailer_content_to_seviri_native_header(seviri_nat_full_file_path, header_null) - return dict(header_type=header_type, header=header_null) - def generate_seviri_native_null_header(): - """Generates the header of the seviri native format which is filled with zeros, hence, the term null!""" + """Generates the header of the seviri native format which is filled with zeros, hence the term null!""" header_type = Msg15NativeHeaderRecord().get(True) null_header = np.zeros(header_type.shape, dtype=header_type).reshape(1, ) return header_type, null_header @@ -1382,7 +1387,6 @@ def append_data_and_trailer_content_to_seviri_native_header(filename, hdr_null_n """ # size of different parts of the seviri native file in bytes size = dict(header_with_archive=450400, data=270344960, trailer=380363) - zero_bytes = bytearray(size["data"] + size["trailer"]) bytes_data = bytes(zero_bytes) From 816a2c77fdd1ba5556e98a5c9d4add25ed2c2514 Mon Sep 17 00:00:00 2001 From: Pouria Khalaj Date: Mon, 29 Jul 2024 11:54:22 +0200 Subject: [PATCH 12/22] Use a header which leads to a smaller seviri nat file on disk This concerns `test_read_physical_seviri_nat_file()`. --- .../reader_tests/test_seviri_l1b_native.py | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/satpy/tests/reader_tests/test_seviri_l1b_native.py b/satpy/tests/reader_tests/test_seviri_l1b_native.py index b3117b91d9..d9d54979df 100644 --- a/satpy/tests/reader_tests/test_seviri_l1b_native.py +++ b/satpy/tests/reader_tests/test_seviri_l1b_native.py @@ -1345,15 +1345,15 @@ def _amend_15_MAIN_PRODUCT_HEADER(): hdr_null_numpy[0][0][0] = (b"FormatName : ", b"NATIVE\n") def _amend_15_SECONDARY_PRODUCT_HEADER(): - hdr_null_numpy[0][1][9] = (b"SelectedBandIDs", b"XXXXXXXXXXXX") - hdr_null_numpy[0][1][10] = (b"SouthLineSelectedRectangle", b"1") - hdr_null_numpy[0][1][11] = (b"NorthLineSelectedRectangle", b"3712") - hdr_null_numpy[0][1][12] = (b"EastColumnSelectedRectangle", b"1") - hdr_null_numpy[0][1][13] = (b"WestColumnSelectedRectangle", b"3712") - hdr_null_numpy[0][1][14] = (b"NumberLinesVISIR", b"3712") - hdr_null_numpy[0][1][15] = (b"NumberColumnsVISIR", b"3712") - hdr_null_numpy[0][1][16] = (b"NumberLinesHRV", b"11136") - hdr_null_numpy[0][1][17] = (b"NumberColumnsHRV", b"11136") + hdr_null_numpy[0][1][9] = (b"SelectedBandIDs", b"XXXXXXXXXXX-") + hdr_null_numpy[0][1][10] = (b"SouthLineSelectedRectangle", b"3360") + hdr_null_numpy[0][1][11] = (b"NorthLineSelectedRectangle", b"3373") + hdr_null_numpy[0][1][12] = (b"EastColumnSelectedRectangle", b"1714") + hdr_null_numpy[0][1][13] = (b"WestColumnSelectedRectangle", b"1729") + hdr_null_numpy[0][1][14] = (b"NumberLinesVISIR", b"14") + hdr_null_numpy[0][1][15] = (b"NumberColumnsVISIR", b"16") + hdr_null_numpy[0][1][16] = (b"NumberLinesHRV", b"42") + hdr_null_numpy[0][1][17] = (b"NumberColumnsHRV", b"48") def _amend_GP_PK_SH1__PacketTime(): hdr_null_numpy[0][3][5] = (23158, 27921912) @@ -1386,7 +1386,7 @@ def append_data_and_trailer_content_to_seviri_native_header(filename, hdr_null_n The data and trailer are also null and appending them to the header results in a complete seviri nat file. """ # size of different parts of the seviri native file in bytes - size = dict(header_with_archive=450400, data=270344960, trailer=380363) + size = {"header_with_archive": 450400, "data": 13090, "trailer": 380363} zero_bytes = bytearray(size["data"] + size["trailer"]) bytes_data = bytes(zero_bytes) From d4976c32e6336b9b0b84ca07e246489a5e5c001c Mon Sep 17 00:00:00 2001 From: Pouria Khalaj Date: Mon, 29 Jul 2024 13:11:07 +0200 Subject: [PATCH 13/22] Make the returned path posix compliant in `compress_seviri_native_file` This is to ensure that the remote reading tests can also run on Windows. Note: `fsspec` expects a POSIX path. --- satpy/tests/reader_tests/test_seviri_l1b_native.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/satpy/tests/reader_tests/test_seviri_l1b_native.py b/satpy/tests/reader_tests/test_seviri_l1b_native.py index d9d54979df..04198f27e3 100644 --- a/satpy/tests/reader_tests/test_seviri_l1b_native.py +++ b/satpy/tests/reader_tests/test_seviri_l1b_native.py @@ -1278,7 +1278,7 @@ def compress_seviri_native_file(tmp_seviri_nat_filename, session_tmp_path): zip_full_path = session_tmp_path / "test_seviri_native.zip" with zipfile.ZipFile(zip_full_path, mode="w") as archive: archive.write(tmp_seviri_nat_filename, os.path.basename(tmp_seviri_nat_filename)) - return f"zip://*.nat::{zip_full_path}" + return f"zip://*.nat::file://{zip_full_path.as_posix()}" @pytest.mark.slow() From db3ddb856747fcc0122c55dbdd7de7da07fd188f Mon Sep 17 00:00:00 2001 From: Pouria Khalaj Date: Mon, 29 Jul 2024 13:18:31 +0200 Subject: [PATCH 14/22] Resolve warnings raised as a result of `slow` & `order` being pytest unknown marks We do not need these marks anymore, as we decreased the size of the generated seviri native file. As a result, the tests now run fast enough. --- satpy/tests/reader_tests/test_seviri_l1b_native.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/satpy/tests/reader_tests/test_seviri_l1b_native.py b/satpy/tests/reader_tests/test_seviri_l1b_native.py index 04198f27e3..1c5faf5736 100644 --- a/satpy/tests/reader_tests/test_seviri_l1b_native.py +++ b/satpy/tests/reader_tests/test_seviri_l1b_native.py @@ -1281,8 +1281,6 @@ def compress_seviri_native_file(tmp_seviri_nat_filename, session_tmp_path): return f"zip://*.nat::file://{zip_full_path.as_posix()}" -@pytest.mark.slow() -@pytest.mark.order("last") @pytest.mark.parametrize(("full_path"), [ lf("tmp_seviri_nat_filename"), lf("compress_seviri_native_file") From fbd02b5b743256ec72788b2c8919634cbcecabb2 Mon Sep 17 00:00:00 2001 From: Pouria Khalaj Date: Mon, 29 Jul 2024 13:28:28 +0200 Subject: [PATCH 15/22] Resolve warnings raised as a failure in the orbit polynomial This warning is totally benign. It is caused as a result of the seviri native file that we create which is essentially filled with zeros. --- satpy/tests/reader_tests/test_seviri_l1b_native.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/satpy/tests/reader_tests/test_seviri_l1b_native.py b/satpy/tests/reader_tests/test_seviri_l1b_native.py index 1c5faf5736..8ba8fcb0c6 100644 --- a/satpy/tests/reader_tests/test_seviri_l1b_native.py +++ b/satpy/tests/reader_tests/test_seviri_l1b_native.py @@ -1298,9 +1298,11 @@ def test_read_physical_seviri_nat_file(full_path): assert len(scene.available_dataset_ids()) == 36 assert set(scene.available_dataset_names()) == set(CHANNEL_INDEX_LIST) - scene.load(["VIS006"]) - assert scene["VIS006"].shape == (3712, 3712) - assert isinstance(scene["VIS006"], xr.core.dataarray.DataArray) + with warnings.catch_warnings(): + warnings.filterwarnings("ignore", category=UserWarning) + scene.load(["VIS006"]) + assert scene["VIS006"].shape == (3712, 3712) + assert isinstance(scene["VIS006"], xr.core.dataarray.DataArray) def scene_from_physical_seviri_nat_file(filename): From f002b00cd3c83ed25808ba5cebe02cf8d21a1647 Mon Sep 17 00:00:00 2001 From: Pouria Khalaj Date: Mon, 29 Jul 2024 13:57:27 +0200 Subject: [PATCH 16/22] Fix the issue with the docstring in `amend_seviri_native_null_header()` The issue was an unexpected indentation on the last line of the docstring! --- satpy/tests/reader_tests/test_seviri_l1b_native.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/satpy/tests/reader_tests/test_seviri_l1b_native.py b/satpy/tests/reader_tests/test_seviri_l1b_native.py index 8ba8fcb0c6..e88ff059b3 100644 --- a/satpy/tests/reader_tests/test_seviri_l1b_native.py +++ b/satpy/tests/reader_tests/test_seviri_l1b_native.py @@ -1338,7 +1338,7 @@ def amend_seviri_native_null_header(hdr_null_numpy): For example, ``_amend_15_DATA_HEADER__SatelliteStatus__SatelliteDefinition__SatelliteId()`` corresponds to an auxiliary function which manipulates the following entry: - ``hdr_null_numpy_as_dict["15_DATA_HEADER"]["SatelliteStatus"]["SatelliteDefinition"]["SatelliteId"]`` + ``hdr_null_numpy_as_dict["15_DATA_HEADER"]["SatelliteStatus"]["SatelliteDefinition"]["SatelliteId"]`` """ def _amend_15_MAIN_PRODUCT_HEADER(): From a7596d3e6bd20e1672e18d87d73a68ff25abbaf0 Mon Sep 17 00:00:00 2001 From: Pouria Khalaj Date: Mon, 29 Jul 2024 15:16:08 +0200 Subject: [PATCH 17/22] Update `fsspec` support column for `seviri_l1b_native` reader in the readers table --- satpy/etc/readers/seviri_l1b_native.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/satpy/etc/readers/seviri_l1b_native.yaml b/satpy/etc/readers/seviri_l1b_native.yaml index 8fbecd4e59..4aea36c174 100644 --- a/satpy/etc/readers/seviri_l1b_native.yaml +++ b/satpy/etc/readers/seviri_l1b_native.yaml @@ -5,7 +5,7 @@ reader: description: > Reader for EUMETSAT MSG SEVIRI Level 1b native format files. status: Nominal - supports_fsspec: false + supports_fsspec: true sensors: [seviri] default_channels: [HRV, IR_016, IR_039, IR_087, IR_097, IR_108, IR_120, IR_134, VIS006, VIS008, WV_062, WV_073] reader: !!python/name:satpy.readers.yaml_reader.GEOFlippableFileYAMLReader From 80f3925818be3b6972c5c3652b42f81c68425ef9 Mon Sep 17 00:00:00 2001 From: Pouria Khalaj Date: Tue, 30 Jul 2024 15:53:50 +0200 Subject: [PATCH 18/22] Use dask `map_blocks()` in `_get_array()` This includes: - Extracting the `_get_array()` method so that it is now a function in the module and not a class method. - Introduction of `NativeMSGFileHandler_make_dask_array_with_map_blocks()` method to utilize the dask `map_blocks()`. - Introduction of a new method, namely `NativeMSGFileHandler._number_of_visir_channels` to facilitate testing and mock patching. - Adapting the mock patches in tests accordingly. --- satpy/readers/seviri_l1b_native.py | 42 +++++++++++++++---- .../reader_tests/test_seviri_l1b_native.py | 21 ++++++++-- 2 files changed, 50 insertions(+), 13 deletions(-) diff --git a/satpy/readers/seviri_l1b_native.py b/satpy/readers/seviri_l1b_native.py index d5f3ba2692..b09a947e1c 100644 --- a/satpy/readers/seviri_l1b_native.py +++ b/satpy/readers/seviri_l1b_native.py @@ -193,10 +193,27 @@ def __init__(self, filename, filename_info, filetype_info, # Available channels are known only after the header has been read self.header_type = get_native_header(has_archive_header(self.filename)) self._read_header() - self.dask_array = da.from_array(self._get_array(), chunks=(CHUNK_SIZE,)) + self._make_dask_array_with_map_blocks() self._read_trailer() self.image_boundaries = ImageBoundaries(self.header, self.trailer, self.mda) + def _make_dask_array_with_map_blocks(self): + """Makes the dask array using the ``da.map_blocks()`` functionality.""" + dtype = self._get_data_dtype() + chunks = da.core.normalize_chunks( + "auto", + shape=(self.mda["number_of_lines"],), + dtype=dtype) + self.dask_array = da.map_blocks( + _get_array, + dtype=dtype, + chunks=chunks, + meta=np.zeros(1, dtype=dtype), + # The following will be passed as keyword arguments to the `_get_array()` function. + filename=self.filename, + hdr_size=self.header_type.itemsize + ) + @property def _repeat_cycle_duration(self): """Get repeat cycle duration from the trailer.""" @@ -266,9 +283,7 @@ def get_lrec(cols): # each pixel is 10-bits -> one line of data has 25% more bytes # than the number of columns suggest (10/8 = 1.25) visir_rec = get_lrec(int(self.mda["number_of_columns"] * 1.25)) - number_of_visir_channels = len( - [s for s in self.mda["channel_list"] if not s == "HRV"]) - drec = [("visir", (visir_rec, number_of_visir_channels))] + drec = [("visir", (visir_rec, self._number_of_visir_channels()))] if self.mda["available_channels"]["HRV"]: hrv_rec = get_lrec(int(self.mda["hrv_number_of_columns"] * 1.25)) @@ -276,11 +291,9 @@ def get_lrec(cols): return np.dtype(drec) - def _get_array(self): - """Get the numpy array for the SEVIRI data.""" - data_dtype = self._get_data_dtype() - hdr_size = self.header_type.itemsize - return fromfile(self.filename, dtype=data_dtype, offset=hdr_size, count=self.mda["number_of_lines"]) + def _number_of_visir_channels(self): + """Returns the number of visir channels, i.e. all channels excluding ``HRV``.""" + return len([s for s in self.mda["channel_list"] if not s == "HRV"]) def _read_header(self): """Read the header info.""" @@ -891,3 +904,14 @@ def read_header(filename): dtype = get_native_header(has_archive_header(filename)) hdr = fromfile(filename, dtype=dtype, count=1) return recarray2dict(hdr) + + +def _get_array(filename=None, hdr_size=None, block_info=None): + """Get the numpy array for the SEVIRI data.""" + output_block_info = block_info[None] + data_dtype = output_block_info["dtype"] + return fromfile( + filename, + dtype=data_dtype, + offset=hdr_size + output_block_info["array-location"][0][0] * data_dtype.itemsize, + count=output_block_info["chunk-shape"][0]) diff --git a/satpy/tests/reader_tests/test_seviri_l1b_native.py b/satpy/tests/reader_tests/test_seviri_l1b_native.py index e88ff059b3..4c09ca0381 100644 --- a/satpy/tests/reader_tests/test_seviri_l1b_native.py +++ b/satpy/tests/reader_tests/test_seviri_l1b_native.py @@ -638,7 +638,9 @@ def prepare_area_definitions(test_dict): with mock.patch("satpy.readers.seviri_l1b_native.fromfile") as fromfile, \ mock.patch("satpy.readers.seviri_l1b_native.recarray2dict") as recarray2dict, \ - mock.patch("satpy.readers.seviri_l1b_native.NativeMSGFileHandler._get_array") as _get_array, \ + mock.patch("satpy.readers.seviri_l1b_native._get_array") as _get_array, \ + mock.patch( + "satpy.readers.seviri_l1b_native.NativeMSGFileHandler._number_of_visir_channels") as _n_visir_ch, \ mock.patch("satpy.readers.seviri_l1b_native.NativeMSGFileHandler._read_trailer"), \ mock.patch( "satpy.readers.seviri_l1b_native.has_archive_header" @@ -647,6 +649,7 @@ def prepare_area_definitions(test_dict): fromfile.return_value = header recarray2dict.side_effect = (lambda x: x) _get_array.return_value = np.arange(3) + _n_visir_ch.return_value = 11 fh = NativeMSGFileHandler(filename=None, filename_info={}, filetype_info=None) fh.fill_disk = fill_disk fh.header = header @@ -722,7 +725,9 @@ def prepare_is_roi(test_dict): with mock.patch("satpy.readers.seviri_l1b_native.fromfile") as fromfile, \ mock.patch("satpy.readers.seviri_l1b_native.recarray2dict") as recarray2dict, \ - mock.patch("satpy.readers.seviri_l1b_native.NativeMSGFileHandler._get_array") as _get_array, \ + mock.patch("satpy.readers.seviri_l1b_native._get_array") as _get_array, \ + mock.patch( + "satpy.readers.seviri_l1b_native.NativeMSGFileHandler._number_of_visir_channels") as _n_visir_ch, \ mock.patch("satpy.readers.seviri_l1b_native.NativeMSGFileHandler._read_trailer"), \ mock.patch( "satpy.readers.seviri_l1b_native.has_archive_header" @@ -731,6 +736,7 @@ def prepare_is_roi(test_dict): fromfile.return_value = header recarray2dict.side_effect = (lambda x: x) _get_array.return_value = np.arange(3) + _n_visir_ch.return_value = 11 fh = NativeMSGFileHandler(filename=None, filename_info={}, filetype_info=None) fh.header = header fh.trailer = trailer @@ -1172,12 +1178,15 @@ def test_header_type(file_content, exp_header_size): header.pop("15_SECONDARY_PRODUCT_HEADER") with mock.patch("satpy.readers.seviri_l1b_native.fromfile") as fromfile, \ mock.patch("satpy.readers.seviri_l1b_native.recarray2dict") as recarray2dict, \ - mock.patch("satpy.readers.seviri_l1b_native.NativeMSGFileHandler._get_array") as _get_array, \ + mock.patch("satpy.readers.seviri_l1b_native._get_array") as _get_array, \ + mock.patch( + "satpy.readers.seviri_l1b_native.NativeMSGFileHandler._number_of_visir_channels") as _n_visir_ch, \ mock.patch("satpy.readers.seviri_l1b_native.NativeMSGFileHandler._read_trailer"), \ mock.patch("satpy.readers.seviri_l1b_native.generic_open", mock.mock_open(read_data=file_content)): fromfile.return_value = header recarray2dict.side_effect = (lambda x: x) _get_array.return_value = np.arange(3) + _n_visir_ch.return_value = 11 fh = NativeMSGFileHandler(filename=None, filename_info={}, filetype_info=None) assert fh.header_type.itemsize == exp_header_size assert "15_SECONDARY_PRODUCT_HEADER" in fh.header @@ -1202,7 +1211,9 @@ def test_header_warning(): with mock.patch("satpy.readers.seviri_l1b_native.fromfile") as fromfile, \ mock.patch("satpy.readers.seviri_l1b_native.recarray2dict") as recarray2dict, \ - mock.patch("satpy.readers.seviri_l1b_native.NativeMSGFileHandler._get_array") as _get_array, \ + mock.patch("satpy.readers.seviri_l1b_native._get_array") as _get_array, \ + mock.patch( + "satpy.readers.seviri_l1b_native.NativeMSGFileHandler._number_of_visir_channels") as _n_visir_ch, \ mock.patch("satpy.readers.seviri_l1b_native.NativeMSGFileHandler._read_trailer"), \ mock.patch("satpy.readers.seviri_l1b_native.generic_open", mock.mock_open(read_data=ASCII_STARTSWITH)): recarray2dict.side_effect = (lambda x: x) @@ -1211,6 +1222,8 @@ def test_header_warning(): exp_warning = "The quality flag for this file indicates not OK. Use this data with caution!" fromfile.return_value = header_good + _n_visir_ch.return_value = 11 + with warnings.catch_warnings(): warnings.simplefilter("error") NativeMSGFileHandler(filename=None, filename_info={}, filetype_info=None) From 42dfc051b7aa4382748681a1908cbc8cd3ffc209 Mon Sep 17 00:00:00 2001 From: Pouria Khalaj Date: Tue, 13 Aug 2024 10:49:07 +0200 Subject: [PATCH 19/22] Replace `np.zeros` with `np.empty` in the meta parameter of the dask array --- satpy/readers/seviri_l1b_native.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/satpy/readers/seviri_l1b_native.py b/satpy/readers/seviri_l1b_native.py index b09a947e1c..f1bf5372a7 100644 --- a/satpy/readers/seviri_l1b_native.py +++ b/satpy/readers/seviri_l1b_native.py @@ -208,7 +208,7 @@ def _make_dask_array_with_map_blocks(self): _get_array, dtype=dtype, chunks=chunks, - meta=np.zeros(1, dtype=dtype), + meta=np.empty(1, dtype=dtype), # The following will be passed as keyword arguments to the `_get_array()` function. filename=self.filename, hdr_size=self.header_type.itemsize From 1c00de7296ec2a354f4291757ec16c5ba25a46c4 Mon Sep 17 00:00:00 2001 From: Pouria Khalaj Date: Tue, 13 Aug 2024 10:50:11 +0200 Subject: [PATCH 20/22] Making `dask_array` private by prefixing it with `_` --- satpy/readers/seviri_l1b_native.py | 14 +++++++------- satpy/tests/reader_tests/test_seviri_l1b_native.py | 2 +- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/satpy/readers/seviri_l1b_native.py b/satpy/readers/seviri_l1b_native.py index f1bf5372a7..2cda9a613a 100644 --- a/satpy/readers/seviri_l1b_native.py +++ b/satpy/readers/seviri_l1b_native.py @@ -204,7 +204,7 @@ def _make_dask_array_with_map_blocks(self): "auto", shape=(self.mda["number_of_lines"],), dtype=dtype) - self.dask_array = da.map_blocks( + self._dask_array = da.map_blocks( _get_array, dtype=dtype, chunks=chunks, @@ -594,10 +594,10 @@ def _get_visir_channel(self, dataset_id): # Check if there is only 1 channel in the list as a change # is needed in the array assignment ie channel id is not present if len(self.mda["channel_list"]) == 1: - raw = self.dask_array["visir"]["line_data"] + raw = self._dask_array["visir"]["line_data"] else: i = self.mda["channel_list"].index(dataset_id["name"]) - raw = self.dask_array["visir"]["line_data"][:, i, :] + raw = self._dask_array["visir"]["line_data"][:, i, :] data = dec10216(raw.flatten()) data = data.reshape(shape) return data @@ -608,7 +608,7 @@ def _get_hrv_channel(self): data_list = [] for i in range(3): - raw = self.dask_array["hrv"]["line_data"][:, i, :] + raw = self._dask_array["hrv"]["line_data"][:, i, :] data = dec10216(raw.flatten()) data = data.reshape(shape_layer) data_list.append(data) @@ -667,7 +667,7 @@ def _add_scanline_acq_time(self, dataset, dataset_id): def _get_acq_time_hrv(self): """Get raw acquisition time for HRV channel.""" - tline = self.dask_array["hrv"]["acq_time"] + tline = self._dask_array["hrv"]["acq_time"] tline0 = tline[:, 0] tline1 = tline[:, 1] tline2 = tline[:, 2] @@ -679,9 +679,9 @@ def _get_acq_time_visir(self, dataset_id): # Check if there is only 1 channel in the list as a change # is needed in the array assignment, i.e. channel id is not present if len(self.mda["channel_list"]) == 1: - return self.dask_array["visir"]["acq_time"].compute() + return self._dask_array["visir"]["acq_time"].compute() i = self.mda["channel_list"].index(dataset_id["name"]) - return self.dask_array["visir"]["acq_time"][:, i].compute() + return self._dask_array["visir"]["acq_time"][:, i].compute() def _update_attrs(self, dataset, dataset_info): """Update dataset attributes.""" diff --git a/satpy/tests/reader_tests/test_seviri_l1b_native.py b/satpy/tests/reader_tests/test_seviri_l1b_native.py index 4c09ca0381..10fb8ef726 100644 --- a/satpy/tests/reader_tests/test_seviri_l1b_native.py +++ b/satpy/tests/reader_tests/test_seviri_l1b_native.py @@ -928,7 +928,7 @@ def file_handler(self): fh.header = header fh.trailer = trailer fh.mda = mda - fh.dask_array = da.from_array(data) + fh._dask_array = da.from_array(data) fh.platform_id = 324 fh.fill_disk = False fh.calib_mode = "NOMINAL" From b58823c90e74dab21fe6a76e25f1e87918b58f4c Mon Sep 17 00:00:00 2001 From: Pouria Khalaj Date: Tue, 13 Aug 2024 11:01:00 +0200 Subject: [PATCH 21/22] Replace `np.empty` with `np.array` for performance (less memory footprint?) --- satpy/readers/seviri_l1b_native.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/satpy/readers/seviri_l1b_native.py b/satpy/readers/seviri_l1b_native.py index 2cda9a613a..340cba2028 100644 --- a/satpy/readers/seviri_l1b_native.py +++ b/satpy/readers/seviri_l1b_native.py @@ -208,7 +208,7 @@ def _make_dask_array_with_map_blocks(self): _get_array, dtype=dtype, chunks=chunks, - meta=np.empty(1, dtype=dtype), + meta=np.array([], dtype=dtype), # The following will be passed as keyword arguments to the `_get_array()` function. filename=self.filename, hdr_size=self.header_type.itemsize From d22d2b6f17c850ce45f399d828435520530b4ff2 Mon Sep 17 00:00:00 2001 From: Pouria Khalaj Date: Wed, 14 Aug 2024 08:59:21 +0200 Subject: [PATCH 22/22] Change the first line of docstrings to imperative mood. --- satpy/readers/seviri_l1b_native.py | 4 ++-- satpy/readers/utils.py | 2 +- .../reader_tests/test_seviri_l1b_native.py | 18 +++++++++--------- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/satpy/readers/seviri_l1b_native.py b/satpy/readers/seviri_l1b_native.py index 340cba2028..3eaa9b4dfd 100644 --- a/satpy/readers/seviri_l1b_native.py +++ b/satpy/readers/seviri_l1b_native.py @@ -198,7 +198,7 @@ def __init__(self, filename, filename_info, filetype_info, self.image_boundaries = ImageBoundaries(self.header, self.trailer, self.mda) def _make_dask_array_with_map_blocks(self): - """Makes the dask array using the ``da.map_blocks()`` functionality.""" + """Make the dask array using the ``da.map_blocks()`` functionality.""" dtype = self._get_data_dtype() chunks = da.core.normalize_chunks( "auto", @@ -292,7 +292,7 @@ def get_lrec(cols): return np.dtype(drec) def _number_of_visir_channels(self): - """Returns the number of visir channels, i.e. all channels excluding ``HRV``.""" + """Return the number of visir channels, i.e. all channels excluding ``HRV``.""" return len([s for s in self.mda["channel_list"] if not s == "HRV"]) def _read_header(self): diff --git a/satpy/readers/utils.py b/satpy/readers/utils.py index e45e78aef1..983225acd5 100644 --- a/satpy/readers/utils.py +++ b/satpy/readers/utils.py @@ -362,7 +362,7 @@ def generic_open(filename, *args, **kwargs): def fromfile(filename, dtype, count=1, offset=0): - """Reads the numpy array from a (remote or local) file using a buffer. + """Read the numpy array from a (remote or local) file using a buffer. Note: This function relies on the :func:`generic_open` context manager to read a file remotely. diff --git a/satpy/tests/reader_tests/test_seviri_l1b_native.py b/satpy/tests/reader_tests/test_seviri_l1b_native.py index 10fb8ef726..eab987d41f 100644 --- a/satpy/tests/reader_tests/test_seviri_l1b_native.py +++ b/satpy/tests/reader_tests/test_seviri_l1b_native.py @@ -1273,13 +1273,13 @@ def test_read_header(): @pytest.fixture(scope="session") def session_tmp_path(tmp_path_factory): - """Generates a single temp path to use for the entire session.""" + """Generate a single temp path to use for the entire session.""" return tmp_path_factory.mktemp("data") @pytest.fixture(scope="session") def tmp_seviri_nat_filename(session_tmp_path): - """Creates a fully-qualified filename for a seviri native format file.""" + """Create a fully-qualified filename for a seviri native format file.""" full_file_path = session_tmp_path / "MSG4-SEVI-MSG15-0100-NA-20210528075743.722000000Z-N.nat" create_physical_seviri_native_file(full_file_path) return full_file_path @@ -1287,7 +1287,7 @@ def tmp_seviri_nat_filename(session_tmp_path): @pytest.fixture(scope="session") def compress_seviri_native_file(tmp_seviri_nat_filename, session_tmp_path): - """Compresses the given seviri native file into a zip file.""" + """Compress the given seviri native file into a zip file.""" zip_full_path = session_tmp_path / "test_seviri_native.zip" with zipfile.ZipFile(zip_full_path, mode="w") as archive: archive.write(tmp_seviri_nat_filename, os.path.basename(tmp_seviri_nat_filename)) @@ -1299,7 +1299,7 @@ def compress_seviri_native_file(tmp_seviri_nat_filename, session_tmp_path): lf("compress_seviri_native_file") ]) def test_read_physical_seviri_nat_file(full_path): - """Tests that the physical seviri native file can be read successfully, in case of both a plain and a zip file. + """Test that the physical seviri native file can be read successfully, in case of both a plain and a zip file. Note: The purpose of this function is not to fully test the properties of the scene. It only provides a test for @@ -1319,26 +1319,26 @@ def test_read_physical_seviri_nat_file(full_path): def scene_from_physical_seviri_nat_file(filename): - """Generates a Scene object from the given seviri native file.""" + """Generate a Scene object from the given seviri native file.""" return Scene([filename], reader="seviri_l1b_native", reader_kwargs={"fill_disk": True}) def create_physical_seviri_native_file(seviri_nat_full_file_path): - """Creates a physical seviri native file on disk.""" + """Create a physical seviri native file on disk.""" header_type, header_null = generate_seviri_native_null_header() amend_seviri_native_null_header(header_null) append_data_and_trailer_content_to_seviri_native_header(seviri_nat_full_file_path, header_null) def generate_seviri_native_null_header(): - """Generates the header of the seviri native format which is filled with zeros, hence the term null!""" + """Generate the header of the seviri native format which is filled with zeros, hence the term null!""" header_type = Msg15NativeHeaderRecord().get(True) null_header = np.zeros(header_type.shape, dtype=header_type).reshape(1, ) return header_type, null_header def amend_seviri_native_null_header(hdr_null_numpy): - """Amends the given null header so that the ``seviri_l1b_native`` reader can properly parse it. + """Amend the given null header so that the ``seviri_l1b_native`` reader can properly parse it. This is achieved by setting values for the bare minimum number of header fields so that the reader can make sense of the given header. This function relies on a number of auxiliary functions all of which are enclosed in the body of @@ -1394,7 +1394,7 @@ def _amend_15_DATA_HEADER__ImageAcquisition__PlannedAcquisitionTime(): def append_data_and_trailer_content_to_seviri_native_header(filename, hdr_null_numpy): - """Generates the data and trailer part (null content) of the file and appends them to the null header. + """Generate the data and trailer part (null content) of the file and appends them to the null header. The data and trailer are also null and appending them to the header results in a complete seviri nat file. """