From 1ed885e9ffb167d4e5b9b5908d3f763703366f2d Mon Sep 17 00:00:00 2001 From: Patrick Peglar Date: Wed, 21 Apr 2021 12:38:41 +0100 Subject: [PATCH] Topology tolerance [AVD-1723] (#4099) * Deduce correct topology-dimension from connectivities, and be tolerant on loading. * Added simple integration tests. * Review changes. * Review changes: use 'logging' module in place of 'warnings'. * Replace deprecated assertion method; check logged message levels. * Simplify logging tests; tidy synthetic file creation. --- lib/iris/experimental/ugrid/__init__.py | 34 +++++++- .../experimental/test_ugrid_load.py | 77 ++++++++++++++++++- .../file_headers/minimal_bad_topology_dim.cdl | 38 +++++++++ 3 files changed, 147 insertions(+), 2 deletions(-) create mode 100644 lib/iris/tests/stock/file_headers/minimal_bad_topology_dim.cdl diff --git a/lib/iris/experimental/ugrid/__init__.py b/lib/iris/experimental/ugrid/__init__.py index b3eb65d18d..b0f0d8c533 100644 --- a/lib/iris/experimental/ugrid/__init__.py +++ b/lib/iris/experimental/ugrid/__init__.py @@ -3656,7 +3656,39 @@ def _build_mesh(cf, mesh_var, file_path): attributes = {} attr_units = get_attr_units(mesh_var, attributes) - topology_dimension = mesh_var.topology_dimension + if hasattr(mesh_var, "volume_node_connectivity"): + topology_dimension = 3 + elif hasattr(mesh_var, "face_node_connectivity"): + topology_dimension = 2 + elif hasattr(mesh_var, "edge_node_connectivity"): + topology_dimension = 1 + else: + # Nodes only. We aren't sure yet whether this is a valid option. + topology_dimension = 0 + + if not hasattr(mesh_var, "topology_dimension"): + msg = ( + f"Mesh variable {mesh_var.cf_name} has no 'topology_dimension'" + f" : *Assuming* topology_dimension={topology_dimension}" + ", consistent with the attached connectivities." + ) + # TODO: reconsider logging level when we have consistent practice. + # TODO: logger always requires extras['cls'] : can we fix this? + logger.warning(msg, extra=dict(cls=None)) + else: + quoted_topology_dimension = mesh_var.topology_dimension + if quoted_topology_dimension != topology_dimension: + msg = ( + f"*Assuming* 'topology_dimension'={topology_dimension}" + f", from the attached connectivities of the mesh variable " + f"{mesh_var.cf_name}. However, " + f"{mesh_var.cf_name}:topology_dimension = " + f"{quoted_topology_dimension}" + " -- ignoring this as it is inconsistent." + ) + # TODO: reconsider logging level when we have consistent practice. + # TODO: logger always requires extras['cls'] : can we fix this? + logger.warning(msg=msg, extra=dict(cls=None)) node_dimension = None edge_dimension = getattr(mesh_var, "edge_dimension", None) diff --git a/lib/iris/tests/integration/experimental/test_ugrid_load.py b/lib/iris/tests/integration/experimental/test_ugrid_load.py index e2dbfe8edc..aa73103130 100644 --- a/lib/iris/tests/integration/experimental/test_ugrid_load.py +++ b/lib/iris/tests/integration/experimental/test_ugrid_load.py @@ -18,7 +18,12 @@ from collections.abc import Iterable from iris import Constraint, load -from iris.experimental.ugrid import PARSE_UGRID_ON_LOAD +from iris.experimental.ugrid import PARSE_UGRID_ON_LOAD, logger + +from iris.tests.stock.netcdf import ( + _file_from_cdl_template as create_file_from_cdl_template, +) +from iris.tests.unit.tests.stock.test_netcdf import XIOSFileMixin def ugrid_load(uris, constraints=None, callback=None): @@ -117,3 +122,73 @@ def test_multiple_phenomena(self): self.assertCML( cube_list, ("experimental", "ugrid", "surface_mean.cml") ) + + +class TestTolerantLoading(XIOSFileMixin): + # N.B. using parts of the XIOS-like file integration testing, to make + # temporary netcdf files from stored CDL templates. + @classmethod + def setUpClass(cls): + super().setUpClass() # create cls.temp_dir = dir for test files + + @classmethod + def tearDownClass(cls): + super().setUpClass() # destroy temp dir + + # Create a testfile according to testcase-specific arguments. + # NOTE: with this, parent "create_synthetic_test_cube" can load a cube. + def create_synthetic_file(self, **create_kwargs): + template_name = create_kwargs["template"] # required kwarg + testfile_name = "tmp_netcdf" + template_subs = dict( + NUM_NODES=7, NUM_FACES=3, DATASET_NAME=testfile_name + ) + kwarg_subs = create_kwargs.get("subs", {}) # optional kwarg + template_subs.update(kwarg_subs) + filepath = create_file_from_cdl_template( + temp_file_dir=self.temp_dir, + dataset_name=testfile_name, + dataset_type=template_name, + template_subs=template_subs, + ) + return str(filepath) # N.B. Path object not usable in iris.load + + def test_mesh_bad_topology_dimension(self): + # Check that the load generates a suitable warning. + with self.assertLogs(logger) as log: + template = "minimal_bad_topology_dim" + dim_line = "mesh_var:topology_dimension = 1 ;" # which is wrong ! + cube = self.create_synthetic_test_cube( + template=template, subs=dict(TOPOLOGY_DIM_DEFINITION=dim_line) + ) + # Check we got just one message, and its content + self.assertEqual(len(log.records), 1) + rec = log.records[0] + self.assertEqual(rec.levelname, "WARNING") + re_msg = r"topology_dimension.* ignoring" + self.assertRegex(rec.msg, re_msg) + + # Check that the result has topology-dimension of 2 (not 1). + self.assertEqual(cube.mesh.topology_dimension, 2) + + def test_mesh_no_topology_dimension(self): + # Check that the load generates a suitable warning. + with self.assertLogs(logger) as log: + template = "minimal_bad_topology_dim" + dim_line = "" # don't create ANY topology_dimension property + cube = self.create_synthetic_test_cube( + template=template, subs=dict(TOPOLOGY_DIM_DEFINITION=dim_line) + ) + # Check we got just one message, and its content + self.assertEqual(len(log.records), 1) + rec = log.records[0] + self.assertEqual(rec.levelname, "WARNING") + re_msg = r"Mesh variable.* has no 'topology_dimension'" + self.assertRegex(rec.msg, re_msg) + + # Check that the result has the correct topology-dimension value. + self.assertEqual(cube.mesh.topology_dimension, 2) + + +if __name__ == "__main__": + tests.main() diff --git a/lib/iris/tests/stock/file_headers/minimal_bad_topology_dim.cdl b/lib/iris/tests/stock/file_headers/minimal_bad_topology_dim.cdl new file mode 100644 index 0000000000..44f3ef18f8 --- /dev/null +++ b/lib/iris/tests/stock/file_headers/minimal_bad_topology_dim.cdl @@ -0,0 +1,38 @@ +// Tolerant loading test example : the mesh has the wrong 'topology_dimension' +// NOTE: *not* truly minimal, as we cannot (yet) handle data with no face coords. +netcdf ${DATASET_NAME} { +dimensions: + NODES = ${NUM_NODES} ; + FACES = ${NUM_FACES} ; + FACE_CORNERS = 4 ; +variables: + int mesh_var ; + mesh_var:cf_role = "mesh_topology" ; + ${TOPOLOGY_DIM_DEFINITION} + mesh_var:node_coordinates = "mesh_node_x mesh_node_y" ; + mesh_var:face_node_connectivity = "mesh_face_nodes" ; + mesh_var:face_coordinates = "mesh_face_x mesh_face_y" ; + float mesh_node_x(NODES) ; + mesh_node_x:standard_name = "longitude" ; + mesh_node_x:long_name = "Longitude of mesh nodes." ; + mesh_node_x:units = "degrees_east" ; + float mesh_node_y(NODES) ; + mesh_node_y:standard_name = "latitude" ; + mesh_node_y:long_name = "Latitude of mesh nodes." ; + mesh_node_y:units = "degrees_north" ; + float mesh_face_x(FACES) ; + mesh_face_x:standard_name = "longitude" ; + mesh_face_x:long_name = "Longitude of mesh nodes." ; + mesh_face_x:units = "degrees_east" ; + float mesh_face_y(FACES) ; + mesh_face_y:standard_name = "latitude" ; + mesh_face_y:long_name = "Latitude of mesh nodes." ; + mesh_face_y:units = "degrees_north" ; + int mesh_face_nodes(FACES, FACE_CORNERS) ; + mesh_face_nodes:cf_role = "face_node_connectivity" ; + mesh_face_nodes:long_name = "Maps every face to its corner nodes." ; + mesh_face_nodes:start_index = 0 ; + float data_var(FACES) ; + data_var:mesh = "mesh_var" ; + data_var:location = "face" ; +}