From 77036993ad403f8e2bee1a78934fdfd7632822aa Mon Sep 17 00:00:00 2001 From: Patrick Peglar Date: Fri, 13 Dec 2019 09:43:22 +0000 Subject: [PATCH 1/6] PI-3473: Netcdf loading ancillary variables (#3556) * _regrid_area_weighted_array: Tweak variable order to near other use in code (#3571) * Fix problems with export and echo command. (#3577) * Pushdocs fix2 (#3580) * Revert to single-line command for doctr invocation. * Added script comment, partly to force Github respin. * Added whatsnew for Black. (#3581) * Fixes required due to the release of iris-grib v0.15.0 (#3582) * Fix python-eccodes pin in travis (#3593) * Netcdf load of ancillary vars: first working. --- .travis.yml | 13 ++- ...ange_2019-Dec-04_black_code_formatting.txt | 6 + lib/iris/experimental/regrid.py | 42 +++---- .../fileformats/_pyke_rules/fc_rules_cf.krb | 76 ++++++++++-- lib/iris/fileformats/netcdf.py | 44 ++++--- lib/iris/tests/__init__.py | 6 - lib/iris/tests/integration/test_grib2.py | 11 +- lib/iris/tests/integration/test_grib_load.py | 1 - lib/iris/tests/results/grib_load/3_layer.cml | 9 ++ .../tests/results/grib_load/earth_shape_0.cml | 3 + .../tests/results/grib_load/earth_shape_1.cml | 3 + .../tests/results/grib_load/earth_shape_2.cml | 3 + .../tests/results/grib_load/earth_shape_3.cml | 3 + .../tests/results/grib_load/earth_shape_4.cml | 3 + .../tests/results/grib_load/earth_shape_5.cml | 3 + .../tests/results/grib_load/earth_shape_6.cml | 3 + .../tests/results/grib_load/earth_shape_7.cml | 3 + .../tests/results/grib_load/ineg_jneg.cml | 3 + .../tests/results/grib_load/ineg_jpos.cml | 3 + .../tests/results/grib_load/ipos_jneg.cml | 3 + .../tests/results/grib_load/ipos_jpos.cml | 3 + .../tests/results/grib_load/lambert_grib2.cml | 3 + .../grib_load/missing_values_grib2.cml | 3 + .../results/grib_load/reduced_gg_grib2.cml | 1 + .../results/grib_load/regular_gg_grib2.cml | 3 + .../results/grib_load/time_bound_grib2.cml | 3 + .../grid_complex_spatial_differencing.cml | 3 + .../integration/grib2/TestGDT30/lambert.cml | 3 + .../integration/grib2/TestGDT40/reduced.cml | 1 + .../integration/grib2/TestGDT40/regular.cml | 3 + .../integration/grib2/TestImport/gdt1.cml | 3 + .../grib2/TestImport/gdt90_with_bitmap.cml | 3 + .../NAMEII/0_TRACER_AIR_CONCENTRATION.cml | 3 + .../name_grib/NAMEII/1_TRACER_DOSAGE.cml | 3 + .../NAMEII/2_TRACER_WET_DEPOSITION.cml | 31 ----- .../NAMEII/3_TRACER_DRY_DEPOSITION.cml | 3 + .../NAMEII/4_TRACER_TOTAL_DEPOSITION.cml | 3 + .../NAMEIII/0_TRACER_AIR_CONCENTRATION.cml | 3 + .../NAMEIII/1_TRACER_AIR_CONCENTRATION.cml | 3 + .../NAMEIII/2_TRACER_DRY_DEPOSITION.cml | 3 + .../NAMEIII/3_TRACER_WET_DEPOSITION.cml | 3 + .../name_grib/NAMEIII/4_TRACER_DEPOSITION.cml | 3 + .../system/supported_filetype_.grib2.cml | 3 + .../results/uri_callback/grib_global.cml | 3 + lib/iris/tests/test_netcdf.py | 108 ++++++++++++++++++ 45 files changed, 357 insertions(+), 88 deletions(-) create mode 100644 docs/iris/src/whatsnew/contributions_3.0.0/docchange_2019-Dec-04_black_code_formatting.txt delete mode 100644 lib/iris/tests/results/integration/name_grib/NAMEII/2_TRACER_WET_DEPOSITION.cml diff --git a/.travis.yml b/.travis.yml index 792f6a6195..a9f6bf3bfe 100644 --- a/.travis.yml +++ b/.travis.yml @@ -105,7 +105,7 @@ install: # Conda-forge versioning is out of order (0.9.* is later than 2.12.*). - > if [[ "${TEST_MINIMAL}" != true ]]; then - conda install --quiet -n ${ENV_NAME} python-eccodes=0.9.3; + conda install --quiet -n ${ENV_NAME} python-eccodes">=0.9.1, <2"; conda install --quiet -n ${ENV_NAME} --no-deps iris-grib; fi @@ -162,13 +162,18 @@ script: fi # Split the organisation out of the slug. See https://stackoverflow.com/a/5257398/741316 for description. - - export ORG=(${TRAVIS_REPO_SLUG//\// }) + # NOTE: a *separate* "export" command appears to be necessary here : A command of the + # form "export ORG=.." failed to define ORG for the following command (?!) + - > + ORG=$(echo ${TRAVIS_REPO_SLUG} | cut -d/ -f1); + export ORG + + - echo "Travis job context ORG=${ORG}; TRAVIS_EVENT_TYPE=${TRAVIS_EVENT_TYPE}; PUSH_BUILT_DOCS=${PUSH_BUILT_DOCS}" # When we merge a change to SciTools/iris, we can push docs to github pages. # At present, only the Python 3.7 "doctest" job does this. # Results appear at https://scitools-docs.github.io/iris/<>/index.html - - > - if [[ "${ORG}" == 'SciTools' && "${TRAVIS_EVENT_TYPE}" == 'push' && "${PUSH_BUILT_DOCS}" == 'true' ]]; then + - if [[ "${ORG}" == 'SciTools' && "${TRAVIS_EVENT_TYPE}" == 'push' && "${PUSH_BUILT_DOCS}" == 'true' ]]; then cd ${INSTALL_DIR}; pip install doctr; doctr deploy --deploy-repo SciTools-docs/iris --built-docs docs/iris/build/html diff --git a/docs/iris/src/whatsnew/contributions_3.0.0/docchange_2019-Dec-04_black_code_formatting.txt b/docs/iris/src/whatsnew/contributions_3.0.0/docchange_2019-Dec-04_black_code_formatting.txt new file mode 100644 index 0000000000..500a215bb9 --- /dev/null +++ b/docs/iris/src/whatsnew/contributions_3.0.0/docchange_2019-Dec-04_black_code_formatting.txt @@ -0,0 +1,6 @@ +* Added support for the `black `_ code formatter. + This is now automatically checked on GitHub PRs, replacing the older, unittest-based + "iris.tests.test_coding_standards.TestCodeFormat". + Black provides automatic code format correction for most IDEs. + See the new developer guide section on this : + https://scitools-docs.github.io/iris/master/developers_guide/code_format.html. diff --git a/lib/iris/experimental/regrid.py b/lib/iris/experimental/regrid.py index 87d9e7e902..afbf87dee8 100644 --- a/lib/iris/experimental/regrid.py +++ b/lib/iris/experimental/regrid.py @@ -473,6 +473,24 @@ def _regrid_area_weighted_array( grid. """ + # Determine which grid bounds are within src extent. + y_within_bounds = _within_bounds( + src_y_bounds, grid_y_bounds, grid_y_decreasing + ) + x_within_bounds = _within_bounds( + src_x_bounds, grid_x_bounds, grid_x_decreasing + ) + + # Cache which src_bounds are within grid bounds + cached_x_bounds = [] + cached_x_indices = [] + for (x_0, x_1) in grid_x_bounds: + if grid_x_decreasing: + x_0, x_1 = x_1, x_0 + x_bounds, x_indices = _cropped_bounds(src_x_bounds, x_0, x_1) + cached_x_bounds.append(x_bounds) + cached_x_indices.append(x_indices) + # Create empty data array to match the new grid. # Note that dtype is not preserved and that the array is # masked to allow for regions that do not overlap. @@ -497,24 +515,6 @@ def _regrid_area_weighted_array( # Assign to mask to explode it, allowing indexed assignment. new_data.mask = False - # Determine which grid bounds are within src extent. - y_within_bounds = _within_bounds( - src_y_bounds, grid_y_bounds, grid_y_decreasing - ) - x_within_bounds = _within_bounds( - src_x_bounds, grid_x_bounds, grid_x_decreasing - ) - - # Cache which src_bounds are within grid bounds - cached_x_bounds = [] - cached_x_indices = [] - for (x_0, x_1) in grid_x_bounds: - if grid_x_decreasing: - x_0, x_1 = x_1, x_0 - x_bounds, x_indices = _cropped_bounds(src_x_bounds, x_0, x_1) - cached_x_bounds.append(x_bounds) - cached_x_indices.append(x_indices) - # Axes of data over which the weighted mean is calculated. axes = [] if y_dim is not None: @@ -565,15 +565,15 @@ def _regrid_area_weighted_array( raise RuntimeError( "Cannot handle split bounds " "in both x and y." ) + # Calculate weights based on areas of cropped bounds. + weights = area_func(y_bounds, x_bounds) + if x_dim is not None: indices[x_dim] = x_indices if y_dim is not None: indices[y_dim] = y_indices data = src_data[tuple(indices)] - # Calculate weights based on areas of cropped bounds. - weights = area_func(y_bounds, x_bounds) - # Transpose weights to match dim ordering in data. weights_shape_y = weights.shape[0] weights_shape_x = weights.shape[1] diff --git a/lib/iris/fileformats/_pyke_rules/fc_rules_cf.krb b/lib/iris/fileformats/_pyke_rules/fc_rules_cf.krb index 5ecfeb77b1..815d71a5f4 100644 --- a/lib/iris/fileformats/_pyke_rules/fc_rules_cf.krb +++ b/lib/iris/fileformats/_pyke_rules/fc_rules_cf.krb @@ -498,6 +498,22 @@ fc_build_cell_measure python engine.rule_triggered.add(rule.name) +# +# Context: +# This rule will trigger for each ancillary_variable case specific fact. +# +# Purpose: +# Add the ancillary variable to the cube. +# +fc_build_ancil_var + foreach + facts_cf.ancillary_variable($var) + assert + python ancil_var = engine.cf_var.cf_group.ancillary_variables[$var] + python build_ancil_var(engine, ancil_var) + python engine.rule_triggered.add(rule.name) + + # # Context: # This rule will trigger iff a CF latitude coordinate exists and @@ -1941,25 +1957,26 @@ fc_extras # Add it to the cube cube.add_aux_coord(coord, data_dims) - # Update the coordinate to CF-netCDF variable mapping. + # Make a list with names, stored on the engine, so we can find them all later. engine.provides['coordinates'].append((coord, cf_coord_var.cf_name)) ################################################################################ - def build_cell_measures(engine, cf_cm_attr, coord_name=None): + def build_cell_measures(engine, cf_cm_var): """Create a CellMeasure instance and add it to the cube.""" cf_var = engine.cf_var cube = engine.cube attributes = {} # Get units - attr_units = get_attr_units(cf_cm_attr, attributes) + attr_units = get_attr_units(cf_cm_var, attributes) - data = _get_cf_var_data(cf_cm_attr, engine.filename) + # Get (lazy) content array + data = _get_cf_var_data(cf_cm_var, engine.filename) # Determine the name of the dimension/s shared between the CF-netCDF data variable # and the coordinate being built. - common_dims = [dim for dim in cf_cm_attr.dimensions + common_dims = [dim for dim in cf_cm_var.dimensions if dim in cf_var.dimensions] data_dims = None if common_dims: @@ -1967,10 +1984,10 @@ fc_extras data_dims = [cf_var.dimensions.index(dim) for dim in common_dims] # Determine the standard_name, long_name and var_name - standard_name, long_name, var_name = get_names(cf_cm_attr, coord_name, attributes) + standard_name, long_name, var_name = get_names(cf_cm_var, None, attributes) # Obtain the cf_measure. - measure = cf_cm_attr.cf_measure + measure = cf_cm_var.cf_measure # Create the CellMeasure cell_measure = iris.coords.CellMeasure(data, @@ -1984,6 +2001,51 @@ fc_extras # Add it to the cube cube.add_cell_measure(cell_measure, data_dims) + # Make a list with names, stored on the engine, so we can find them all later. + engine.provides['cell_measures'].append((cell_measure, cf_cm_var.cf_name)) + + + + ################################################################################ + def build_ancil_var(engine, cf_av_var): + """Create an AncillaryVariable instance and add it to the cube.""" + cf_var = engine.cf_var + cube = engine.cube + attributes = {} + + # Get units + attr_units = get_attr_units(cf_av_var, attributes) + + # Get (lazy) content array + data = _get_cf_var_data(cf_av_var, engine.filename) + + # Determine the name of the dimension/s shared between the CF-netCDF data variable + # and the AV being built. + common_dims = [dim for dim in cf_av_var.dimensions + if dim in cf_var.dimensions] + data_dims = None + if common_dims: + # Calculate the offset of each common dimension. + data_dims = [cf_var.dimensions.index(dim) for dim in common_dims] + + # Determine the standard_name, long_name and var_name + standard_name, long_name, var_name = get_names(cf_av_var, None, attributes) + + # Create the AncillaryVariable + av = iris.coords.AncillaryVariable( + data, + standard_name=standard_name, + long_name=long_name, + var_name=var_name, + units=attr_units, + attributes=attributes) + + # Add it to the cube + cube.add_ancillary_variable(av, data_dims) + + # Make a list with names, stored on the engine, so we can find them all later. + engine.provides['ancillary_variables'].append((av, cf_av_var.cf_name)) + ################################################################################ diff --git a/lib/iris/fileformats/netcdf.py b/lib/iris/fileformats/netcdf.py index 4d7ddedc61..08b079c3ed 100644 --- a/lib/iris/fileformats/netcdf.py +++ b/lib/iris/fileformats/netcdf.py @@ -459,7 +459,10 @@ def __setstate__(self, state): def _assert_case_specific_facts(engine, cf, cf_group): # Initialise pyke engine "provides" hooks. + # These are used to patch non-processed element attributes after rules activation. engine.provides["coordinates"] = [] + engine.provides["cell_measures"] = [] + engine.provides["ancillary_variables"] = [] # Assert facts for CF coordinates. for cf_name in cf_group.coordinates.keys(): @@ -479,6 +482,12 @@ def _assert_case_specific_facts(engine, cf, cf_group): _PYKE_FACT_BASE, "cell_measure", (cf_name,) ) + # Assert facts for CF ancillary variables. + for cf_name in cf_group.ancillary_variables.keys(): + engine.add_case_specific_fact( + _PYKE_FACT_BASE, "ancillary_variable", (cf_name,) + ) + # Assert facts for CF grid_mappings. for cf_name in cf_group.grid_mappings.keys(): engine.add_case_specific_fact( @@ -597,31 +606,38 @@ def _load_cube(engine, cf, cf_var, filename): # Run pyke inference engine with forward chaining rules. engine.activate(_PYKE_RULE_BASE) - # Populate coordinate attributes with the untouched attributes from the - # associated CF-netCDF variable. - coordinates = engine.provides.get("coordinates", []) - + # Having run the rules, now populate the attributes of all the cf elements with the + # "unused" attributes from the associated CF-netCDF variable. + # That is, all those that aren't CF reserved terms. def attribute_predicate(item): return item[0] not in _CF_ATTRS - for coord, cf_var_name in coordinates: - tmpvar = filter( - attribute_predicate, cf.cf_group[cf_var_name].cf_attrs_unused() - ) + def add_unused_attributes(iris_object, cf_var): + tmpvar = filter(attribute_predicate, cf_var.cf_attrs_unused()) for attr_name, attr_value in tmpvar: - _set_attributes(coord.attributes, attr_name, attr_value) + _set_attributes(iris_object.attributes, attr_name, attr_value) + + def fix_attributes_all_elements(role_name): + elements_and_names = engine.provides.get(role_name, []) + + for iris_object, cf_var_name in elements_and_names: + add_unused_attributes(iris_object, cf.cf_group[cf_var_name]) + + # Populate the attributes of all coordinates, cell-measures and ancillary-vars. + fix_attributes_all_elements("coordinates") + fix_attributes_all_elements("ancillary_variables") + fix_attributes_all_elements("cell_measures") - tmpvar = filter(attribute_predicate, cf_var.cf_attrs_unused()) - # Attach untouched attributes of the associated CF-netCDF data variable to - # the cube. - for attr_name, attr_value in tmpvar: - _set_attributes(cube.attributes, attr_name, attr_value) + # Also populate attributes of the top-level cube itself. + add_unused_attributes(cube, cf_var) + # Work out reference names for all the coords. names = { coord.var_name: coord.standard_name or coord.var_name or "unknown" for coord in cube.coords() } + # Add all the cube cell methods. cube.cell_methods = [ iris.coords.CellMethod( method=method.method, diff --git a/lib/iris/tests/__init__.py b/lib/iris/tests/__init__.py index f260bd0d26..d689738008 100644 --- a/lib/iris/tests/__init__.py +++ b/lib/iris/tests/__init__.py @@ -1297,12 +1297,6 @@ class MyPlotTests(test.GraphicsTest): ) -# TODO: remove these skips when iris-grib is fixed -skip_grib_fail = unittest.skipIf( - True, "Test(s) are failing due to known problems " 'with "iris-grib".' -) - - skip_sample_data = unittest.skipIf( not SAMPLE_DATA_AVAILABLE, ('Test(s) require "iris-sample-data", ' "which is not available."), diff --git a/lib/iris/tests/integration/test_grib2.py b/lib/iris/tests/integration/test_grib2.py index 1c94d424b0..691c4469d3 100644 --- a/lib/iris/tests/integration/test_grib2.py +++ b/lib/iris/tests/integration/test_grib2.py @@ -24,6 +24,7 @@ if tests.GRIB_AVAILABLE: from iris_grib import load_pairs_from_fields from iris_grib.message import GribMessage + from iris_grib.grib_phenom_translation import GRIBCode @tests.skip_data @@ -36,7 +37,6 @@ def test_gdt1(self): cube = load_cube(path) self.assertCMLApproxData(cube) - @tests.skip_grib_fail def test_gdt90_with_bitmap(self): path = tests.get_data_path(("GRIB", "umukv", "ukv_chan9.grib2")) cube = load_cube(path) @@ -156,6 +156,7 @@ def test_save_load(self): cube.add_aux_coord(tcoord) cube.add_aux_coord(fpcoord) cube.attributes["WMO_constituent_type"] = 0 + cube.attributes["GRIB_PARAM"] = GRIBCode("GRIB2:d000c014n000") with self.temp_filename("test_grib_pdt40.grib2") as temp_file_path: save(cube, temp_file_path) @@ -232,9 +233,12 @@ def test_save_load(self): self.assertEqual(test_cube.shape, (744, 744)) self.assertEqual(test_cube.cell_methods, ()) - # Check no cube attributes on the re-loaded cube. + # Check only the GRIB_PARAM attribute exists on the re-loaded cube. # Note: this does *not* match the original, but is as expected. - self.assertEqual(cube_loaded_from_saved.attributes, {}) + self.assertEqual( + cube_loaded_from_saved.attributes, + {"GRIB_PARAM": GRIBCode("GRIB2:d000c003n001")}, + ) # Now remaining to check: coordinates + data... @@ -300,7 +304,6 @@ def test_regular(self): cube = load_cube(path) self.assertCMLApproxData(cube) - @tests.skip_grib_fail def test_reduced(self): path = tests.get_data_path(("GRIB", "reduced", "reduced_gg.grib2")) cube = load_cube(path) diff --git a/lib/iris/tests/integration/test_grib_load.py b/lib/iris/tests/integration/test_grib_load.py index 230c792756..0e7548ee34 100644 --- a/lib/iris/tests/integration/test_grib_load.py +++ b/lib/iris/tests/integration/test_grib_load.py @@ -136,7 +136,6 @@ def test_reduced_ll(self): ) self.assertCML(cube, ("grib_load", "reduced_ll_grib1.cml")) - @tests.skip_grib_fail def test_reduced_gg(self): cube = iris.load_cube( tests.get_data_path(("GRIB", "reduced", "reduced_gg.grib2")) diff --git a/lib/iris/tests/results/grib_load/3_layer.cml b/lib/iris/tests/results/grib_load/3_layer.cml index 24f24ed777..76cc41a04a 100644 --- a/lib/iris/tests/results/grib_load/3_layer.cml +++ b/lib/iris/tests/results/grib_load/3_layer.cml @@ -1,6 +1,9 @@ + + + @@ -31,6 +34,9 @@ + + + @@ -82,6 +88,9 @@ + + + diff --git a/lib/iris/tests/results/grib_load/earth_shape_0.cml b/lib/iris/tests/results/grib_load/earth_shape_0.cml index 1e1e491d58..bb51db3201 100644 --- a/lib/iris/tests/results/grib_load/earth_shape_0.cml +++ b/lib/iris/tests/results/grib_load/earth_shape_0.cml @@ -1,6 +1,9 @@ + + + diff --git a/lib/iris/tests/results/grib_load/earth_shape_1.cml b/lib/iris/tests/results/grib_load/earth_shape_1.cml index dd409ebb20..774e9921b5 100644 --- a/lib/iris/tests/results/grib_load/earth_shape_1.cml +++ b/lib/iris/tests/results/grib_load/earth_shape_1.cml @@ -1,6 +1,9 @@ + + + diff --git a/lib/iris/tests/results/grib_load/earth_shape_2.cml b/lib/iris/tests/results/grib_load/earth_shape_2.cml index 0e3a4a14ea..3ff9ccccb5 100644 --- a/lib/iris/tests/results/grib_load/earth_shape_2.cml +++ b/lib/iris/tests/results/grib_load/earth_shape_2.cml @@ -1,6 +1,9 @@ + + + diff --git a/lib/iris/tests/results/grib_load/earth_shape_3.cml b/lib/iris/tests/results/grib_load/earth_shape_3.cml index 0213c4a4a0..47d11467ee 100644 --- a/lib/iris/tests/results/grib_load/earth_shape_3.cml +++ b/lib/iris/tests/results/grib_load/earth_shape_3.cml @@ -1,6 +1,9 @@ + + + diff --git a/lib/iris/tests/results/grib_load/earth_shape_4.cml b/lib/iris/tests/results/grib_load/earth_shape_4.cml index 2573e867d1..e6aa14e45a 100644 --- a/lib/iris/tests/results/grib_load/earth_shape_4.cml +++ b/lib/iris/tests/results/grib_load/earth_shape_4.cml @@ -1,6 +1,9 @@ + + + diff --git a/lib/iris/tests/results/grib_load/earth_shape_5.cml b/lib/iris/tests/results/grib_load/earth_shape_5.cml index 56462c684b..1257c9c2ad 100644 --- a/lib/iris/tests/results/grib_load/earth_shape_5.cml +++ b/lib/iris/tests/results/grib_load/earth_shape_5.cml @@ -1,6 +1,9 @@ + + + diff --git a/lib/iris/tests/results/grib_load/earth_shape_6.cml b/lib/iris/tests/results/grib_load/earth_shape_6.cml index 1ad54d1f77..eb96657104 100644 --- a/lib/iris/tests/results/grib_load/earth_shape_6.cml +++ b/lib/iris/tests/results/grib_load/earth_shape_6.cml @@ -1,6 +1,9 @@ + + + diff --git a/lib/iris/tests/results/grib_load/earth_shape_7.cml b/lib/iris/tests/results/grib_load/earth_shape_7.cml index cea76b2739..d27ce04a4c 100644 --- a/lib/iris/tests/results/grib_load/earth_shape_7.cml +++ b/lib/iris/tests/results/grib_load/earth_shape_7.cml @@ -1,6 +1,9 @@ + + + diff --git a/lib/iris/tests/results/grib_load/ineg_jneg.cml b/lib/iris/tests/results/grib_load/ineg_jneg.cml index 344fbbacf2..a7d7741092 100644 --- a/lib/iris/tests/results/grib_load/ineg_jneg.cml +++ b/lib/iris/tests/results/grib_load/ineg_jneg.cml @@ -1,6 +1,9 @@ + + + diff --git a/lib/iris/tests/results/grib_load/ineg_jpos.cml b/lib/iris/tests/results/grib_load/ineg_jpos.cml index 14967e6a88..f578fceadb 100644 --- a/lib/iris/tests/results/grib_load/ineg_jpos.cml +++ b/lib/iris/tests/results/grib_load/ineg_jpos.cml @@ -1,6 +1,9 @@ + + + diff --git a/lib/iris/tests/results/grib_load/ipos_jneg.cml b/lib/iris/tests/results/grib_load/ipos_jneg.cml index 1e1e491d58..bb51db3201 100644 --- a/lib/iris/tests/results/grib_load/ipos_jneg.cml +++ b/lib/iris/tests/results/grib_load/ipos_jneg.cml @@ -1,6 +1,9 @@ + + + diff --git a/lib/iris/tests/results/grib_load/ipos_jpos.cml b/lib/iris/tests/results/grib_load/ipos_jpos.cml index 373d8fc475..4dc6d7f980 100644 --- a/lib/iris/tests/results/grib_load/ipos_jpos.cml +++ b/lib/iris/tests/results/grib_load/ipos_jpos.cml @@ -1,6 +1,9 @@ + + + diff --git a/lib/iris/tests/results/grib_load/lambert_grib2.cml b/lib/iris/tests/results/grib_load/lambert_grib2.cml index e8b3f1c4c6..dc938f0aca 100644 --- a/lib/iris/tests/results/grib_load/lambert_grib2.cml +++ b/lib/iris/tests/results/grib_load/lambert_grib2.cml @@ -1,6 +1,9 @@ + + + diff --git a/lib/iris/tests/results/grib_load/missing_values_grib2.cml b/lib/iris/tests/results/grib_load/missing_values_grib2.cml index b090d56a92..c4c0d81915 100644 --- a/lib/iris/tests/results/grib_load/missing_values_grib2.cml +++ b/lib/iris/tests/results/grib_load/missing_values_grib2.cml @@ -1,6 +1,9 @@ + + + diff --git a/lib/iris/tests/results/grib_load/reduced_gg_grib2.cml b/lib/iris/tests/results/grib_load/reduced_gg_grib2.cml index f34938ce3f..fa3ba45e3d 100644 --- a/lib/iris/tests/results/grib_load/reduced_gg_grib2.cml +++ b/lib/iris/tests/results/grib_load/reduced_gg_grib2.cml @@ -2,6 +2,7 @@ + diff --git a/lib/iris/tests/results/grib_load/regular_gg_grib2.cml b/lib/iris/tests/results/grib_load/regular_gg_grib2.cml index 20230aee0f..14213c1602 100644 --- a/lib/iris/tests/results/grib_load/regular_gg_grib2.cml +++ b/lib/iris/tests/results/grib_load/regular_gg_grib2.cml @@ -1,6 +1,9 @@ + + + diff --git a/lib/iris/tests/results/grib_load/time_bound_grib2.cml b/lib/iris/tests/results/grib_load/time_bound_grib2.cml index 1e1e491d58..bb51db3201 100644 --- a/lib/iris/tests/results/grib_load/time_bound_grib2.cml +++ b/lib/iris/tests/results/grib_load/time_bound_grib2.cml @@ -1,6 +1,9 @@ + + + diff --git a/lib/iris/tests/results/integration/grib2/TestDRT3/grid_complex_spatial_differencing.cml b/lib/iris/tests/results/integration/grib2/TestDRT3/grid_complex_spatial_differencing.cml index b15c6a4308..2cfe06f8f6 100644 --- a/lib/iris/tests/results/integration/grib2/TestDRT3/grid_complex_spatial_differencing.cml +++ b/lib/iris/tests/results/integration/grib2/TestDRT3/grid_complex_spatial_differencing.cml @@ -1,6 +1,9 @@ + + + diff --git a/lib/iris/tests/results/integration/grib2/TestGDT30/lambert.cml b/lib/iris/tests/results/integration/grib2/TestGDT30/lambert.cml index a33d0b04ba..215a0de88d 100644 --- a/lib/iris/tests/results/integration/grib2/TestGDT30/lambert.cml +++ b/lib/iris/tests/results/integration/grib2/TestGDT30/lambert.cml @@ -1,6 +1,9 @@ + + + diff --git a/lib/iris/tests/results/integration/grib2/TestGDT40/reduced.cml b/lib/iris/tests/results/integration/grib2/TestGDT40/reduced.cml index f2ca666998..3a963b3203 100644 --- a/lib/iris/tests/results/integration/grib2/TestGDT40/reduced.cml +++ b/lib/iris/tests/results/integration/grib2/TestGDT40/reduced.cml @@ -2,6 +2,7 @@ + diff --git a/lib/iris/tests/results/integration/grib2/TestGDT40/regular.cml b/lib/iris/tests/results/integration/grib2/TestGDT40/regular.cml index fb6445b8b1..e5eea0fc7c 100644 --- a/lib/iris/tests/results/integration/grib2/TestGDT40/regular.cml +++ b/lib/iris/tests/results/integration/grib2/TestGDT40/regular.cml @@ -1,6 +1,9 @@ + + + diff --git a/lib/iris/tests/results/integration/grib2/TestImport/gdt1.cml b/lib/iris/tests/results/integration/grib2/TestImport/gdt1.cml index d3cc6b4732..d304d8a843 100644 --- a/lib/iris/tests/results/integration/grib2/TestImport/gdt1.cml +++ b/lib/iris/tests/results/integration/grib2/TestImport/gdt1.cml @@ -1,6 +1,9 @@ + + + diff --git a/lib/iris/tests/results/integration/grib2/TestImport/gdt90_with_bitmap.cml b/lib/iris/tests/results/integration/grib2/TestImport/gdt90_with_bitmap.cml index 9f950b5e1f..3118f86823 100644 --- a/lib/iris/tests/results/integration/grib2/TestImport/gdt90_with_bitmap.cml +++ b/lib/iris/tests/results/integration/grib2/TestImport/gdt90_with_bitmap.cml @@ -1,6 +1,9 @@ + + + diff --git a/lib/iris/tests/results/integration/name_grib/NAMEII/0_TRACER_AIR_CONCENTRATION.cml b/lib/iris/tests/results/integration/name_grib/NAMEII/0_TRACER_AIR_CONCENTRATION.cml index 4d0fddbba5..b0daf50907 100644 --- a/lib/iris/tests/results/integration/name_grib/NAMEII/0_TRACER_AIR_CONCENTRATION.cml +++ b/lib/iris/tests/results/integration/name_grib/NAMEII/0_TRACER_AIR_CONCENTRATION.cml @@ -1,6 +1,9 @@ + + + diff --git a/lib/iris/tests/results/integration/name_grib/NAMEII/1_TRACER_DOSAGE.cml b/lib/iris/tests/results/integration/name_grib/NAMEII/1_TRACER_DOSAGE.cml index fd61a67eb6..aef4988ce6 100644 --- a/lib/iris/tests/results/integration/name_grib/NAMEII/1_TRACER_DOSAGE.cml +++ b/lib/iris/tests/results/integration/name_grib/NAMEII/1_TRACER_DOSAGE.cml @@ -1,6 +1,9 @@ + + + diff --git a/lib/iris/tests/results/integration/name_grib/NAMEII/2_TRACER_WET_DEPOSITION.cml b/lib/iris/tests/results/integration/name_grib/NAMEII/2_TRACER_WET_DEPOSITION.cml deleted file mode 100644 index 029aa022ea..0000000000 --- a/lib/iris/tests/results/integration/name_grib/NAMEII/2_TRACER_WET_DEPOSITION.cml +++ /dev/null @@ -1,31 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/lib/iris/tests/results/integration/name_grib/NAMEII/3_TRACER_DRY_DEPOSITION.cml b/lib/iris/tests/results/integration/name_grib/NAMEII/3_TRACER_DRY_DEPOSITION.cml index 429d9db2ff..5787c19643 100644 --- a/lib/iris/tests/results/integration/name_grib/NAMEII/3_TRACER_DRY_DEPOSITION.cml +++ b/lib/iris/tests/results/integration/name_grib/NAMEII/3_TRACER_DRY_DEPOSITION.cml @@ -1,6 +1,9 @@ + + + diff --git a/lib/iris/tests/results/integration/name_grib/NAMEII/4_TRACER_TOTAL_DEPOSITION.cml b/lib/iris/tests/results/integration/name_grib/NAMEII/4_TRACER_TOTAL_DEPOSITION.cml index 429d9db2ff..5787c19643 100644 --- a/lib/iris/tests/results/integration/name_grib/NAMEII/4_TRACER_TOTAL_DEPOSITION.cml +++ b/lib/iris/tests/results/integration/name_grib/NAMEII/4_TRACER_TOTAL_DEPOSITION.cml @@ -1,6 +1,9 @@ + + + diff --git a/lib/iris/tests/results/integration/name_grib/NAMEIII/0_TRACER_AIR_CONCENTRATION.cml b/lib/iris/tests/results/integration/name_grib/NAMEIII/0_TRACER_AIR_CONCENTRATION.cml index 8412a4f814..1a31427de0 100644 --- a/lib/iris/tests/results/integration/name_grib/NAMEIII/0_TRACER_AIR_CONCENTRATION.cml +++ b/lib/iris/tests/results/integration/name_grib/NAMEIII/0_TRACER_AIR_CONCENTRATION.cml @@ -1,6 +1,9 @@ + + + diff --git a/lib/iris/tests/results/integration/name_grib/NAMEIII/1_TRACER_AIR_CONCENTRATION.cml b/lib/iris/tests/results/integration/name_grib/NAMEIII/1_TRACER_AIR_CONCENTRATION.cml index 590e8ef463..7007836e62 100644 --- a/lib/iris/tests/results/integration/name_grib/NAMEIII/1_TRACER_AIR_CONCENTRATION.cml +++ b/lib/iris/tests/results/integration/name_grib/NAMEIII/1_TRACER_AIR_CONCENTRATION.cml @@ -1,6 +1,9 @@ + + + diff --git a/lib/iris/tests/results/integration/name_grib/NAMEIII/2_TRACER_DRY_DEPOSITION.cml b/lib/iris/tests/results/integration/name_grib/NAMEIII/2_TRACER_DRY_DEPOSITION.cml index d3edb03a56..850ef89ed2 100644 --- a/lib/iris/tests/results/integration/name_grib/NAMEIII/2_TRACER_DRY_DEPOSITION.cml +++ b/lib/iris/tests/results/integration/name_grib/NAMEIII/2_TRACER_DRY_DEPOSITION.cml @@ -1,6 +1,9 @@ + + + diff --git a/lib/iris/tests/results/integration/name_grib/NAMEIII/3_TRACER_WET_DEPOSITION.cml b/lib/iris/tests/results/integration/name_grib/NAMEIII/3_TRACER_WET_DEPOSITION.cml index 3e8d62ef3d..ade4cea92d 100644 --- a/lib/iris/tests/results/integration/name_grib/NAMEIII/3_TRACER_WET_DEPOSITION.cml +++ b/lib/iris/tests/results/integration/name_grib/NAMEIII/3_TRACER_WET_DEPOSITION.cml @@ -1,6 +1,9 @@ + + + diff --git a/lib/iris/tests/results/integration/name_grib/NAMEIII/4_TRACER_DEPOSITION.cml b/lib/iris/tests/results/integration/name_grib/NAMEIII/4_TRACER_DEPOSITION.cml index 586aaa6b56..088b622c46 100644 --- a/lib/iris/tests/results/integration/name_grib/NAMEIII/4_TRACER_DEPOSITION.cml +++ b/lib/iris/tests/results/integration/name_grib/NAMEIII/4_TRACER_DEPOSITION.cml @@ -1,6 +1,9 @@ + + + diff --git a/lib/iris/tests/results/system/supported_filetype_.grib2.cml b/lib/iris/tests/results/system/supported_filetype_.grib2.cml index c230684fbd..f334b13863 100644 --- a/lib/iris/tests/results/system/supported_filetype_.grib2.cml +++ b/lib/iris/tests/results/system/supported_filetype_.grib2.cml @@ -1,6 +1,9 @@ + + + diff --git a/lib/iris/tests/results/uri_callback/grib_global.cml b/lib/iris/tests/results/uri_callback/grib_global.cml index a7a23e7235..aef0310a96 100644 --- a/lib/iris/tests/results/uri_callback/grib_global.cml +++ b/lib/iris/tests/results/uri_callback/grib_global.cml @@ -1,6 +1,9 @@ + + + diff --git a/lib/iris/tests/test_netcdf.py b/lib/iris/tests/test_netcdf.py index a550e1ed4b..91e37dd3a8 100644 --- a/lib/iris/tests/test_netcdf.py +++ b/lib/iris/tests/test_netcdf.py @@ -16,6 +16,7 @@ import os.path import shutil import stat +from subprocess import check_call import tempfile from unittest import mock @@ -27,8 +28,10 @@ import iris.analysis.trajectory import iris.fileformats._pyke_rules.compiled_krb.fc_rules_cf_fc as pyke_rules import iris.fileformats.netcdf +from iris.fileformats.netcdf import load_cubes as nc_load_cubes import iris.std_names import iris.util +from iris.coords import AncillaryVariable, CellMeasure import iris.coord_systems as icoord_systems import iris.tests.stock as stock from iris._lazy_data import is_lazy_data @@ -36,6 +39,13 @@ @tests.skip_data class TestNetCDFLoad(tests.IrisTest): + def setUp(self): + self.tmpdir = None + + def tearDown(self): + if self.tmpdir is not None: + shutil.rmtree(self.tmpdir) + def test_monotonic(self): cubes = iris.load( tests.get_data_path( @@ -240,6 +250,104 @@ def test_cell_methods(self): self.assertCML(cubes, ("netcdf", "netcdf_cell_methods.cml")) + def test_ancillary_variables(self): + # Note: using a CDL string as a test data reference, rather than a binary file. + ref_cdl = """ + netcdf cm_attr { + dimensions: + axv = 3 ; + variables: + int64 qqv(axv) ; + qqv:long_name = "qq" ; + qqv:units = "1" ; + qqv:ancillary_variables = "my_av" ; + int64 axv(axv) ; + axv:units = "1" ; + axv:long_name = "x" ; + double my_av(axv) ; + my_av:units = "1" ; + my_av:long_name = "refs" ; + my_av:custom = "extra-attribute"; + data: + axv = 1, 2, 3; + my_av = 11., 12., 13.; + } + """ + self.tmpdir = tempfile.mkdtemp() + cdl_path = os.path.join(self.tmpdir, "tst.cdl") + nc_path = os.path.join(self.tmpdir, "tst.nc") + # Write CDL string into a temporary CDL file. + with open(cdl_path, "w") as f_out: + f_out.write(ref_cdl) + # Use ncgen to convert this into an actual (temporary) netCDF file. + command = "ncgen -o {} {}".format(nc_path, cdl_path) + check_call(command, shell=True) + # Load with iris.fileformats.netcdf.load_cubes, and check expected content. + cubes = list(nc_load_cubes(nc_path)) + self.assertEqual(len(cubes), 1) + avs = cubes[0].ancillary_variables() + self.assertEqual(len(avs), 1) + expected = AncillaryVariable( + np.ma.array([11.0, 12.0, 13.0]), + long_name="refs", + var_name="my_av", + units="1", + attributes={"custom": "extra-attribute"}, + ) + self.assertEqual(avs[0], expected) + + def test_cell_measures(self): + # Note: using a CDL string as a test data reference, rather than a binary file. + ref_cdl = """ + netcdf cm_attr { + dimensions: + axv = 3 ; + ayv = 2 ; + variables: + int64 qqv(ayv, axv) ; + qqv:long_name = "qq" ; + qqv:units = "1" ; + qqv:cell_measures = "area: my_areas" ; + int64 ayv(ayv) ; + ayv:units = "1" ; + ayv:long_name = "y" ; + int64 axv(axv) ; + axv:units = "1" ; + axv:long_name = "x" ; + double my_areas(ayv, axv) ; + my_areas:units = "m2" ; + my_areas:long_name = "standardised cell areas" ; + my_areas:custom = "extra-attribute"; + data: + axv = 11, 12, 13; + ayv = 21, 22; + my_areas = 110., 120., 130., 221., 231., 241.; + } + """ + self.tmpdir = tempfile.mkdtemp() + cdl_path = os.path.join(self.tmpdir, "tst.cdl") + nc_path = os.path.join(self.tmpdir, "tst.nc") + # Write CDL string into a temporary CDL file. + with open(cdl_path, "w") as f_out: + f_out.write(ref_cdl) + # Use ncgen to convert this into an actual (temporary) netCDF file. + command = "ncgen -o {} {}".format(nc_path, cdl_path) + check_call(command, shell=True) + # Load with iris.fileformats.netcdf.load_cubes, and check expected content. + cubes = list(nc_load_cubes(nc_path)) + self.assertEqual(len(cubes), 1) + cms = cubes[0].cell_measures() + self.assertEqual(len(cms), 1) + expected = CellMeasure( + np.ma.array([[110.0, 120.0, 130.0], [221.0, 231.0, 241.0]]), + measure="area", + var_name="my_areas", + long_name="standardised cell areas", + units="m2", + attributes={"custom": "extra-attribute"}, + ) + self.assertEqual(cms[0], expected) + def test_deferred_loading(self): # Test exercising CF-netCDF deferred loading and deferred slicing. # shape (31, 161, 320) From db080fd3e8b6f5c764e014c10da896faade7374d Mon Sep 17 00:00:00 2001 From: Patrick Peglar Date: Tue, 17 Dec 2019 13:28:23 +0000 Subject: [PATCH 2/6] PI-3473: Whatsnews relating to ancillary load + save. (#3557) * Whatsnews relating to ancillary load + save. --- .../bugfix_2019-Nov-21_cell_measure_attributes.txt | 2 ++ .../newfeature_2019-Nov-21_netcdf_ancillary_data.txt | 2 ++ 2 files changed, 4 insertions(+) create mode 100644 docs/iris/src/whatsnew/contributions_3.0.0/bugfix_2019-Nov-21_cell_measure_attributes.txt create mode 100644 docs/iris/src/whatsnew/contributions_3.0.0/newfeature_2019-Nov-21_netcdf_ancillary_data.txt diff --git a/docs/iris/src/whatsnew/contributions_3.0.0/bugfix_2019-Nov-21_cell_measure_attributes.txt b/docs/iris/src/whatsnew/contributions_3.0.0/bugfix_2019-Nov-21_cell_measure_attributes.txt new file mode 100644 index 0000000000..577e193fe5 --- /dev/null +++ b/docs/iris/src/whatsnew/contributions_3.0.0/bugfix_2019-Nov-21_cell_measure_attributes.txt @@ -0,0 +1,2 @@ +* Fixed a bug where the attributes of cell measures in netcdf-CF files were discarded on + loading. They now appear on the CellMeasure in the loaded cube. diff --git a/docs/iris/src/whatsnew/contributions_3.0.0/newfeature_2019-Nov-21_netcdf_ancillary_data.txt b/docs/iris/src/whatsnew/contributions_3.0.0/newfeature_2019-Nov-21_netcdf_ancillary_data.txt new file mode 100644 index 0000000000..3d17ce3dd4 --- /dev/null +++ b/docs/iris/src/whatsnew/contributions_3.0.0/newfeature_2019-Nov-21_netcdf_ancillary_data.txt @@ -0,0 +1,2 @@ +* CF Ancillary Variables are now loaded from and saved to netcdf-CF files. + From 61e1efe24572053e216cddb203d0a6b74eab2f8a Mon Sep 17 00:00:00 2001 From: Patrick Peglar Date: Fri, 20 Dec 2019 22:06:00 +0000 Subject: [PATCH 3/6] PI-3473 Rename "engine.provides" (#3590) --- .../fileformats/_pyke_rules/fc_rules_cf.krb | 68 +++++++++---------- lib/iris/fileformats/netcdf.py | 12 ++-- .../netcdf/test__load_aux_factory.py | 22 +++--- .../fileformats/netcdf/test__load_cube.py | 2 +- .../test_build_auxiliary_coordinate.py | 17 +++-- .../test_build_dimension_coordinate.py | 14 ++-- 6 files changed, 70 insertions(+), 65 deletions(-) diff --git a/lib/iris/fileformats/_pyke_rules/fc_rules_cf.krb b/lib/iris/fileformats/_pyke_rules/fc_rules_cf.krb index 815d71a5f4..852c017651 100644 --- a/lib/iris/fileformats/_pyke_rules/fc_rules_cf.krb +++ b/lib/iris/fileformats/_pyke_rules/fc_rules_cf.krb @@ -42,7 +42,7 @@ fc_provides_grid_mapping_rotated_latitude_longitude assert python cf_grid_var = engine.cf_var.cf_group.grid_mappings[$grid_mapping] python coordinate_system = build_rotated_coordinate_system(engine, cf_grid_var) - python engine.provides['coordinate_system'] = coordinate_system + python engine.cube_parts['coordinate_system'] = coordinate_system facts_cf.provides(coordinate_system, rotated_latitude_longitude) python engine.rule_triggered.add(rule.name) @@ -62,7 +62,7 @@ fc_provides_grid_mapping_latitude_longitude assert python cf_grid_var = engine.cf_var.cf_group.grid_mappings[$grid_mapping] python coordinate_system = build_coordinate_system(cf_grid_var) - python engine.provides['coordinate_system'] = coordinate_system + python engine.cube_parts['coordinate_system'] = coordinate_system facts_cf.provides(coordinate_system, latitude_longitude) python engine.rule_triggered.add(rule.name) @@ -81,7 +81,7 @@ fc_provides_grid_mapping_transverse_mercator assert python cf_grid_var = engine.cf_var.cf_group.grid_mappings[$grid_mapping] python coordinate_system = build_transverse_mercator_coordinate_system(engine, cf_grid_var) - python engine.provides['coordinate_system'] = coordinate_system + python engine.cube_parts['coordinate_system'] = coordinate_system facts_cf.provides(coordinate_system, transverse_mercator) python engine.rule_triggered.add(rule.name) @@ -102,7 +102,7 @@ fc_provides_grid_mapping_mercator assert python cf_grid_var = engine.cf_var.cf_group.grid_mappings[$grid_mapping] python coordinate_system = build_mercator_coordinate_system(engine, cf_grid_var) - python engine.provides['coordinate_system'] = coordinate_system + python engine.cube_parts['coordinate_system'] = coordinate_system facts_cf.provides(coordinate_system, mercator) python engine.rule_triggered.add(rule.name) @@ -123,7 +123,7 @@ fc_provides_grid_mapping_stereographic assert python cf_grid_var = engine.cf_var.cf_group.grid_mappings[$grid_mapping] python coordinate_system = build_stereographic_coordinate_system(engine, cf_grid_var) - python engine.provides['coordinate_system'] = coordinate_system + python engine.cube_parts['coordinate_system'] = coordinate_system facts_cf.provides(coordinate_system, stereographic) python engine.rule_triggered.add(rule.name) @@ -142,7 +142,7 @@ fc_provides_grid_mapping_lambert_conformal assert python cf_grid_var = engine.cf_var.cf_group.grid_mappings[$grid_mapping] python coordinate_system = build_lambert_conformal_coordinate_system(engine, cf_grid_var) - python engine.provides['coordinate_system'] = coordinate_system + python engine.cube_parts['coordinate_system'] = coordinate_system facts_cf.provides(coordinate_system, lambert_conformal) python engine.rule_triggered.add(rule.name) @@ -161,7 +161,7 @@ fc_provides_grid_mapping_lambert_azimuthal_equal_area assert python cf_grid_var = engine.cf_var.cf_group.grid_mappings[$grid_mapping] python coordinate_system = build_lambert_azimuthal_equal_area_coordinate_system(engine, cf_grid_var) - python engine.provides['coordinate_system'] = coordinate_system + python engine.cube_parts['coordinate_system'] = coordinate_system facts_cf.provides(coordinate_system, lambert_azimuthal_equal_area) python engine.rule_triggered.add(rule.name) @@ -180,7 +180,7 @@ fc_provides_grid_mapping_albers_equal_area assert python cf_grid_var = engine.cf_var.cf_group.grid_mappings[$grid_mapping] python coordinate_system = build_albers_equal_area_coordinate_system(engine, cf_grid_var) - python engine.provides['coordinate_system'] = coordinate_system + python engine.cube_parts['coordinate_system'] = coordinate_system facts_cf.provides(coordinate_system, albers_equal_area) python engine.rule_triggered.add(rule.name) @@ -200,7 +200,7 @@ fc_provides_grid_mapping_vertical_perspective python cf_grid_var = engine.cf_var.cf_group.grid_mappings[$grid_mapping] python coordinate_system = \ build_vertical_perspective_coordinate_system(engine, cf_grid_var) - python engine.provides['coordinate_system'] = coordinate_system + python engine.cube_parts['coordinate_system'] = coordinate_system facts_cf.provides(coordinate_system, vertical_perspective) python engine.rule_triggered.add(rule.name) @@ -221,7 +221,7 @@ fc_provides_grid_mapping_geostationary python cf_grid_var = engine.cf_var.cf_group.grid_mappings[$grid_mapping] python coordinate_system = \ build_geostationary_coordinate_system(engine, cf_grid_var) - python engine.provides['coordinate_system'] = coordinate_system + python engine.cube_parts['coordinate_system'] = coordinate_system facts_cf.provides(coordinate_system, geostationary) python engine.rule_triggered.add(rule.name) @@ -531,7 +531,7 @@ fc_build_coordinate_latitude python cf_coord_var = engine.cf_var.cf_group.coordinates[$coordinate] python build_dimension_coordinate(engine, cf_coord_var, coord_name=CF_VALUE_STD_NAME_LAT, - coord_system=engine.provides['coordinate_system']) + coord_system=engine.cube_parts['coordinate_system']) python engine.rule_triggered.add(rule.name) @@ -552,7 +552,7 @@ fc_build_coordinate_latitude_rotated python cf_coord_var = engine.cf_var.cf_group.coordinates[$coordinate] python build_dimension_coordinate(engine, cf_coord_var, coord_name=CF_VALUE_STD_NAME_GRID_LAT, - coord_system=engine.provides['coordinate_system']) + coord_system=engine.cube_parts['coordinate_system']) python engine.rule_triggered.add(rule.name) @@ -573,7 +573,7 @@ fc_build_coordinate_longitude python cf_coord_var = engine.cf_var.cf_group.coordinates[$coordinate] python build_dimension_coordinate(engine, cf_coord_var, coord_name=CF_VALUE_STD_NAME_LON, - coord_system=engine.provides['coordinate_system']) + coord_system=engine.cube_parts['coordinate_system']) python engine.rule_triggered.add(rule.name) @@ -594,7 +594,7 @@ fc_build_coordinate_longitude_rotated python cf_coord_var = engine.cf_var.cf_group.coordinates[$coordinate] python build_dimension_coordinate(engine, cf_coord_var, coord_name=CF_VALUE_STD_NAME_GRID_LON, - coord_system=engine.provides['coordinate_system']) + coord_system=engine.cube_parts['coordinate_system']) python engine.rule_triggered.add(rule.name) @@ -660,7 +660,7 @@ fc_build_coordinate_projection_x_transverse_mercator python cf_coord_var = engine.cf_var.cf_group.coordinates[$coordinate] python build_dimension_coordinate(engine, cf_coord_var, coord_name=CF_VALUE_STD_NAME_PROJ_X, - coord_system=engine.provides['coordinate_system']) + coord_system=engine.cube_parts['coordinate_system']) python engine.rule_triggered.add(rule.name) @@ -680,7 +680,7 @@ fc_build_coordinate_projection_y_transverse_mercator python cf_coord_var = engine.cf_var.cf_group.coordinates[$coordinate] python build_dimension_coordinate(engine, cf_coord_var, coord_name=CF_VALUE_STD_NAME_PROJ_Y, - coord_system=engine.provides['coordinate_system']) + coord_system=engine.cube_parts['coordinate_system']) python engine.rule_triggered.add(rule.name) # @@ -699,7 +699,7 @@ fc_build_coordinate_projection_x_lambert_conformal python cf_coord_var = engine.cf_var.cf_group.coordinates[$coordinate] python build_dimension_coordinate(engine, cf_coord_var, coord_name=CF_VALUE_STD_NAME_PROJ_X, - coord_system=engine.provides['coordinate_system']) + coord_system=engine.cube_parts['coordinate_system']) python engine.rule_triggered.add(rule.name) @@ -719,7 +719,7 @@ fc_build_coordinate_projection_y_lambert_conformal python cf_coord_var = engine.cf_var.cf_group.coordinates[$coordinate] python build_dimension_coordinate(engine, cf_coord_var, coord_name=CF_VALUE_STD_NAME_PROJ_Y, - coord_system=engine.provides['coordinate_system']) + coord_system=engine.cube_parts['coordinate_system']) python engine.rule_triggered.add(rule.name) @@ -739,7 +739,7 @@ fc_build_coordinate_projection_x_mercator python cf_coord_var = engine.cf_var.cf_group.coordinates[$coordinate] python build_dimension_coordinate(engine, cf_coord_var, coord_name=CF_VALUE_STD_NAME_PROJ_X, - coord_system=engine.provides['coordinate_system']) + coord_system=engine.cube_parts['coordinate_system']) python engine.rule_triggered.add(rule.name) # @@ -758,7 +758,7 @@ fc_build_coordinate_projection_y_mercator python cf_coord_var = engine.cf_var.cf_group.coordinates[$coordinate] python build_dimension_coordinate(engine, cf_coord_var, coord_name=CF_VALUE_STD_NAME_PROJ_Y, - coord_system=engine.provides['coordinate_system']) + coord_system=engine.cube_parts['coordinate_system']) python engine.rule_triggered.add(rule.name) # @@ -777,7 +777,7 @@ fc_build_coordinate_projection_x_stereographic python cf_coord_var = engine.cf_var.cf_group.coordinates[$coordinate] python build_dimension_coordinate(engine, cf_coord_var, coord_name=CF_VALUE_STD_NAME_PROJ_X, - coord_system=engine.provides['coordinate_system']) + coord_system=engine.cube_parts['coordinate_system']) python engine.rule_triggered.add(rule.name) # @@ -796,7 +796,7 @@ fc_build_coordinate_projection_y_stereographic python cf_coord_var = engine.cf_var.cf_group.coordinates[$coordinate] python build_dimension_coordinate(engine, cf_coord_var, coord_name=CF_VALUE_STD_NAME_PROJ_Y, - coord_system=engine.provides['coordinate_system']) + coord_system=engine.cube_parts['coordinate_system']) python engine.rule_triggered.add(rule.name) @@ -816,7 +816,7 @@ fc_build_coordinate_projection_x_lambert_azimuthal_equal_area python cf_coord_var = engine.cf_var.cf_group.coordinates[$coordinate] python build_dimension_coordinate(engine, cf_coord_var, coord_name=CF_VALUE_STD_NAME_PROJ_X, - coord_system=engine.provides['coordinate_system']) + coord_system=engine.cube_parts['coordinate_system']) python engine.rule_triggered.add(rule.name) @@ -836,7 +836,7 @@ fc_build_coordinate_projection_y_lambert_azimuthal_equal_area python cf_coord_var = engine.cf_var.cf_group.coordinates[$coordinate] python build_dimension_coordinate(engine, cf_coord_var, coord_name=CF_VALUE_STD_NAME_PROJ_Y, - coord_system=engine.provides['coordinate_system']) + coord_system=engine.cube_parts['coordinate_system']) python engine.rule_triggered.add(rule.name) # @@ -855,7 +855,7 @@ fc_build_coordinate_projection_x_albers_equal_area python cf_coord_var = engine.cf_var.cf_group.coordinates[$coordinate] python build_dimension_coordinate(engine, cf_coord_var, coord_name=CF_VALUE_STD_NAME_PROJ_X, - coord_system=engine.provides['coordinate_system']) + coord_system=engine.cube_parts['coordinate_system']) python engine.rule_triggered.add(rule.name) @@ -875,7 +875,7 @@ fc_build_coordinate_projection_y_albers_equal_area python cf_coord_var = engine.cf_var.cf_group.coordinates[$coordinate] python build_dimension_coordinate(engine, cf_coord_var, coord_name=CF_VALUE_STD_NAME_PROJ_Y, - coord_system=engine.provides['coordinate_system']) + coord_system=engine.cube_parts['coordinate_system']) python engine.rule_triggered.add(rule.name) # @@ -894,7 +894,7 @@ fc_build_coordinate_projection_x_vertical_perspective python cf_coord_var = engine.cf_var.cf_group.coordinates[$coordinate] python build_dimension_coordinate(engine, cf_coord_var, coord_name=CF_VALUE_STD_NAME_PROJ_X, - coord_system=engine.provides['coordinate_system']) + coord_system=engine.cube_parts['coordinate_system']) python engine.rule_triggered.add(rule.name) @@ -914,7 +914,7 @@ fc_build_coordinate_projection_y_vertical_perspective python cf_coord_var = engine.cf_var.cf_group.coordinates[$coordinate] python build_dimension_coordinate(engine, cf_coord_var, coord_name=CF_VALUE_STD_NAME_PROJ_Y, - coord_system=engine.provides['coordinate_system']) + coord_system=engine.cube_parts['coordinate_system']) python engine.rule_triggered.add(rule.name) # @@ -933,7 +933,7 @@ fc_build_coordinate_projection_x_geostationary python cf_coord_var = engine.cf_var.cf_group.coordinates[$coordinate] python build_dimension_coordinate(engine, cf_coord_var, coord_name=CF_VALUE_STD_NAME_PROJ_X, - coord_system=engine.provides['coordinate_system']) + coord_system=engine.cube_parts['coordinate_system']) python engine.rule_triggered.add(rule.name) @@ -953,7 +953,7 @@ fc_build_coordinate_projection_y_geostationary python cf_coord_var = engine.cf_var.cf_group.coordinates[$coordinate] python build_dimension_coordinate(engine, cf_coord_var, coord_name=CF_VALUE_STD_NAME_PROJ_Y, - coord_system=engine.provides['coordinate_system']) + coord_system=engine.cube_parts['coordinate_system']) python engine.rule_triggered.add(rule.name) @@ -1892,7 +1892,7 @@ fc_extras cube.add_aux_coord(coord, data_dims) # Update the coordinate to CF-netCDF variable mapping. - engine.provides['coordinates'].append((coord, cf_coord_var.cf_name)) + engine.cube_parts['coordinates'].append((coord, cf_coord_var.cf_name)) ################################################################################ @@ -1958,7 +1958,7 @@ fc_extras cube.add_aux_coord(coord, data_dims) # Make a list with names, stored on the engine, so we can find them all later. - engine.provides['coordinates'].append((coord, cf_coord_var.cf_name)) + engine.cube_parts['coordinates'].append((coord, cf_coord_var.cf_name)) ################################################################################ @@ -2002,7 +2002,7 @@ fc_extras cube.add_cell_measure(cell_measure, data_dims) # Make a list with names, stored on the engine, so we can find them all later. - engine.provides['cell_measures'].append((cell_measure, cf_cm_var.cf_name)) + engine.cube_parts['cell_measures'].append((cell_measure, cf_cm_var.cf_name)) @@ -2044,7 +2044,7 @@ fc_extras cube.add_ancillary_variable(av, data_dims) # Make a list with names, stored on the engine, so we can find them all later. - engine.provides['ancillary_variables'].append((av, cf_av_var.cf_name)) + engine.cube_parts['ancillary_variables'].append((av, cf_av_var.cf_name)) diff --git a/lib/iris/fileformats/netcdf.py b/lib/iris/fileformats/netcdf.py index 08b079c3ed..f70e993d0a 100644 --- a/lib/iris/fileformats/netcdf.py +++ b/lib/iris/fileformats/netcdf.py @@ -460,9 +460,9 @@ def __setstate__(self, state): def _assert_case_specific_facts(engine, cf, cf_group): # Initialise pyke engine "provides" hooks. # These are used to patch non-processed element attributes after rules activation. - engine.provides["coordinates"] = [] - engine.provides["cell_measures"] = [] - engine.provides["ancillary_variables"] = [] + engine.cube_parts["coordinates"] = [] + engine.cube_parts["cell_measures"] = [] + engine.cube_parts["ancillary_variables"] = [] # Assert facts for CF coordinates. for cf_name in cf_group.coordinates.keys(): @@ -595,7 +595,7 @@ def _load_cube(engine, cf, cf_var, filename): # Initialise pyke engine rule processing hooks. engine.cf_var = cf_var engine.cube = cube - engine.provides = {} + engine.cube_parts = {} engine.requires = {} engine.rule_triggered = set() engine.filename = filename @@ -618,7 +618,7 @@ def add_unused_attributes(iris_object, cf_var): _set_attributes(iris_object.attributes, attr_name, attr_value) def fix_attributes_all_elements(role_name): - elements_and_names = engine.provides.get(role_name, []) + elements_and_names = engine.cube_parts.get(role_name, []) for iris_object, cf_var_name in elements_and_names: add_unused_attributes(iris_object, cf.cf_group[cf_var_name]) @@ -677,7 +677,7 @@ def coord_from_term(term): # Convert term names to coordinates (via netCDF variable names). name = engine.requires["formula_terms"].get(term, None) if name is not None: - for coord, cf_var_name in engine.provides["coordinates"]: + for coord, cf_var_name in engine.cube_parts["coordinates"]: if cf_var_name == name: return coord warnings.warn( diff --git a/lib/iris/tests/unit/fileformats/netcdf/test__load_aux_factory.py b/lib/iris/tests/unit/fileformats/netcdf/test__load_aux_factory.py index 3bbac6b309..59e4356e70 100644 --- a/lib/iris/tests/unit/fileformats/netcdf/test__load_aux_factory.py +++ b/lib/iris/tests/unit/fileformats/netcdf/test__load_aux_factory.py @@ -24,8 +24,10 @@ def setUp(self): standard_name = "atmosphere_hybrid_sigma_pressure_coordinate" self.requires = dict(formula_type=standard_name) coordinates = [(mock.sentinel.b, "b"), (mock.sentinel.ps, "ps")] - self.provides = dict(coordinates=coordinates) - self.engine = mock.Mock(requires=self.requires, provides=self.provides) + self.cube_parts = dict(coordinates=coordinates) + self.engine = mock.Mock( + requires=self.requires, cube_parts=self.cube_parts + ) self.cube = mock.create_autospec(Cube, spec_set=True, instance=True) # Patch out the check_dependencies functionality. func = "iris.aux_factory.HybridPressureFactory._check_dependencies" @@ -34,7 +36,7 @@ def setUp(self): self.addCleanup(patcher.stop) def test_formula_terms_ap(self): - self.provides["coordinates"].append((mock.sentinel.ap, "ap")) + self.cube_parts["coordinates"].append((mock.sentinel.ap, "ap")) self.requires["formula_terms"] = dict(ap="ap", b="b", ps="ps") _load_aux_factory(self.engine, self.cube) # Check cube.add_aux_coord method. @@ -57,7 +59,9 @@ def test_formula_terms_a_p0(self): long_name="vertical pressure", var_name="ap", ) - self.provides["coordinates"].extend([(coord_a, "a"), (coord_p0, "p0")]) + self.cube_parts["coordinates"].extend( + [(coord_a, "a"), (coord_p0, "p0")] + ) self.requires["formula_terms"] = dict(a="a", b="b", ps="ps", p0="p0") _load_aux_factory(self.engine, self.cube) # Check cube.coord_dims method. @@ -82,7 +86,7 @@ def test_formula_terms_a_p0(self): def test_formula_terms_p0_non_scalar(self): coord_p0 = DimCoord(np.arange(5)) - self.provides["coordinates"].append((coord_p0, "p0")) + self.cube_parts["coordinates"].append((coord_p0, "p0")) self.requires["formula_terms"] = dict(p0="p0") with self.assertRaises(ValueError): _load_aux_factory(self.engine, self.cube) @@ -90,7 +94,9 @@ def test_formula_terms_p0_non_scalar(self): def test_formula_terms_p0_bounded(self): coord_a = DimCoord(np.arange(5)) coord_p0 = DimCoord(1, bounds=[0, 2], var_name="p0") - self.provides["coordinates"].extend([(coord_a, "a"), (coord_p0, "p0")]) + self.cube_parts["coordinates"].extend( + [(coord_a, "a"), (coord_p0, "p0")] + ) self.requires["formula_terms"] = dict(a="a", b="b", ps="ps", p0="p0") with warnings.catch_warnings(record=True) as warn: warnings.simplefilter("always") @@ -131,14 +137,14 @@ def test_formula_terms_no_delta_terms(self): def test_formula_terms_no_p0_term(self): coord_a = DimCoord(np.arange(5), units="Pa") - self.provides["coordinates"].append((coord_a, "a")) + self.cube_parts["coordinates"].append((coord_a, "a")) self.requires["formula_terms"] = dict(a="a", b="b", ps="ps") _load_aux_factory(self.engine, self.cube) self._check_no_delta() def test_formula_terms_no_a_term(self): coord_p0 = DimCoord(10, units="1") - self.provides["coordinates"].append((coord_p0, "p0")) + self.cube_parts["coordinates"].append((coord_p0, "p0")) self.requires["formula_terms"] = dict(a="p0", b="b", ps="ps") _load_aux_factory(self.engine, self.cube) self._check_no_delta() diff --git a/lib/iris/tests/unit/fileformats/netcdf/test__load_cube.py b/lib/iris/tests/unit/fileformats/netcdf/test__load_cube.py index ffe48d437d..b4c93c45ab 100644 --- a/lib/iris/tests/unit/fileformats/netcdf/test__load_cube.py +++ b/lib/iris/tests/unit/fileformats/netcdf/test__load_cube.py @@ -25,7 +25,7 @@ def _patcher(engine, cf, cf_group): for coord in cf_group: engine.cube.add_aux_coord(coord) coordinates.append((coord, coord.name())) - engine.provides["coordinates"] = coordinates + engine.cube_parts["coordinates"] = coordinates def setUp(self): this = "iris.fileformats.netcdf._assert_case_specific_facts" diff --git a/lib/iris/tests/unit/fileformats/pyke_rules/compiled_krb/fc_rules_cf_fc/test_build_auxiliary_coordinate.py b/lib/iris/tests/unit/fileformats/pyke_rules/compiled_krb/fc_rules_cf_fc/test_build_auxiliary_coordinate.py index 70d72fb133..62daff146d 100644 --- a/lib/iris/tests/unit/fileformats/pyke_rules/compiled_krb/fc_rules_cf_fc/test_build_auxiliary_coordinate.py +++ b/lib/iris/tests/unit/fileformats/pyke_rules/compiled_krb/fc_rules_cf_fc/test_build_auxiliary_coordinate.py @@ -63,12 +63,11 @@ def setUp(self): cf_var=mock.Mock(dimensions=('foo', 'bar'), cf_data=cf_data), filename='DUMMY', - provides=dict(coordinates=[])) + cube_parts=dict(coordinates=[])) # Patch the deferred loading that prevents attempted file access. # This assumes that self.cf_bounds_var is defined in the test case. def patched__getitem__(proxy_self, keys): - variable = None for var in (self.cf_coord_var, self.cf_bounds_var): if proxy_self.variable_name == var.cf_name: return var[keys] @@ -121,9 +120,9 @@ def _check_case(self, dimension_names): self.engine.cube.add_aux_coord.assert_called_with( self.expected_coord, [0, 1]) - # Test that engine.provides container is correctly populated. + # Test that engine.cube_parts container is correctly populated. expected_list = [(self.expected_coord, self.cf_coord_var.cf_name)] - self.assertEqual(self.engine.provides['coordinates'], + self.assertEqual(self.engine.cube_parts['coordinates'], expected_list) def test_fastest_varying_vertex_dim(self): @@ -164,7 +163,7 @@ def setUp(self): cube=mock.Mock(), cf_var=mock.Mock(dimensions=('foo', 'bar')), filename='DUMMY', - provides=dict(coordinates=[])) + cube_parts=dict(coordinates=[])) def patched__getitem__(proxy_self, keys): if proxy_self.variable_name == self.cf_coord_var.cf_name: @@ -182,7 +181,7 @@ def test_scale_factor_add_offset_int(self): with self.deferred_load_patch: build_auxiliary_coordinate(self.engine, self.cf_coord_var) - coord, _ = self.engine.provides['coordinates'][0] + coord, _ = self.engine.cube_parts['coordinates'][0] self.assertEqual(coord.dtype.kind, 'i') def test_scale_factor_float(self): @@ -191,7 +190,7 @@ def test_scale_factor_float(self): with self.deferred_load_patch: build_auxiliary_coordinate(self.engine, self.cf_coord_var) - coord, _ = self.engine.provides['coordinates'][0] + coord, _ = self.engine.cube_parts['coordinates'][0] self.assertEqual(coord.dtype.kind, 'f') def test_add_offset_float(self): @@ -200,7 +199,7 @@ def test_add_offset_float(self): with self.deferred_load_patch: build_auxiliary_coordinate(self.engine, self.cf_coord_var) - coord, _ = self.engine.provides['coordinates'][0] + coord, _ = self.engine.cube_parts['coordinates'][0] self.assertEqual(coord.dtype.kind, 'f') @@ -211,7 +210,7 @@ def setUp(self): cube=mock.Mock(), cf_var=mock.Mock(dimensions=('foo', 'bar')), filename='DUMMY', - provides=dict(coordinates=[])) + cube_parts=dict(coordinates=[])) points = np.arange(6) self.cf_coord_var = mock.Mock( diff --git a/lib/iris/tests/unit/fileformats/pyke_rules/compiled_krb/fc_rules_cf_fc/test_build_dimension_coordinate.py b/lib/iris/tests/unit/fileformats/pyke_rules/compiled_krb/fc_rules_cf_fc/test_build_dimension_coordinate.py index 1917034a6b..d753636e70 100644 --- a/lib/iris/tests/unit/fileformats/pyke_rules/compiled_krb/fc_rules_cf_fc/test_build_dimension_coordinate.py +++ b/lib/iris/tests/unit/fileformats/pyke_rules/compiled_krb/fc_rules_cf_fc/test_build_dimension_coordinate.py @@ -30,7 +30,7 @@ def setUp(self): cube=mock.Mock(), cf_var=mock.Mock(dimensions=('foo', 'bar')), filename='DUMMY', - provides=dict(coordinates=[])) + cube_parts=dict(coordinates=[])) # Create patch for deferred loading that prevents attempted # file access. This assumes that self.cf_coord_var and @@ -258,9 +258,9 @@ def test_slowest_varying_vertex_dim(self): self.engine.cube.add_dim_coord.assert_called_with( expected_coord, [0]) - # Test that engine.provides container is correctly populated. + # Test that engine.cube_parts container is correctly populated. expected_list = [(expected_coord, self.cf_coord_var.cf_name)] - self.assertEqual(self.engine.provides['coordinates'], + self.assertEqual(self.engine.cube_parts['coordinates'], expected_list) def test_fastest_varying_vertex_dim(self): @@ -286,9 +286,9 @@ def test_fastest_varying_vertex_dim(self): self.engine.cube.add_dim_coord.assert_called_with( expected_coord, [0]) - # Test that engine.provides container is correctly populated. + # Test that engine.cube_parts container is correctly populated. expected_list = [(expected_coord, self.cf_coord_var.cf_name)] - self.assertEqual(self.engine.provides['coordinates'], + self.assertEqual(self.engine.cube_parts['coordinates'], expected_list) def test_fastest_with_different_dim_names(self): @@ -317,9 +317,9 @@ def test_fastest_with_different_dim_names(self): self.engine.cube.add_dim_coord.assert_called_with( expected_coord, [0]) - # Test that engine.provides container is correctly populated. + # Test that engine.cube_parts container is correctly populated. expected_list = [(expected_coord, self.cf_coord_var.cf_name)] - self.assertEqual(self.engine.provides['coordinates'], + self.assertEqual(self.engine.cube_parts['coordinates'], expected_list) From 25ea1baa04158e7e892eea08e2c968d48696dda5 Mon Sep 17 00:00:00 2001 From: stephenworsley <49274989+stephenworsley@users.noreply.github.com> Date: Fri, 5 Jun 2020 12:35:01 +0100 Subject: [PATCH 4/6] PI:3358 (WIP) Ensure flags load/save without units (#3613) * change dependencies : NOTE these changes need removing when remerging to master --- .../fileformats/_pyke_rules/fc_rules_cf.krb | 3 ++ lib/iris/fileformats/netcdf.py | 6 +-- .../results/netcdf/netcdf_save_no_name.cdl | 1 - lib/iris/tests/test_netcdf.py | 49 +++++++++++++++++++ .../test_build_auxiliary_coordinate.py | 6 +-- .../test_build_cube_metadata.py | 2 +- .../test_build_dimension_coordinate.py | 6 ++- .../fc_rules_cf_fc/test_get_attr_units.py | 3 +- requirements/core.txt | 6 +-- requirements/test.txt | 3 +- 10 files changed, 71 insertions(+), 14 deletions(-) diff --git a/lib/iris/fileformats/_pyke_rules/fc_rules_cf.krb b/lib/iris/fileformats/_pyke_rules/fc_rules_cf.krb index 852c017651..34d5f7edc0 100644 --- a/lib/iris/fileformats/_pyke_rules/fc_rules_cf.krb +++ b/lib/iris/fileformats/_pyke_rules/fc_rules_cf.krb @@ -1689,6 +1689,9 @@ fc_extras if np.issubdtype(cf_var.dtype, np.str_): attr_units = cf_units._NO_UNIT_STRING + if any(hasattr(cf_var.cf_data, name) for name in ("flag_values", "flag_masks", "flag_meanings")): + attr_units = cf_units._NO_UNIT_STRING + # Get any assoicated calendar for a time reference coordinate. if cf_units.as_unit(attr_units).is_time_reference(): attr_calendar = getattr(cf_var, CF_ATTR_CALENDAR, None) diff --git a/lib/iris/fileformats/netcdf.py b/lib/iris/fileformats/netcdf.py index f70e993d0a..e1481cf5ed 100644 --- a/lib/iris/fileformats/netcdf.py +++ b/lib/iris/fileformats/netcdf.py @@ -1776,7 +1776,7 @@ def _inner_create_cf_cellmeasure_or_ancil_variable( # Add the data to the CF-netCDF variable. cf_var[:] = data - if dimensional_metadata.units != "unknown": + if dimensional_metadata.units not in ("no_unit", "unknown"): _setncattr(cf_var, "units", str(dimensional_metadata.units)) if dimensional_metadata.standard_name is not None: @@ -1942,7 +1942,7 @@ def _create_cf_coord_variable(self, cube, dimension_names, coord): # Deal with CF-netCDF units and standard name. standard_name, long_name, units = self._cf_coord_identity(coord) - if units != "unknown": + if units not in ("no_unit", "unknown"): _setncattr(cf_var, "units", units) if standard_name is not None: @@ -2387,7 +2387,7 @@ def store(data, cf_var, fill_value): if cube.long_name: _setncattr(cf_var, "long_name", cube.long_name) - if cube.units != "unknown": + if cube.units not in ("no_unit", "unknown"): _setncattr(cf_var, "units", str(cube.units)) # Add the CF-netCDF calendar attribute. diff --git a/lib/iris/tests/results/netcdf/netcdf_save_no_name.cdl b/lib/iris/tests/results/netcdf/netcdf_save_no_name.cdl index e67316b2f7..e01eb1b31a 100644 --- a/lib/iris/tests/results/netcdf/netcdf_save_no_name.cdl +++ b/lib/iris/tests/results/netcdf/netcdf_save_no_name.cdl @@ -10,7 +10,6 @@ variables: double dim1(dim1) ; dim1:units = "m" ; char unknown_scalar(string6) ; - unknown_scalar:units = "no_unit" ; // global attributes: :Conventions = "CF-1.7" ; diff --git a/lib/iris/tests/test_netcdf.py b/lib/iris/tests/test_netcdf.py index 91e37dd3a8..96e432714b 100644 --- a/lib/iris/tests/test_netcdf.py +++ b/lib/iris/tests/test_netcdf.py @@ -296,6 +296,55 @@ def test_ancillary_variables(self): ) self.assertEqual(avs[0], expected) + def test_status_flags(self): + # Note: using a CDL string as a test data reference, rather than a binary file. + ref_cdl = """ + netcdf cm_attr { + dimensions: + axv = 3 ; + variables: + int64 qqv(axv) ; + qqv:long_name = "qq" ; + qqv:units = "1" ; + qqv:ancillary_variables = "my_av" ; + int64 axv(axv) ; + axv:units = "1" ; + axv:long_name = "x" ; + byte my_av(axv) ; + my_av:long_name = "qq status_flag" ; + my_av:flag_values = 1b, 2b ; + my_av:flag_meanings = "a b" ; + data: + axv = 11, 21, 31; + my_av = 1b, 1b, 2b; + } + """ + self.tmpdir = tempfile.mkdtemp() + cdl_path = os.path.join(self.tmpdir, "tst.cdl") + nc_path = os.path.join(self.tmpdir, "tst.nc") + # Write CDL string into a temporary CDL file. + with open(cdl_path, "w") as f_out: + f_out.write(ref_cdl) + # Use ncgen to convert this into an actual (temporary) netCDF file. + command = "ncgen -o {} {}".format(nc_path, cdl_path) + check_call(command, shell=True) + # Load with iris.fileformats.netcdf.load_cubes, and check expected content. + cubes = list(nc_load_cubes(nc_path)) + self.assertEqual(len(cubes), 1) + avs = cubes[0].ancillary_variables() + self.assertEqual(len(avs), 1) + expected = AncillaryVariable( + np.ma.array([1, 1, 2], dtype=np.int8), + long_name="qq status_flag", + var_name="my_av", + units="no_unit", + attributes={ + "flag_values": np.array([1, 2], dtype=np.int8), + "flag_meanings": "a b", + }, + ) + self.assertEqual(avs[0], expected) + def test_cell_measures(self): # Note: using a CDL string as a test data reference, rather than a binary file. ref_cdl = """ diff --git a/lib/iris/tests/unit/fileformats/pyke_rules/compiled_krb/fc_rules_cf_fc/test_build_auxiliary_coordinate.py b/lib/iris/tests/unit/fileformats/pyke_rules/compiled_krb/fc_rules_cf_fc/test_build_auxiliary_coordinate.py index 62daff146d..bd2dc9d6ee 100644 --- a/lib/iris/tests/unit/fileformats/pyke_rules/compiled_krb/fc_rules_cf_fc/test_build_auxiliary_coordinate.py +++ b/lib/iris/tests/unit/fileformats/pyke_rules/compiled_krb/fc_rules_cf_fc/test_build_auxiliary_coordinate.py @@ -91,7 +91,7 @@ def _get_per_test_bounds_var(_coord_unused): def _make_array_and_cf_data(cls, dimension_names): shape = tuple(cls.dim_names_lens[name] for name in dimension_names) - cf_data = mock.Mock(_FillValue=None) + cf_data = mock.MagicMock(_FillValue=None, spec=[]) cf_data.chunking = mock.MagicMock(return_value=shape) return np.zeros(shape), cf_data @@ -144,7 +144,7 @@ class TestDtype(tests.IrisTest): def setUp(self): # Create coordinate cf variables and pyke engine. points = np.arange(6).reshape(2, 3) - cf_data = mock.Mock(_FillValue=None) + cf_data = mock.MagicMock(_FillValue=None) cf_data.chunking = mock.MagicMock(return_value=points.shape) self.cf_coord_var = mock.Mock( @@ -218,7 +218,7 @@ def setUp(self): scale_factor=1, add_offset=0, cf_name='wibble', - cf_data=mock.MagicMock(chunking=mock.Mock(return_value=None)), + cf_data=mock.MagicMock(chunking=mock.Mock(return_value=None), spec=[]), standard_name=None, long_name='wibble', units='days since 1970-01-01', diff --git a/lib/iris/tests/unit/fileformats/pyke_rules/compiled_krb/fc_rules_cf_fc/test_build_cube_metadata.py b/lib/iris/tests/unit/fileformats/pyke_rules/compiled_krb/fc_rules_cf_fc/test_build_cube_metadata.py index 7f6ecb27c2..fc31092b58 100644 --- a/lib/iris/tests/unit/fileformats/pyke_rules/compiled_krb/fc_rules_cf_fc/test_build_cube_metadata.py +++ b/lib/iris/tests/unit/fileformats/pyke_rules/compiled_krb/fc_rules_cf_fc/test_build_cube_metadata.py @@ -28,7 +28,7 @@ def _make_engine(global_attributes=None, standard_name=None, long_name=None): cf_group = mock.Mock(global_attributes=global_attributes) - cf_var = mock.Mock( + cf_var = mock.MagicMock( cf_name='wibble', standard_name=standard_name, long_name=long_name, diff --git a/lib/iris/tests/unit/fileformats/pyke_rules/compiled_krb/fc_rules_cf_fc/test_build_dimension_coordinate.py b/lib/iris/tests/unit/fileformats/pyke_rules/compiled_krb/fc_rules_cf_fc/test_build_dimension_coordinate.py index d753636e70..eea2051eb6 100644 --- a/lib/iris/tests/unit/fileformats/pyke_rules/compiled_krb/fc_rules_cf_fc/test_build_dimension_coordinate.py +++ b/lib/iris/tests/unit/fileformats/pyke_rules/compiled_krb/fc_rules_cf_fc/test_build_dimension_coordinate.py @@ -75,6 +75,7 @@ def _set_cf_coord_var(self, points): self.cf_coord_var = mock.Mock( dimensions=('foo',), cf_name='wibble', + cf_data=mock.Mock(spec=[]), standard_name=None, long_name='wibble', units='days since 1970-01-01', @@ -226,6 +227,7 @@ def setUp(self): cf_name='wibble', standard_name=None, long_name='wibble', + cf_data=mock.Mock(spec=[]), units='m', shape=points.shape, dtype=points.dtype, @@ -332,11 +334,12 @@ def setUp(self): def _make_vars(self, points, bounds=None, units='degrees'): points = np.array(points) - self.cf_coord_var = mock.Mock( + self.cf_coord_var = mock.MagicMock( dimensions=('foo',), cf_name='wibble', standard_name=None, long_name='wibble', + cf_data=mock.Mock(spec=[]), units=units, shape=points.shape, dtype=points.dtype, @@ -435,6 +438,7 @@ def _make_vars(self, bounds): standard_name=None, long_name='wibble', units='degrees', + cf_data=mock.Mock(spec=[]), shape=(), dtype=points.dtype, __getitem__=lambda self, key: points[key]) diff --git a/lib/iris/tests/unit/fileformats/pyke_rules/compiled_krb/fc_rules_cf_fc/test_get_attr_units.py b/lib/iris/tests/unit/fileformats/pyke_rules/compiled_krb/fc_rules_cf_fc/test_get_attr_units.py index c5e36e8d8e..4df00fe209 100644 --- a/lib/iris/tests/unit/fileformats/pyke_rules/compiled_krb/fc_rules_cf_fc/test_get_attr_units.py +++ b/lib/iris/tests/unit/fileformats/pyke_rules/compiled_krb/fc_rules_cf_fc/test_get_attr_units.py @@ -29,8 +29,9 @@ def _make_cf_var(global_attributes=None): cf_group = mock.Mock(global_attributes=global_attributes) - cf_var = mock.Mock( + cf_var = mock.MagicMock( cf_name='sound_frequency', + cf_data=mock.Mock(spec=[]), standard_name=None, long_name=None, units=u'\u266b', diff --git a/requirements/core.txt b/requirements/core.txt index c3f5775d7e..c7adb469cc 100644 --- a/requirements/core.txt +++ b/requirements/core.txt @@ -6,9 +6,9 @@ cartopy>=0.12 #conda: proj4<6 cf-units>=2 -cftime +cftime<1.1 dask[array]>=2 #conda: dask>=2 -matplotlib +matplotlib<=3.1 netcdf4 -numpy>=1.14 +numpy<=1.17 scipy diff --git a/requirements/test.txt b/requirements/test.txt index 89358f7f76..89cd043fa1 100644 --- a/requirements/test.txt +++ b/requirements/test.txt @@ -3,7 +3,8 @@ black==19.10b0 #conda: black=19.10b0 filelock -imagehash>=4.0 +pillow<7 +imagehash nose pre-commit requests From e34eb20855bb81d3e013540e95dab3fc6a429a98 Mon Sep 17 00:00:00 2001 From: abooton Date: Wed, 10 Jun 2020 15:50:21 +0100 Subject: [PATCH 5/6] Resolve conflicts with requirements/test.txt to update the imagehash pin to imagehash>=4.0 (#3735) Co-authored-by: Patrick Peglar --- requirements/test.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements/test.txt b/requirements/test.txt index 89cd043fa1..0a0ebb1b8f 100644 --- a/requirements/test.txt +++ b/requirements/test.txt @@ -4,7 +4,7 @@ black==19.10b0 #conda: black=19.10b0 filelock pillow<7 -imagehash +imagehash>=4.0 nose pre-commit requests From b422cd8a074d9f306438ebcf2cd121674a204666 Mon Sep 17 00:00:00 2001 From: stephenworsley <49274989+stephenworsley@users.noreply.github.com> Date: Fri, 21 Aug 2020 15:32:27 +0100 Subject: [PATCH 6/6] [FB] [PI-3474] Add tests for the loading and saving of ancillary variables (#3799) --- .../TestNetCDFSave__ancillaries/flag.cdl | 46 +++++++++++++++++++ lib/iris/tests/test_netcdf.py | 28 +++++++++++ 2 files changed, 74 insertions(+) create mode 100644 lib/iris/tests/results/netcdf/TestNetCDFSave__ancillaries/flag.cdl diff --git a/lib/iris/tests/results/netcdf/TestNetCDFSave__ancillaries/flag.cdl b/lib/iris/tests/results/netcdf/TestNetCDFSave__ancillaries/flag.cdl new file mode 100644 index 0000000000..22ee23e2f6 --- /dev/null +++ b/lib/iris/tests/results/netcdf/TestNetCDFSave__ancillaries/flag.cdl @@ -0,0 +1,46 @@ +dimensions: + grid_latitude = 9 ; + grid_longitude = 11 ; + time = 7 ; +variables: + int64 air_potential_temperature(time, grid_latitude, grid_longitude) ; + air_potential_temperature:standard_name = "air_potential_temperature" ; + air_potential_temperature:units = "K" ; + air_potential_temperature:grid_mapping = "rotated_latitude_longitude" ; + air_potential_temperature:coordinates = "air_pressure forecast_period" ; + air_potential_temperature:ancillary_variables = "quality_flag" ; + int rotated_latitude_longitude ; + rotated_latitude_longitude:grid_mapping_name = "rotated_latitude_longitude" ; + rotated_latitude_longitude:longitude_of_prime_meridian = 0. ; + rotated_latitude_longitude:earth_radius = 6371229. ; + rotated_latitude_longitude:grid_north_pole_latitude = 37.5 ; + rotated_latitude_longitude:grid_north_pole_longitude = 177.5 ; + rotated_latitude_longitude:north_pole_grid_longitude = 0. ; + double time(time) ; + time:axis = "T" ; + time:units = "hours since 1970-01-01 00:00:00" ; + time:standard_name = "time" ; + time:calendar = "gregorian" ; + double grid_latitude(grid_latitude) ; + grid_latitude:axis = "Y" ; + grid_latitude:units = "degrees" ; + grid_latitude:standard_name = "grid_latitude" ; + double grid_longitude(grid_longitude) ; + grid_longitude:axis = "X" ; + grid_longitude:units = "degrees" ; + grid_longitude:standard_name = "grid_longitude" ; + double air_pressure ; + air_pressure:units = "Pa" ; + air_pressure:standard_name = "air_pressure" ; + double forecast_period(time) ; + forecast_period:units = "hours" ; + forecast_period:standard_name = "forecast_period" ; + byte quality_flag(time, grid_latitude, grid_longitude) ; + quality_flag:long_name = "quality_flag" ; + quality_flag:flag_meanings = "PASS FAIL MISSING" ; + quality_flag:flag_values = 1b, 2b, 9b ; + +// global attributes: + :source = "Iris test case" ; + :Conventions = "CF-1.7" ; +} diff --git a/lib/iris/tests/test_netcdf.py b/lib/iris/tests/test_netcdf.py index b012f6a89a..13a03a76c3 100644 --- a/lib/iris/tests/test_netcdf.py +++ b/lib/iris/tests/test_netcdf.py @@ -460,14 +460,21 @@ def test_default_units(self): variables: int64 qqv(ayv, axv) ; qqv:long_name = "qq" ; + qqv:ancillary_variables = "my_av" ; + qqv:cell_measures = "area: my_areas" ; int64 ayv(ayv) ; ayv:long_name = "y" ; int64 axv(axv) ; axv:units = "1" ; axv:long_name = "x" ; + double my_av(axv) ; + my_av:long_name = "refs" ; + double my_areas(ayv, axv) ; + my_areas:long_name = "areas" ; data: axv = 11, 12, 13; ayv = 21, 22; + my_areas = 110., 120., 130., 221., 231., 241.; } """ self.tmpdir = tempfile.mkdtemp() @@ -485,6 +492,12 @@ def test_default_units(self): self.assertEqual(cubes[0].units, as_unit("unknown")) self.assertEqual(cubes[0].coord("y").units, as_unit("unknown")) self.assertEqual(cubes[0].coord("x").units, as_unit(1)) + self.assertEqual( + cubes[0].ancillary_variable("refs").units, as_unit("unknown") + ) + self.assertEqual( + cubes[0].cell_measure("areas").units, as_unit("unknown") + ) def test_units(self): # Test exercising graceful cube and coordinate units loading. @@ -1263,6 +1276,21 @@ def test_aliases(self): iris.save([testcube_1, testcube_2], filename) self.assertCDL(filename) + def test_flag(self): + testcube = stock.realistic_3d() + flag = iris.coords.AncillaryVariable( + np.ones(testcube.shape, dtype=np.int8), + long_name="quality_flag", + attributes={ + "flag_meanings": "PASS FAIL MISSING", + "flag_values": np.array([1, 2, 9], dtype=np.int8), + }, + ) + testcube.add_ancillary_variable(flag, (0, 1, 2)) + with self.temp_filename(suffix=".nc") as filename: + iris.save(testcube, filename) + self.assertCDL(filename) + class TestNetCDF3SaveInteger(tests.IrisTest): def setUp(self):