diff --git a/docs/iris/src/whatsnew/contributions_3.0.0/incompatiblechange_2020-May-15_change_default_unit_loading.txt b/docs/iris/src/whatsnew/contributions_3.0.0/incompatiblechange_2020-May-15_change_default_unit_loading.txt new file mode 100644 index 0000000000..be048990f3 --- /dev/null +++ b/docs/iris/src/whatsnew/contributions_3.0.0/incompatiblechange_2020-May-15_change_default_unit_loading.txt @@ -0,0 +1 @@ +* When loading data from netcdf-CF files, where a variable has no "units" property, the corresponding Iris object will have "units='unknown'". Prior to Iris 3.0, these cases defaulted to "units='1'". \ No newline at end of file diff --git a/lib/iris/fileformats/_pyke_rules/fc_rules_cf.krb b/lib/iris/fileformats/_pyke_rules/fc_rules_cf.krb index 5ecfeb77b1..b26275f4db 100644 --- a/lib/iris/fileformats/_pyke_rules/fc_rules_cf.krb +++ b/lib/iris/fileformats/_pyke_rules/fc_rules_cf.krb @@ -1195,6 +1195,8 @@ fc_extras UD_UNITS_LON = ['degrees_east', 'degree_east', 'degree_e', 'degrees_e', 'degreee', 'degreese', 'degrees', 'degrees east', 'degree east', 'degree e', 'degrees e'] + UNKNOWN_UNIT_STRING = "?" + NO_UNIT_STRING = "-" # # CF Dimensionless Vertical Coordinates @@ -1651,9 +1653,9 @@ fc_extras ################################################################################ def get_attr_units(cf_var, attributes): - attr_units = getattr(cf_var, CF_ATTR_UNITS, cf_units._UNIT_DIMENSIONLESS) + attr_units = getattr(cf_var, CF_ATTR_UNITS, UNKNOWN_UNIT_STRING) if not attr_units: - attr_units = '1' + attr_units = UNKNOWN_UNIT_STRING # Sanitise lat/lon units. if attr_units in UD_UNITS_LAT or attr_units in UD_UNITS_LON: @@ -1668,10 +1670,10 @@ fc_extras cf_var.cf_name, attr_units) warnings.warn(msg) attributes['invalid_units'] = attr_units - attr_units = cf_units._UNKNOWN_UNIT_STRING + attr_units = UNKNOWN_UNIT_STRING if np.issubdtype(cf_var.dtype, np.str_): - attr_units = cf_units._NO_UNIT_STRING + attr_units = NO_UNIT_STRING # Get any assoicated calendar for a time reference coordinate. if cf_units.as_unit(attr_units).is_time_reference(): diff --git a/lib/iris/tests/results/netcdf/int64_auxiliary_coord_netcdf3.cml b/lib/iris/tests/results/netcdf/int64_auxiliary_coord_netcdf3.cml index 39cb8f2950..e48cf41d2a 100644 --- a/lib/iris/tests/results/netcdf/int64_auxiliary_coord_netcdf3.cml +++ b/lib/iris/tests/results/netcdf/int64_auxiliary_coord_netcdf3.cml @@ -6,7 +6,7 @@ - + diff --git a/lib/iris/tests/results/netcdf/int64_dimension_coord_netcdf3.cml b/lib/iris/tests/results/netcdf/int64_dimension_coord_netcdf3.cml index 1c59fc947e..78fec459e9 100644 --- a/lib/iris/tests/results/netcdf/int64_dimension_coord_netcdf3.cml +++ b/lib/iris/tests/results/netcdf/int64_dimension_coord_netcdf3.cml @@ -6,7 +6,7 @@ - + diff --git a/lib/iris/tests/results/netcdf/netcdf_cell_methods.cml b/lib/iris/tests/results/netcdf/netcdf_cell_methods.cml index 8dd0e43b71..ca4a0eb017 100644 --- a/lib/iris/tests/results/netcdf/netcdf_cell_methods.cml +++ b/lib/iris/tests/results/netcdf/netcdf_cell_methods.cml @@ -1,6 +1,6 @@ - + @@ -20,7 +20,7 @@ - + @@ -41,7 +41,7 @@ - + @@ -66,7 +66,7 @@ - + @@ -89,7 +89,7 @@ - + @@ -112,7 +112,7 @@ - + @@ -131,7 +131,7 @@ - + @@ -150,7 +150,7 @@ - + @@ -170,7 +170,7 @@ - + @@ -190,7 +190,7 @@ - + @@ -213,7 +213,7 @@ - + @@ -232,7 +232,7 @@ - + @@ -252,7 +252,7 @@ - + @@ -272,7 +272,7 @@ - + @@ -295,7 +295,7 @@ - + @@ -320,7 +320,7 @@ - + @@ -333,7 +333,7 @@ - + @@ -346,7 +346,7 @@ - + @@ -359,7 +359,7 @@ - + @@ -372,7 +372,7 @@ - + @@ -385,7 +385,7 @@ - + @@ -404,7 +404,7 @@ - + @@ -424,7 +424,7 @@ - + @@ -447,7 +447,7 @@ - + @@ -460,7 +460,7 @@ - + @@ -473,7 +473,7 @@ - + @@ -486,7 +486,7 @@ - + @@ -499,7 +499,7 @@ - + diff --git a/lib/iris/tests/results/netcdf/netcdf_global_xyzt_gems.cml b/lib/iris/tests/results/netcdf/netcdf_global_xyzt_gems.cml index 27d4569236..ac41f4a8b8 100644 --- a/lib/iris/tests/results/netcdf/netcdf_global_xyzt_gems.cml +++ b/lib/iris/tests/results/netcdf/netcdf_global_xyzt_gems.cml @@ -13,11 +13,11 @@ - + 51, 52, 53, 54, 55, 56, 57, 58, 59, 60]" shape="(60,)" units="Unit('unknown')" value_type="int32" var_name="levelist"/> @@ -39,11 +39,11 @@ - + 51, 52, 53, 54, 55, 56, 57, 58, 59, 60]" shape="(60,)" units="Unit('unknown')" value_type="int32" var_name="levelist"/> diff --git a/lib/iris/tests/results/netcdf/netcdf_global_xyzt_gems_iter_0.cml b/lib/iris/tests/results/netcdf/netcdf_global_xyzt_gems_iter_0.cml index d677191beb..4234b5cc84 100644 --- a/lib/iris/tests/results/netcdf/netcdf_global_xyzt_gems_iter_0.cml +++ b/lib/iris/tests/results/netcdf/netcdf_global_xyzt_gems_iter_0.cml @@ -13,11 +13,11 @@ - + 51, 52, 53, 54, 55, 56, 57, 58, 59, 60]" shape="(60,)" units="Unit('unknown')" value_type="int32" var_name="levelist"/> diff --git a/lib/iris/tests/results/netcdf/netcdf_global_xyzt_gems_iter_1.cml b/lib/iris/tests/results/netcdf/netcdf_global_xyzt_gems_iter_1.cml index 775f480c66..17d87a0190 100644 --- a/lib/iris/tests/results/netcdf/netcdf_global_xyzt_gems_iter_1.cml +++ b/lib/iris/tests/results/netcdf/netcdf_global_xyzt_gems_iter_1.cml @@ -13,11 +13,11 @@ - + 51, 52, 53, 54, 55, 56, 57, 58, 59, 60]" shape="(60,)" units="Unit('unknown')" value_type="int32" var_name="levelist"/> diff --git a/lib/iris/tests/results/netcdf/uint32_auxiliary_coord_netcdf3.cml b/lib/iris/tests/results/netcdf/uint32_auxiliary_coord_netcdf3.cml index 39cb8f2950..e48cf41d2a 100644 --- a/lib/iris/tests/results/netcdf/uint32_auxiliary_coord_netcdf3.cml +++ b/lib/iris/tests/results/netcdf/uint32_auxiliary_coord_netcdf3.cml @@ -6,7 +6,7 @@ - + diff --git a/lib/iris/tests/results/netcdf/uint32_dimension_coord_netcdf3.cml b/lib/iris/tests/results/netcdf/uint32_dimension_coord_netcdf3.cml index 1c59fc947e..78fec459e9 100644 --- a/lib/iris/tests/results/netcdf/uint32_dimension_coord_netcdf3.cml +++ b/lib/iris/tests/results/netcdf/uint32_dimension_coord_netcdf3.cml @@ -6,7 +6,7 @@ - + diff --git a/lib/iris/tests/results/usecases/pp_to_cf_conversion/from_netcdf/002000000000.44.101.131200.1920.09.01.00.00.b_0.cml b/lib/iris/tests/results/usecases/pp_to_cf_conversion/from_netcdf/002000000000.44.101.131200.1920.09.01.00.00.b_0.cml index 3ea688d1fa..0bf359e9c4 100644 --- a/lib/iris/tests/results/usecases/pp_to_cf_conversion/from_netcdf/002000000000.44.101.131200.1920.09.01.00.00.b_0.cml +++ b/lib/iris/tests/results/usecases/pp_to_cf_conversion/from_netcdf/002000000000.44.101.131200.1920.09.01.00.00.b_0.cml @@ -1,6 +1,6 @@ - + diff --git a/lib/iris/tests/results/usecases/pp_to_cf_conversion/from_netcdf/008000000000.44.101.000128.1890.09.01.00.00.b_0.cml b/lib/iris/tests/results/usecases/pp_to_cf_conversion/from_netcdf/008000000000.44.101.000128.1890.09.01.00.00.b_0.cml index 829c7ce38e..e5cec55565 100644 --- a/lib/iris/tests/results/usecases/pp_to_cf_conversion/from_netcdf/008000000000.44.101.000128.1890.09.01.00.00.b_0.cml +++ b/lib/iris/tests/results/usecases/pp_to_cf_conversion/from_netcdf/008000000000.44.101.000128.1890.09.01.00.00.b_0.cml @@ -1,6 +1,6 @@ - + diff --git a/lib/iris/tests/results/usecases/pp_to_cf_conversion/from_netcdf/st0fc699.b_0.cml b/lib/iris/tests/results/usecases/pp_to_cf_conversion/from_netcdf/st0fc699.b_0.cml index 4f84609832..b484ebb305 100644 --- a/lib/iris/tests/results/usecases/pp_to_cf_conversion/from_netcdf/st0fc699.b_0.cml +++ b/lib/iris/tests/results/usecases/pp_to_cf_conversion/from_netcdf/st0fc699.b_0.cml @@ -1,6 +1,6 @@ - + diff --git a/lib/iris/tests/results/usecases/pp_to_cf_conversion/from_netcdf/st0fc942.b_0.cml b/lib/iris/tests/results/usecases/pp_to_cf_conversion/from_netcdf/st0fc942.b_0.cml index caafa5845c..c594c748cd 100644 --- a/lib/iris/tests/results/usecases/pp_to_cf_conversion/from_netcdf/st0fc942.b_0.cml +++ b/lib/iris/tests/results/usecases/pp_to_cf_conversion/from_netcdf/st0fc942.b_0.cml @@ -1,6 +1,6 @@ - + diff --git a/lib/iris/tests/test_netcdf.py b/lib/iris/tests/test_netcdf.py index a550e1ed4b..b43fdf02ae 100644 --- a/lib/iris/tests/test_netcdf.py +++ b/lib/iris/tests/test_netcdf.py @@ -16,9 +16,11 @@ import os.path import shutil import stat +from subprocess import check_call import tempfile from unittest import mock +from cf_units import as_unit import netCDF4 as nc import numpy as np import numpy.ma as ma @@ -27,6 +29,7 @@ import iris.analysis.trajectory import iris.fileformats._pyke_rules.compiled_krb.fc_rules_cf_fc as pyke_rules import iris.fileformats.netcdf +from iris.fileformats.netcdf import load_cubes as nc_load_cubes import iris.std_names import iris.util import iris.coord_systems as icoord_systems @@ -292,6 +295,42 @@ def test_deferred_loading(self): cube[0][(0, 2), (1, 3)], ("netcdf", "netcdf_deferred_mix_1.cml") ) + def test_default_units(self): + # Note: using a CDL string as a test data reference, rather than a binary file. + ref_cdl = """ + netcdf cm_attr { + dimensions: + axv = 3 ; + ayv = 2 ; + variables: + int64 qqv(ayv, axv) ; + qqv:long_name = "qq" ; + int64 ayv(ayv) ; + ayv:long_name = "y" ; + int64 axv(axv) ; + axv:units = "1" ; + axv:long_name = "x" ; + data: + axv = 11, 12, 13; + ayv = 21, 22; + } + """ + self.tmpdir = tempfile.mkdtemp() + cdl_path = os.path.join(self.tmpdir, "tst.cdl") + nc_path = os.path.join(self.tmpdir, "tst.nc") + # Write CDL string into a temporary CDL file. + with open(cdl_path, "w") as f_out: + f_out.write(ref_cdl) + # Use ncgen to convert this into an actual (temporary) netCDF file. + command = "ncgen -o {} {}".format(nc_path, cdl_path) + check_call(command, shell=True) + # Load with iris.fileformats.netcdf.load_cubes, and check expected content. + cubes = list(nc_load_cubes(nc_path)) + self.assertEqual(len(cubes), 1) + self.assertEqual(cubes[0].units, as_unit("unknown")) + self.assertEqual(cubes[0].coord("y").units, as_unit("unknown")) + self.assertEqual(cubes[0].coord("x").units, as_unit(1)) + def test_units(self): # Test exercising graceful cube and coordinate units loading. cube0, cube1 = sorted(