From 6a8367c9d7ffb0eef038c13e13a1a4c6ec704c7b Mon Sep 17 00:00:00 2001 From: Lukas Pilz Date: Mon, 24 Oct 2022 22:48:30 +0200 Subject: [PATCH 1/8] Amended unit harmonization --- tests/test_postprocess.py | 7 ---- xwrf/config.py | 4 ++ xwrf/config.yaml | 27 ------------- xwrf/postprocess.py | 14 ------- xwrf/unit_harmonization_map.yaml | 68 ++++++++++++++++++++++++++++++++ 5 files changed, 72 insertions(+), 48 deletions(-) create mode 100644 xwrf/unit_harmonization_map.yaml diff --git a/tests/test_postprocess.py b/tests/test_postprocess.py index 3904bdac..83e5c8d7 100644 --- a/tests/test_postprocess.py +++ b/tests/test_postprocess.py @@ -144,13 +144,6 @@ def test_assign_coord_to_dim_of_different_name_keyerror(sample_dataset): xr.testing.assert_equal(ds, dataset) -@pytest.mark.parametrize('sample_dataset', ['lambert_conformal'], indirect=True) -@pytest.mark.parametrize('variable,bracket', (('QNRAIN', '('), ('NOAHRES', '{'))) -def test_clean_brackets_from_units(sample_dataset, variable, bracket): - ds = sample_dataset.pipe(xwrf.postprocess._clean_brackets_from_units) - assert bracket not in ds[variable].attrs['units'] - - @pytest.mark.parametrize('sample_dataset', ['lambert_conformal'], indirect=True) def test_calc_base_diagnostics(sample_dataset): subset = ( diff --git a/xwrf/config.py b/xwrf/config.py index 433b2164..f490e732 100644 --- a/xwrf/config.py +++ b/xwrf/config.py @@ -8,5 +8,9 @@ with open(fn) as f: defaults = yaml.safe_load(f) +unit_harm_fn = os.path.join(os.path.dirname(__file__), 'unit_harmonization_map.yaml') +with open(unit_harm_fn) as f: + defaults['unit_harmonization_map'] = yaml.safe_load(f) + config = Config('xwrf', defaults=[defaults]) config.ensure_file(fn, comment=False) diff --git a/xwrf/config.yaml b/xwrf/config.yaml index e10cc9b2..43fa67a9 100644 --- a/xwrf/config.yaml +++ b/xwrf/config.yaml @@ -38,33 +38,6 @@ brackets_to_clean_from_units: - '(' - ')' -unit_harmonization_map: - kelvin: - - Kelvin - '1': - - fraction - - Dimensionless - invalid: - - '-' - - flag - - '0/1 Flag' - - whoknows - - category - - none - - dBZ - - dbZ - meters: - - 'meters MSL' - g/m2/s: - - 'g/m2/s CO2' - umol/m2/s: - - 'umol co2/m2/s' - J/m**3/kelvin: - - 'J/m**3/Kelvin' - W/m/kelvin: - - 'W/m-K' - - 'W/m/Kelvin' - cf_attribute_map: south_north: units: m diff --git a/xwrf/postprocess.py b/xwrf/postprocess.py index 5280b3e2..5af357b3 100644 --- a/xwrf/postprocess.py +++ b/xwrf/postprocess.py @@ -1,6 +1,5 @@ from __future__ import annotations # noqa: F401 -import re import warnings import numpy as np @@ -37,23 +36,10 @@ def _decode_times(ds: xr.Dataset) -> xr.Dataset: return ds -def _clean_brackets_from_units(ds: xr.Dataset) -> xr.Dataset: - """ - Cleans brackets from units attributes - """ - sep = '\\' - regex = re.compile(f'[{sep.join(config.get("brackets_to_clean_from_units"))}]') - for var in ds.variables: - if 'units' in ds[var].attrs: - ds[var].attrs['units'] = regex.sub('', ds[var].attrs['units']) - return ds - - def _make_units_pint_friendly(ds: xr.Dataset) -> xr.Dataset: """ Harmonizes awkward WRF units into pint-friendly ones """ - ds = _clean_brackets_from_units(ds) # We have to invert the mapping from "new_unit -> wrf_units" to "wrf_unit -> new_unit" wrf_units_map = { v: k for (k, val_list) in config.get('unit_harmonization_map').items() for v in val_list diff --git a/xwrf/unit_harmonization_map.yaml b/xwrf/unit_harmonization_map.yaml new file mode 100644 index 00000000..3e1df5e4 --- /dev/null +++ b/xwrf/unit_harmonization_map.yaml @@ -0,0 +1,68 @@ +1e-6: + - ppmv + - ppm +1e-6 m2 s-1 K kg-1: + - pvu +J m-2: + - Joules m^-2 +J m-3 K-1: + - J/m**3/Kelvin +W m-1 K-1: + - W/m-K + - W/m/Kelvin +W m-2: + - W m{-2} +cm-3: + - '#/cm-3' +'1': + - Dimensionless + - fraction + - numerical value + - dimensionless + - area/area + - '#' + - '0/1 Flag' +g kg-1: + - g_smoke/kg_air +g m-2 s-1: + - g/m2/s CO2 + - g/m2/s C +h-1: + - flash origin density + - flash column-1 + - Source column-1 +invalid: + - '?' + - none + - '-' + - flag + - category + - ~? + - dBz + - dbZ + - dBZ +kJ m-2: + - KJ/m2 +kg kg-1: + - '# kg kg-1' + - Kg Kg-1 +kg kg-1 s-1: + - kg kg_wet-1 s-1 +kg-1: + - '#/kg-dryair' + - '# kg-1' + - '# kg(-1)' + - ' kg(-1)' +m: + - m asl + - m agl +m-3: + - C m-3 +min: + - minutes since simulation start +min-1: + - min{-1} +ug/kg: + - ug/kg-dryair +umol m-2 s-1: + - umol co2/m2/s From 39df05480b1b97588864e6179fd9995a8d9a87f3 Mon Sep 17 00:00:00 2001 From: Lukas Pilz Date: Mon, 24 Oct 2022 23:01:30 +0200 Subject: [PATCH 2/8] Updated harmonization map --- xwrf/unit_harmonization_map.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/xwrf/unit_harmonization_map.yaml b/xwrf/unit_harmonization_map.yaml index 3e1df5e4..532cb005 100644 --- a/xwrf/unit_harmonization_map.yaml +++ b/xwrf/unit_harmonization_map.yaml @@ -41,6 +41,8 @@ invalid: - dBz - dbZ - dBZ +kelvin: + - Kelvin kJ m-2: - KJ/m2 kg kg-1: From 5468b266f9754d301ce6ecf2a86daf7dcfdf2782 Mon Sep 17 00:00:00 2001 From: Lukas Pilz Date: Mon, 24 Oct 2022 23:06:06 +0200 Subject: [PATCH 3/8] Updated harmonization map --- xwrf/unit_harmonization_map.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/xwrf/unit_harmonization_map.yaml b/xwrf/unit_harmonization_map.yaml index 532cb005..a74f0184 100644 --- a/xwrf/unit_harmonization_map.yaml +++ b/xwrf/unit_harmonization_map.yaml @@ -43,6 +43,8 @@ invalid: - dBZ kelvin: - Kelvin +meters: + - 'meters MSL' kJ m-2: - KJ/m2 kg kg-1: From f3f567ccc53367f836b5510d263ecb3a37bc6b0d Mon Sep 17 00:00:00 2001 From: Lukas Pilz Date: Mon, 24 Oct 2022 23:10:36 +0200 Subject: [PATCH 4/8] Sorry, overlooked the entry --- xwrf/unit_harmonization_map.yaml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/xwrf/unit_harmonization_map.yaml b/xwrf/unit_harmonization_map.yaml index a74f0184..bd03dc67 100644 --- a/xwrf/unit_harmonization_map.yaml +++ b/xwrf/unit_harmonization_map.yaml @@ -43,8 +43,6 @@ invalid: - dBZ kelvin: - Kelvin -meters: - - 'meters MSL' kJ m-2: - KJ/m2 kg kg-1: @@ -60,6 +58,7 @@ kg-1: m: - m asl - m agl + - 'meters MSL' m-3: - C m-3 min: From 1dc438b707e6a02ec8e9614105e95cc6808da86a Mon Sep 17 00:00:00 2001 From: Lukas Pilz Date: Tue, 25 Oct 2022 00:26:41 +0200 Subject: [PATCH 5/8] Moved 0/1 Flag to correct translation --- xwrf/unit_harmonization_map.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/xwrf/unit_harmonization_map.yaml b/xwrf/unit_harmonization_map.yaml index bd03dc67..1093e8e5 100644 --- a/xwrf/unit_harmonization_map.yaml +++ b/xwrf/unit_harmonization_map.yaml @@ -21,7 +21,6 @@ cm-3: - dimensionless - area/area - '#' - - '0/1 Flag' g kg-1: - g_smoke/kg_air g m-2 s-1: @@ -41,6 +40,7 @@ invalid: - dBz - dbZ - dBZ + - '0/1 Flag' kelvin: - Kelvin kJ m-2: From 898a39f61a395bef98379e3c489d45d6c06656b4 Mon Sep 17 00:00:00 2001 From: Lukas Pilz Date: Tue, 25 Oct 2022 00:35:47 +0200 Subject: [PATCH 6/8] Removed wrong conversion --- xwrf/unit_harmonization_map.yaml | 2 -- 1 file changed, 2 deletions(-) diff --git a/xwrf/unit_harmonization_map.yaml b/xwrf/unit_harmonization_map.yaml index 1093e8e5..88794a1a 100644 --- a/xwrf/unit_harmonization_map.yaml +++ b/xwrf/unit_harmonization_map.yaml @@ -59,8 +59,6 @@ m: - m asl - m agl - 'meters MSL' -m-3: - - C m-3 min: - minutes since simulation start min-1: From 16c0ea139180852521e45c48d30adfeadf7ae6fb Mon Sep 17 00:00:00 2001 From: Lukas Pilz Date: Wed, 28 Dec 2022 19:21:53 +0100 Subject: [PATCH 7/8] Added test for bracketed units --- tests/test_postprocess.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tests/test_postprocess.py b/tests/test_postprocess.py index 83e5c8d7..53ebac56 100644 --- a/tests/test_postprocess.py +++ b/tests/test_postprocess.py @@ -144,6 +144,12 @@ def test_assign_coord_to_dim_of_different_name_keyerror(sample_dataset): xr.testing.assert_equal(ds, dataset) +@pytest.mark.parametrize('sample_dataset', ['lambert_conformal'], indirect=True) +def test_bracket_units_transl(sample_dataset): + assert {'QNRAIN', 'NOAHRES'}.issubset(set(sample_dataset.variables)) + xwrf.postprocess._make_units_pint_friendly(sample_dataset) + + @pytest.mark.parametrize('sample_dataset', ['lambert_conformal'], indirect=True) def test_calc_base_diagnostics(sample_dataset): subset = ( From 22a7e3b124864d94b15d1cfe6d8b62f8ffbd421d Mon Sep 17 00:00:00 2001 From: Lukas Pilz Date: Wed, 28 Dec 2022 19:26:25 +0100 Subject: [PATCH 8/8] Added uncertain translations --- xwrf/unit_harmonization_map.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/xwrf/unit_harmonization_map.yaml b/xwrf/unit_harmonization_map.yaml index 88794a1a..5b994a3b 100644 --- a/xwrf/unit_harmonization_map.yaml +++ b/xwrf/unit_harmonization_map.yaml @@ -9,6 +9,7 @@ J m-3 K-1: - J/m**3/Kelvin W m-1 K-1: - W/m-K + - W/m.K - W/m/Kelvin W m-2: - W m{-2} @@ -59,6 +60,8 @@ m: - m asl - m agl - 'meters MSL' +m s-1: + - 'm/s * area' min: - minutes since simulation start min-1: