CDAT Migration Phase 2: Refactor core utilities and lat_lon set

.pre-commit-config.yaml

@@ -4,14 +4,14 @@ fail_fast: true
 
 repos:
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.3.0
+    rev: v4.4.0
     hooks:
       - id: trailing-whitespace
       - id: end-of-file-fixer
       - id: check-yaml
 
   - repo: https://github.com/psf/black
-    rev: 22.10.0
+    rev: 23.9.1
     hooks:
       - id: black
 
@@ -23,15 +23,16 @@ repos:
   # Need to use flake8 GitHub mirror due to CentOS git issue with GitLab
   # https://github.com/pre-commit/pre-commit/issues/1206
   - repo: https://github.com/pycqa/flake8
-    rev: 6.0.0
+    rev: 6.1.0
     hooks:
       - id: flake8
         args: ["--config=setup.cfg"]
-        additional_dependencies: [flake8-isort]
+        additional_dependencies: [flake8-isort==6.1.0]
 
   - repo: https://github.com/pre-commit/mirrors-mypy
-    rev: v0.991
+    rev: v1.5.1
     hooks:
       - id: mypy
-        args: ["--config=setup.cfg"]
-        additional_dependencies: [types-pyYAML==6.0.12.6]
+        args: ["--config=pyproject.toml"]
+        additional_dependencies:
+          [dask, numpy>=1.23.0, xarray>=2023.3.0, types-PyYAML]

.vscode/e3sm_diags.code-workspace

@@ -8,6 +8,9 @@
       "path": ".."
     }
   ],
+  // ===========================
+  // VS Code Workspace Settings.
+  // ===========================
   "settings": {
     // ===================
     // Editor settings
@@ -22,17 +25,14 @@
       "editor.rulers": [80, 88, 120],
       "editor.wordWrap": "wordWrapColumn",
       "editor.wordWrapColumn": 120,
-      "editor.defaultFormatter": "ms-python.python"
+      "editor.defaultFormatter": "ms-python.black-formatter"
     },
     // Code Formatting and Linting
     // ---------------------------
-    "python.formatting.provider": "black",
-    "python.linting.flake8Enabled": true,
-    "python.linting.flake8Args": ["--config=setup.cfg"],
+    "flake8.args": ["--config=setup.cfg"],
     // Type checking
     // ---------------------------
-    "python.linting.mypyEnabled": true,
-    "python.linting.mypyArgs": ["--config=setup.cfg"],
+    "mypy-type-checker.args": ["--config=pyproject.toml"],
     // Testing
     // ---------------------------
     // NOTE: Debugger doesn't work if pytest-cov is enabled, so set "--no-cov"
@@ -49,5 +49,24 @@
       "editor.wordWrap": "wordWrapColumn",
       "editor.wordWrapColumn": 120
     }
+  },
+  // =====================================
+  // VS Code Python Debugger Configuration
+  // =====================================
+  "launch": {
+    "version": "0.2.0",
+    "configurations": [
+      {
+        "name": "Python: Current File",
+        "type": "python",
+        "request": "launch",
+        "program": "${file}",
+        "console": "integratedTerminal",
+        "justMyCode": true,
+        "env": {
+          "PYTHONPATH": "${workspaceFolder}"
+        }
+      }
+    ]
   }
 }

auxiliary_tools/cdat_regression_test.py

@@ -0,0 +1,171 @@
+# %%
+"""
+This script checks for regressions between the refactored and `main` branches
+of a diagnostic set.
+
+How it works
+------------
+  It compares the absolute and relative differences between two sets of
+  `.json` files in two separate directories, one for the refactored code
+  and the other for the `main` branch. This script will generate an Excel file
+  containing:
+    1. The raw metrics by each branch for each variable.
+    2. The absolute and relative differences of each variable between branches.
+    3. The highest relative differences (threshold > 2% difference)
+
+How to use
+-----------
+    1. mamba env create -f conda/dev-yml -n e3sm_diags_dev_<gh-issue-#>
+    2. mamba activate e3sm_diags_dev_<gh-issue-#>
+    3. Update `DEV_PATH` and `PROD_PATH` in `/auxiliary_tools/cdat_regression_test.py`
+    4. python auxiliary_tools/cdat_regression_test.py
+    5. Excel file generated in `/auxiliary_tools`
+
+Tips
+-----------
+Relative differences should be taken into consideration moreso than absolute
+differences.
+  - Relative differences show the scale using a percentage unit.
+  - Absolute differences is just a raw number that doesn't factor in
+    floating point size (e.g., 100.00 vs. 0.0001), which can be misleading.
+"""
+import glob
+import logging
+import os
+import time
+from typing import List
+
+import pandas as pd
+
+log_format = (
+    "%(asctime)s [%(levelname)s]: %(filename)s(%(funcName)s:%(lineno)s) >> %(message)s"
+)
+logging.basicConfig(format=log_format, filemode="w", level=logging.INFO)
+logger = logging.getLogger(__name__)
+
+# TODO: Update DEV_RESULTS and PROD_RESULTS.
+# ------------------------------------------------------------------------------
+DEV_PATH = "/global/cfs/cdirs/e3sm/www/vo13/examples_658/ex1_modTS_vs_modTS_3years/lat_lon/model_vs_model"
+PRO_PATH = "/global/cfs/cdirs/e3sm/www/vo13/examples/ex1_modTS_vs_modTS_3years/lat_lon/model_vs_model"
+# ------------------------------------------------------------------------------
+
+if not os.path.exists(DEV_PATH):
+    raise ValueError(f"DEV_RESULTS path does not exist ({DEV_PATH})")
+if not os.path.exists(PRO_PATH):
+    raise ValueError(f"PROD_RESULTS path does not exist ({PRO_PATH})")
+
+DEV_GLOB = sorted(glob.glob(DEV_PATH + "/*.json"))
+PROD_GLOB = sorted(glob.glob(PRO_PATH + "/*.json"))
+
+TIME_STR = time.strftime("%Y%m%d-%H%M%S")
+EXCEL_FILENAME = f"{TIME_STR}-metrics-diffs.xlsx"
+
+
+def get_metrics(filepaths: List[str]) -> pd.DataFrame:
+    """Get the metrics using a glob of `.json` metric files in a directory.
+
+    Parameters
+    ----------
+    filepaths : List[str]
+        The filepaths for metrics `.json` files.
+
+    Returns
+    -------
+    pd.DataFrame
+        The DataFrame containing the metrics for all of the variables in
+        the results directory.
+    """
+    metrics = []
+
+    for filepath in filepaths:
+        df = pd.read_json(filepath)
+
+        filename = filepath.split("/")[-1]
+        var_key = filename.split("-")[1]
+
+        # Add the variable key to the MultiIndex and update the index
+        # before stacking to make the DataFrame easier to parse.
+        multiindex = pd.MultiIndex.from_product([[var_key], [*df.index]])
+        df = df.set_index(multiindex)
+        df.stack()
+
+        metrics.append(df)
+
+    df_final = pd.concat(metrics)
+
+    # Reorder columns and drop "unit" column (string dtype breaks Pandas
+    # arithmetic).
+    df_final = df_final[["test", "ref", "test_regrid", "ref_regrid", "diff", "misc"]]
+
+    return df_final
+
+
+def get_diffs(df_a: pd.DataFrame, df_b: pd.DataFrame) -> pd.DataFrame:
+    """The metrics differences between two DataFrames.
+
+    Parameters
+    ----------
+    df_a : pd.DataFrame
+        The first DataFrame representing "actual" results (aka development).
+    df_b : pd.DataFrame
+        The second DataFrame representing "reference" results (aka production).
+
+    Returns
+    -------
+    pd.DataFrame
+        The DataFrame containing absolute and relative differences between
+        the metrics DataFrames.
+    """
+    #  Absolute difference: abs(actual - reference)
+    df_abs = abs(df_a - df_b)
+    df_abs = df_abs.add_suffix("_abs")
+
+    # Relative difference: abs(actual - reference) / abs(actual)
+    df_rel = abs(df_a - df_b) / abs(df_a)
+    df_rel = df_rel.add_suffix("_rel")
+
+    # Combine both DataFrames
+    df_final = pd.concat([df_abs, df_rel], axis=1, join="outer")
+
+    return df_final
+
+
+# %% Get the metrics DataFrames.
+df_dev = get_metrics(DEV_GLOB)
+df_prod = get_metrics(PROD_GLOB)
+
+# %% Combine metrics DataFrames.
+df_dev_pref = df_dev.add_prefix("dev_")
+df_prod_pref = df_prod.add_prefix("prod_")
+df_metrics = pd.concat([df_dev_pref, df_prod_pref], axis=1, join="outer")
+#%%
+# Sort the columns
+df_metrics = df_metrics[
+    [
+        "dev_test",
+        "prod_test",
+        "dev_ref",
+        "prod_ref",
+        "dev_test_regrid",
+        "prod_test_regrid",
+        "dev_ref_regrid",
+        "prod_ref_regrid",
+        "dev_diff",
+        "prod_diff",
+        "dev_misc",
+        "prod_misc",
+    ]
+]
+
+# %% Get differences between metrics.
+df_diffs = get_diffs(df_dev, df_prod)
+
+
+#%%
+with pd.ExcelWriter(EXCEL_FILENAME) as writer:
+    df_metrics.to_excel(writer, sheet_name="metrics")
+    df_diffs.to_excel(writer, sheet_name="metric_diffs")
+
+
+# %% Only get the metrics where the absolute and relative differences are
+# greater than a specific threshold (>1%)

conda-env/ci.yml

@@ -26,6 +26,8 @@ dependencies:
   - numpy >=1.23.0
   - shapely >=2.0.0,<3.0.0
   - xarray >=2023.02.0
+  - xcdat >=0.5.0
+  - xesmf >=0.7.0
   # Testing
   # ==================
   - scipy

conda-env/dev-nompi.yml

@@ -0,0 +1,61 @@
+# Conda development environment for testing local source code changes to `e3sm_diags` before merging them to production (`master` branch).
+# This version contains the no MPI version of `esmf` as a workaround for allowing VS Code's testing API to work.
+# The MPI version of `esmf` is usually installed by default, but it breaks VS Code's testing API because it throws a mysterious
+# `yaksa` warning.
+# More info: https://github.com/E3SM-Project/e3sm_diags/issues/737
+name: e3sm_diags_dev_nompi
+channels:
+  - conda-forge/label/xcdat_dev
+  - conda-forge
+  - defaults
+dependencies:
+  # Base
+  # =================
+  - python >=3.9
+  - pip
+  - beautifulsoup4
+  - cartopy >=0.17.0
+  - cartopy_offlinedata
+  - cdp 1.7.0
+  - cdms2 3.1.5
+  - cdutil 8.2.1
+  - dask
+  - esmf >=8.4.0 nompi*
+  - esmpy >=8.4.0
+  - genutil 8.2.1
+  - lxml
+  - mache >=0.15.0
+  - matplotlib-base
+  - netcdf4
+  - numpy >=1.23.0
+  - shapely >=2.0.0,<3.0.0
+  - xarray >=2023.02.0
+  - xskillscore >=0.0.20
+  - xcdat==0.6.0rc1
+  - xesmf >=0.7.0
+  # Testing
+  # =======================
+  - scipy
+  - pytest
+  - pytest-cov
+  # Documentation
+  # =======================
+  - sphinx
+  - sphinx_rtd_theme
+  - sphinx-multiversion
+  # Quality Assurance Tools
+  # =======================
+  # Run `pre-commit autoupdate` to get the latest pinned versions of 'rev' in
+  # `.pre-commit.config.yaml`, then update the pinned versions here.
+  - black=23.9.1
+  - flake8=6.1.0
+  - flake8-isort=6.1.0
+  - isort=5.12.0
+  - mypy=1.5.1
+  - pre-commit >=3.0.0
+  - types-PyYAML >=6.0.0
+  # Developer Tools
+  # =======================
+  - tbump=6.9.0
+  - ipykernel
+prefix: /opt/miniconda3/envs/e3sm_diags_dev

conda-env/dev.yml

@@ -1,11 +1,12 @@
 # Conda development environment for testing local source code changes to `e3sm_diags` before merging them to production (`master` branch).
 name: e3sm_diags_dev
 channels:
+  - conda-forge/label/xcdat_dev
   - conda-forge
   - defaults
 dependencies:
   # Base
-  # =================
+  # =======================
   - python >=3.9
   - pip
   - beautifulsoup4
@@ -24,29 +25,32 @@ dependencies:
   - numpy >=1.23.0
   - shapely >=2.0.0,<3.0.0
   - xarray >=2023.02.0
+  - xskillscore >=0.0.20
+  - xcdat==0.6.0rc1
+  - xesmf >=0.7.0
   # Testing
-  # ==================
+  # =======================
   - scipy
   - pytest
   - pytest-cov
   # Documentation
-  # =================
+  # =======================
   - sphinx
   - sphinx_rtd_theme
   - sphinx-multiversion
+  # Quality Assurance Tools
+  # =======================
+  # Run `pre-commit autoupdate` to get the latest pinned versions of 'rev' in
+  # `.pre-commit.config.yaml`, then update the pinned versions here.
+  - black=23.9.1
+  - flake8=6.1.0
+  - flake8-isort=6.1.0
+  - isort=5.12.0
+  - mypy=1.5.1
+  - pre-commit >=3.0.0
+  - types-PyYAML >=6.0.0
   # Developer Tools
-  # =================
-  # If versions are updated, also update 'rev' in `.pre-commit.config.yaml`
-  - black=22.10.0
-  - flake8=6.0.0
-  - flake8-isort=5.0.3
-  - isort=5.11.3
-  - mypy=0.991
-  - pre-commit=2.20.0
-  - pytest=7.2.0
-  - pytest-cov=4.0.0
-  - types-PyYAML=6.0.12.6
-  # Developer Tools
-  # =================
+  # =======================
   - tbump=6.9.0
+  - ipykernel
 prefix: /opt/miniconda3/envs/e3sm_diags_dev