sync to the develop branch

CDEPS-interface/cdeps_files.cmake

@@ -44,6 +44,7 @@ list(APPEND cdeps_datm_files
   CDEPS/datm/datm_datamode_era5_mod.F90
   CDEPS/datm/datm_datamode_gefs_mod.F90
   CDEPS/datm/datm_datamode_gfs_mod.F90
+  CDEPS/datm/datm_datamode_gfs_hafs_mod.F90
   CDEPS/datm/datm_datamode_jra_mod.F90
 )
 
@@ -62,6 +63,8 @@ list(APPEND cdeps_docn_files
   CDEPS/docn/docn_datamode_copyall_mod.F90
   CDEPS/docn/docn_datamode_iaf_mod.F90
   CDEPS/docn/docn_datamode_som_mod.F90
+  CDEPS/docn/docn_datamode_cplhist_mod.F90
+  CDEPS/docn/docn_import_data_mod.F90
   CDEPS/docn/ocn_comp_nuopc.F90
 )
 

CMakeLists.txt

@@ -70,6 +70,7 @@ set(OPENMP          ON  CACHE BOOL "Enable OpenMP threading")
 set(PARALLEL_NETCDF OFF CACHE BOOL "Enable parallel NetCDF")
 set(JEDI_DRIVER     OFF CACHE BOOL "Enable JEDI as top level driver")
 set(CMEPS_AOFLUX    OFF CACHE BOOL "Enable atmosphere-ocean flux calculation in mediator")
+set(PDLIB           OFF CACHE BOOL "Enable Domain Decomposition in WW3 via PDLIB")
 
 set(CMAKE_Platform $ENV{CMAKE_Platform})
 if(CMAKE_Platform)
@@ -206,7 +207,11 @@ if(WW3)
     set(UFS_CAP "MULTI_ESMF" CACHE STRING "Build with MULTI_ESMF cap")
     list(APPEND _ufs_defs_private FRONT_WW3=WMESMFMD)
   else()
-    set(SWITCH "meshcap" CACHE STRING "NUOPC mesh cap")
+    if(PDLIB)
+      set(SWITCH "meshcap_pdlib" CACHE STRING "NUOPC mesh cap")
+    else()
+      set(SWITCH "meshcap" CACHE STRING "NUOPC mesh cap")
+    endif()
     set(UFS_CAP "NUOPC_MESH" CACHE STRING "Build with NUOPC_MESH cap")
     list(APPEND _ufs_defs_private FRONT_WW3=wav_comp_nuopc)
   endif()

MOM6-interface/mom6_files.cmake

@@ -12,7 +12,6 @@ list(APPEND mom6_src_files
   MOM6/src/ALE/coord_hycom.F90
   MOM6/src/ALE/coord_rho.F90
   MOM6/src/ALE/coord_sigma.F90
-  MOM6/src/ALE/coord_slight.F90
   MOM6/src/ALE/coord_zlike.F90
   MOM6/src/ALE/MOM_hybgen_regrid.F90
   MOM6/src/ALE/MOM_hybgen_remap.F90
@@ -22,6 +21,7 @@ list(APPEND mom6_src_files
   MOM6/src/ALE/regrid_edge_values.F90
   MOM6/src/ALE/regrid_interp.F90
   MOM6/src/ALE/regrid_solvers.F90
+  MOM6/src/ALE/remapping_attic.F90
 
   MOM6/src/core/MOM.F90
   MOM6/src/core/MOM_CoriolisAdv.F90
@@ -109,7 +109,6 @@ list(APPEND mom6_src_files
   MOM6/src/framework/MOM_data_override.F90
   MOM6/src/framework/MOM_diag_mediator.F90
   MOM6/src/framework/MOM_diag_remap.F90
-  MOM6/src/framework/MOM_diag_vkernels.F90
   MOM6/src/framework/MOM_document.F90
   MOM6/src/framework/MOM_domains.F90
   MOM6/src/framework/MOM_dyn_horgrid.F90
@@ -122,6 +121,8 @@ list(APPEND mom6_src_files
   MOM6/src/framework/MOM_interpolate.F90
   MOM6/src/framework/MOM_intrinsic_functions.F90
   MOM6/src/framework/MOM_io.F90
+  MOM6/src/framework/MOM_io_file.F90
+  MOM6/src/framework/MOM_netcdf.F90
   MOM6/src/framework/posix.F90
   MOM6/src/framework/MOM_random.F90
   MOM6/src/framework/MOM_restart.F90

doc/UsersGuide/requirements.txt

@@ -1,2 +1,3 @@
 sphinxcontrib-bibtex
 sphinx_rtd_theme
+docutils==0.16

doc/UsersGuide/source/CodeOverview.rst

@@ -1,14 +1,52 @@
 .. _CodeOverview:
 
 *************************
-Code Overview
+Technical Overview
 *************************
 
+.. _SupportedPlatforms:
+
+=====================================================================
+Supported Platforms and Compilers for Running the UFS Weather Model
+=====================================================================
+
+Four levels of support have been defined for :term:`UFS` applications, and the UFS Weather Model (:term:`WM`) operates under this paradigm: 
+
+* **Level 1** *(Preconfigured)*: Prerequisite software libraries are pre-built and available in a central location; code builds and runs; full testing of model.
+* **Level 2** *(Configurable)*: Prerequisite libraries are not available in a centralized location but are expected to install successfully; code builds and runs; full testing of model.
+* **Level 3** *(Limited-test platforms)*: Libraries and code build on these systems, but there is limited testing with running the model.
+* **Level 4** *(Build-only platforms)*: Libraries and code build, but running the model is not tested.
+
+Level 1 Systems
+==================
+Preconfigured (Level 1) systems for the UFS WM already have the required external libraries available in a central location via :term:`HPC-Stack` or :term:`spack-stack`. The WM is expected to build and run out-of-the-box on these systems, and users can download the WM code without first installing prerequisite software. Additionally, regression test data is already available on these systems. In general, users must have access to these Level 1 systems in order to use them.
+
+Currently, Level 1 (or Tier-1) platforms for regression testing are: 
+
+   * WCOSS2 (Intel)
+   * Cheyenne (Intel/GNU compilers)
+   * Gaea (Intel)
+   * Hera (Intel/GNU compilers)
+   * Jet (Intel)
+   * Orion (Intel)
+   * AWS Docker container (Intel)
+
+More information is available in the `UFS WM wiki <https://github.com/ufs-community/ufs-weather-model/wiki/Regression-Test-Policy-for-Weather-Model-Platforms-and-Compilers>`__. 
+
+Level 2-4 Systems
+===================
+
+On non-Level 1 platforms, users must install the required libraries before building the UFS WM. Additionally, users must stage the required data in order to run regression tests. Once the prerequisite libraries are installed, and the data has been staged, the WM should build and run successfully. However, users may need to perform additional troubleshooting on Level 3 or 4 systems since little or no testing is conducted on these systems.
+
+Currently, Level 2 platforms for regression testing are:
+
+   * S4 (Intel)
+
 ===================================================
 UFS Weather Model Hierarchical Repository Structure
 ===================================================
 
-The UFS Weather Model (:term:`WM`) repository supports the :term:`UFS` short- and medium-range weather applications (:term:`SRW` / :term:`MRW` Apps). The WM repository contains atmosphere, ocean, sea ice, and wave components, as well as some infrastructure components. Each of these subcomponents has its own repository. All the repositories are currently located in GitHub with public access to the broad community. :numref:`Table %s <Repo_Structure>` describes the list of repositories that comprises the UFS WM.
+The UFS :term:`WM` repository supports the :term:`UFS` short- and medium-range weather applications (:term:`SRW` / :term:`MRW` Apps). The WM repository contains atmosphere, ocean, sea ice, land, and wave components, as well as some infrastructure components. Each of these subcomponents has its own repository. All the repositories are currently located in GitHub with public access to the broader community. :numref:`Table %s <Repo_Structure>` describes the list of repositories that comprise the UFS WM.
 
 .. _Repo_Structure:
 
@@ -20,103 +58,107 @@ The UFS Weather Model (:term:`WM`) repository supports the :term:`UFS` short- an
     - Authoritative repository URL
   * - Umbrella repository for the UFS Weather Model
     - https://github.com/ufs-community/ufs-weather-model
-  * - Framework to connect the CCPP library to a host model
+  * - Framework to connect the :term:`CCPP` library to a host model
     - https://github.com/NCAR/ccpp-framework
   * - CCPP library of physical parameterizations
     - https://github.com/NCAR/ccpp-physics
   * - Umbrella repository for the physics and dynamics of the atmospheric model (FV3) 
     - https://github.com/NOAA-EMC/fv3atm
-  * - FV3 dynamical core
+  * - :term:`FV3` dynamical core
     - https://github.com/NOAA-GFDL/GFDL_atmos_cubed_sphere
   * - Stochastic physics pattern generator
     - https://github.com/noaa-psd/stochastic_physics
-  * - Modular Ocean Model (MOM6)
+  * - Modular Ocean Model (:term:`MOM6`)
     - https://github.com/NOAA-EMC/MOM6
-  * - HYbrid Coordinate Ocean Model (HYCOM)
+  * - HYbrid Coordinate Ocean Model (:term:`HYCOM`)
     - https://github.com/NOAA-EMC/HYCOM-src
-  * - Los Alamos sea ice model (CICE6)
+  * - Los Alamos sea ice model (:term:`CICE6`)
     - https://github.com/NOAA-EMC/CICE
-  * - NOAA/NCEP WAVEWATCH III Model (WW3)
+  * - NOAA/NCEP WAVEWATCH III Model (:term:`WW3`)
     - https://github.com/NOAA-EMC/WW3
-  * - The Goddard Chemistry Aerosol Radiation and Transport (GOCART)
+  * - The Goddard Chemistry Aerosol Radiation and Transport (:term:`GOCART`)
     - https://github.com/GEOS-ESM/GOCART 
-  * - NUOPC Community Mediator for Earth Prediction Systems (CMEPS)
+  * - NUOPC Community Mediator for Earth Prediction Systems (:term:`CMEPS`)
     - https://github.com/NOAA-EMC/CMEPS
-  * - Community Data Models for Earth Prediction Systems (CDEPS)
+  * - Community Data Models for Earth Prediction Systems (:term:`CDEPS`)
     - https://github.com/NOAA-EMC/CDEPS
-  * - Air Quality Model (AQM)
+  * - Air Quality Model (:term:`AQM`)
     - https://github.com/NOAA-EMC/AQM
+  * - Noah-MP Land Surface Model (Noah-MP)
+    - https://github.com/NOAA-EMC/noahmp
 
-In the table, the left column contains a description of each repository, and the right column shows the component repositories which are pointing to (or will point to) the authoritative repositories. The UFS WM currently uses Git submodules to manage the sub-components.
+In the table, the left-hand column contains a description of each repository, and the 
+right-hand column shows the GitHub location of the authoritative component repositories. 
+The UFS WM currently uses Git submodules to manage these subcomponents.
 
 ===================
 Directory Structure
 ===================
 
-The umbrella repository for the UFS Weather Model is named ``ufs-weather-model``.  Under this repository reside a number of submodules that are nested in specific directories under the parent repository's working directory. When the ``ufs-weather-model`` repository is cloned, the basic directory structure will be similar to the example below. Files and some directories have been removed for brevity. Directories in parentheses will appear only after a submodule update (``git submodule update --init --recursive``). 
+The umbrella repository for the UFS WM is named ``ufs-weather-model``. Under this repository reside a number of submodules that are nested in specific directories under the parent repository's working directory. When the ``ufs-weather-model`` repository is cloned, the basic directory structure will be similar to the example below. Files and some directories have been removed for brevity. Directories in parentheses will appear only after a recursive clone or submodule update (``git submodule update --init --recursive``). 
 
 .. code-block:: console
 
-   ufs-weather-model/
-   ├── build.sh                 -------- script for building the WM
-   ├── cmake                    -------- cmake configuration files
-   ├── CMakeLists.txt         
-   ├── CMakeModules           
-   ├── doc                      -------- User Guide files
-   ├── driver                 
-   ├── FV3                      -------- UFSAtm atmosphere model
-   │   ├── (atmos_cubed_sphere) -------- FV3 dynamical core
-   │   │   ├── (docs)
-   │   │   ├── (driver)
-   │   │   ├── (model)
-   │   │   └── (tools)
-   │   ├── (ccpp)               -------- Common Community Physics Package
-   │   │   ├── (config)
-   │   │   ├── (driver)
-   │   │   ├── (framework)      -------- CCPP framework
-   │   │   ├── (physics)        -------- CCPP compliant physics schemes
-   │   │   └── (suites)         -------- CCPP physics suite definition files (SDFs)
-   │   ├── (cpl)                -------- Coupling field data structures
-   │   ├── (io)                 -------- UFSAtm write grid comp code
-   │   └── (stochastic_physics) -------- Wrapper for stochastic physics
-   │
-   ├── stochastic_physics       -------- stochastic physics pattern generator
-   ├── AQM
-   │     └── (src)
-   │         ├── (model)
-   │            └── (CMAQ)                         --------- EPA AQ Model
-   ├── CICE-interface
-   │    └── CICE                                   --------- CICE6 sea ice model
-   │        ├── (icepack)                          --------- Sea ice column physics
-   │        └── (cicecore/drivers/nuopc/cmeps)     --------- NUOPC CICE6 cap
-   ├── GOCART
-   │    └── (ESMF)                                 --------- GOCART model
-   ├── HYCOM-interface
-   │    └── HYCOM                                  --------- HYCOM ocean model
-   │        └── (NUOPC)                            --------- NUOPC HYCOM cap
-   ├── MOM6-interface
-   │    └── MOM6
-   │        ├── (src)                              --------- MOM6 ocean model
-   │        └── (config_source/drivers/nuopc_cap)  --------- NUOPC MOM6 cap
-   ├── WW3
-   │    └── (model)                                --------- WW3 model
-   │        └── (esmf)                             --------- NUOPC WW3 cap
-   ├── CDEPS-interface
-   │     └── CDEPS
-   │         ├── (datm)                            --------- CDEPS DATM
-   │         └── (docn)                            --------- CDEPS DOCN
-   ├── CMEPS-interface
-   │    └── CMEPS
-   │         └── (cesm)                            --------- CMEPS CESM
-   ├── modulefiles          -------- system module files for supported HPC systems
-   └── tests                -------- regression test infrastructure
-       └── parm
-       └── tests
-       └── fv3_conf   
-
-The physics subdirectory in the ``gfsphysics`` directory  is not used or supported
-as part of this release (all physics is available through the :term:`CCPP` using
-the repository described in :numref:`Table %s <Repo_Structure>`).
-
-.. COMMENT: Should we omit CMakeLists.txt, CMakeModules, driver (which I added) or add a description?
-.. COMMENT: I don't see a "gfsphysics" directory... Can we remove it or be more specific about where it is? There are two CCPP repos in the table referenced above... Framework and Physics.
+   ufs-weather-model
+    ├── AQM
+    │     └── (src)
+    │         ├── (model)
+    │            └── (CMAQ)                      -------- EPA Air Quality Model
+    ├── build.sh                                 -------- script for building the WM
+    ├── CDEPS-interface
+    │     └── CDEPS
+    │         ├── (datm)                         -------- CDEPS DATM
+    │         └── (docn)                         -------- CDEPS DOCN
+    ├── CICE-interface
+    │    └── CICE                                -------- CICE6 sea ice model
+    │        ├── (icepack)                       -------- Sea ice column physics
+    │        └── (cicecore/drivers/nuopc/cmeps)  -------- NUOPC CICE6 cap
+    ├── cmake                                    -------- cmake configuration files
+    ├── CMakeLists.txt         
+    ├── CMakeModules           
+    ├── CMEPS-interface
+    │    └── CMEPS
+    │         └── (cesm)                         -------- CMEPS CESM
+    ├── doc                                      -------- User Guide files
+    ├── driver                 
+    ├── FV3                                      -------- UFSAtm atmosphere model
+    │   ├── (atmos_cubed_sphere)                 -------- FV3 dynamical core
+    │   │   ├── (docs)
+    │   │   ├── (driver)
+    │   │   ├── (model)
+    │   │   └── (tools)
+    │   ├── (ccpp)                               -------- Common Community Physics Package
+    │   │   ├── (config)
+    │   │   ├── (driver)
+    │   │   ├── (framework)                      -------- CCPP framework
+    │   │   ├── (physics)                        -------- CCPP-compliant physics schemes
+    │   │   └── (suites)                         -------- CCPP physics suite definition files (SDFs)
+    │   ├── (cpl)                                -------- Coupling field data structures
+    │   ├── (io)                                 -------- UFSAtm write grid comp code
+    │   └── (stochastic_physics)                 -------- Wrapper for stochastic physics
+    ├── GOCART
+    │    └── (ESMF)                              -------- GOCART model
+    ├── HYCOM-interface
+    │    └── HYCOM                               -------- HYCOM ocean model
+    │        └── (NUOPC)                         -------- NUOPC HYCOM cap
+    ├── LICENSE.md
+    ├── modulefiles                              -------- system module files for supported HPC systems
+    ├── MOM6-interface
+    │    └── MOM6
+    │        ├── (src)                           -------- MOM6 ocean model
+    │        └── (config_source/drivers/nuopc_cap)  -------- NUOPC MOM6 cap
+    ├── NOAHMP-interface
+    │    └── noahmp
+    │        ├── (cmake)                         -------- Noah-MP land model
+    │        ├── (drivers/nuopc)                 -------- NUOPC Noah-MP cap
+    │        ├── (parameters)
+    │        └── (src)
+    ├── README.md
+    ├── stochastic_physics                       -------- stochastic physics pattern generator
+    ├── tests                                    -------- regression test infrastructure
+    │   └── parm
+    │   └── tests
+    │   └── fv3_conf   
+    └── WW3
+         └── (model)                             -------- WW3 model
+             └── (src)                           -------- NUOPC WW3 caps