ufs-community · DusanJovic-NOAA · Jul 1, 2021 · Apr 15, 2021 · Apr 15, 2021 · Apr 15, 2021
diff --git a/.gitmodules b/.gitmodules
@@ -1,11 +1,11 @@
 [submodule "FV3"]
   path = FV3
-  url = https://github.com/NOAA-EMC/fv3atm
-  branch = develop
+  url = https://github.com/rmontuoro/fv3atm
+  branch = feature/gocart-integration
 [submodule "NEMS"]
   path = NEMS
-  url = https://github.com/NOAA-EMC/NEMS
-  branch = develop
+  url = https://github.com/rmontuoro/NEMS
+  branch = feature/ufs-gocart-integration
 [submodule "WW3"]
   path = WW3
   url = https://github.com/NOAA-EMC/WW3
@@ -38,3 +38,7 @@
   path = CDEPS-interface/CDEPS
   url = https://github.com/NOAA-EMC/CDEPS
   branch = develop
+[submodule "GOCART"]
+  path = GOCART
+  url = https://github.com/GEOS-ESM/GOCART
+  branch = feature/rmontuoro/ufs-integration
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -16,14 +16,15 @@ list(APPEND CMAKE_MODULE_PATH ${CMAKE_SOURCE_DIR}/CMakeModules/Modules)
 ###############################################################################
 
 # Valid applications and choices
-list(APPEND VALID_APPS ATM ATMW S2S S2SW NG-GODAS NG-GODAS-NEMSDATM)
+list(APPEND VALID_APPS ATM ATMAERO ATMW S2S S2SW NG-GODAS NG-GODAS-NEMSDATM)
 set(APP NONE CACHE BOOL "Application Name")
 if(NOT (APP IN_LIST VALID_APPS))
   message(FATAL_ERROR "${APP} is not a valid application.\nValid Applications are: ${VALID_APPS}")
 endif()
 
 set(FMS             OFF CACHE BOOL "Enable FMS")
 set(FV3             OFF CACHE BOOL "Enable FV3")
+set(UFS_GOCART      OFF CACHE BOOL "Enable GOCART")
 set(MOM6            OFF CACHE BOOL "Enable MOM6")
 set(CICE6           OFF CACHE BOOL "Enable CICE6")
 set(WW3             OFF CACHE BOOL "Enable WW3")
@@ -39,6 +40,7 @@ include(cmake/configure_apps.cmake)
 message("")
 message("FMS .............. ${FMS}")
 message("FV3 .............. ${FV3}")
+message("GOCART ........... ${UFS_GOCART}")
 message("MOM6 ............. ${MOM6}")
 message("CICE6 ............ ${CICE6}")
 message("WW3 .............. ${WW3}")
@@ -149,6 +151,13 @@ if(FV3)
   add_subdirectory(FV3)
 endif()
 
+###############################################################################
+### GOCART
+###############################################################################
+if(UFS_GOCART)
+  add_subdirectory(GOCART)
+endif()
+
 ###############################################################################
 ### Wave components [WW3]
 ###############################################################################
@@ -227,6 +236,12 @@ if(JEDI_DRIVER)
   list(APPEND _ufs_defs_private JEDI_DRIVER=ON)
 endif()
 
+if(UFS_GOCART)
+  add_dependencies(ufs UFS_Aerosols)
+  list(APPEND _ufs_defs_private FRONT_GOCART=Aerosol_Cap)
+  list(APPEND _ufs_libs_public UFS_Aerosols)
+endif()
+
 if(WW3)
   add_dependencies(ufs ww3_nems)
   list(APPEND _ufs_defs_private FRONT_WW3=WMESMFMD)

diff --git a/FV3 b/FV3
diff --git a/GOCART b/GOCART
diff --git a/NEMS b/NEMS
diff --git a/build.sh b/build.sh
@@ -17,6 +17,7 @@ BUILD_DIR=${BUILD_DIR:-${UFS_MODEL_DIR}/build}
 mkdir -p ${BUILD_DIR}
 
 [[ -n "${CCPP_SUITES:-""}" ]] && CMAKE_FLAGS+=" -DCCPP_SUITES=${CCPP_SUITES}"
+[[ -n "${MAPL_ROOT:-""}" ]] && CMAKE_FLAGS+=" -DCMAKE_MODULE_PATH=${MAPL_ROOT}/share/MAPL/cmake"
 
 cd ${BUILD_DIR}
 cmake ${UFS_MODEL_DIR} ${CMAKE_FLAGS}

diff --git a/cmake/configure_apps.cmake b/cmake/configure_apps.cmake
@@ -53,3 +53,11 @@ if(APP MATCHES "^(S2S|S2SW)$")
     message("Configuring UFS app in S2S mode")
   endif()
 endif()
+
+if(APP MATCHES "^(ATMAERO)$")
+  set(FMS        ON  CACHE BOOL "Enable FMS"                 FORCE)
+  set(FV3        ON  CACHE BOOL "Enable FV3"                 FORCE)
+  set(STOCH_PHYS ON  CACHE BOOL "Enable Stochastic Physics"  FORCE)
+  set(UFS_GOCART ON  CACHE BOOL "Enable GOCART"              FORCE)
+  message("Configuring UFS app in Atmosphere with Aerosols mode")
+endif()
diff --git a/modulefiles/ufs_common b/modulefiles/ufs_common
@@ -20,3 +20,8 @@ module load sp/2.3.3
 module load w3emc/2.7.3
 module load w3nco/2.4.1
 module load upp/10.0.8
+
+module load gftl-shared/v1.3.0
+module load yafyaml/v0.5.1
+module load mapl/v2.7.3
+module load intelpython/3.6.8
diff --git a/modulefiles/ufs_common_debug b/modulefiles/ufs_common_debug
@@ -20,3 +20,8 @@ module load sp/2.3.3
 module load w3emc/2.7.3
 module load w3nco/2.4.1
 module load upp/10.0.8
+
+module load gftl-shared/v1.3.0
+module load yafyaml/v0.5.1
+module load mapl/v2.7.3
+module load intelpython/3.6.8
diff --git a/modulefiles/ufs_hera.intel b/modulefiles/ufs_hera.intel
@@ -24,3 +24,6 @@ setenv CC mpiicc
 setenv CXX mpiicpc
 setenv FC mpiifort
 setenv CMAKE_Platform hera.intel
+
+# use shared memory and OpenFabrics Alliance (OFA) fabric with Intel MPI to circumvent RDMA-related bug in DAPL.
+setenv I_MPI_FABRICS shm:ofa
diff --git a/modulefiles/ufs_hera.intel_debug b/modulefiles/ufs_hera.intel_debug
@@ -24,3 +24,6 @@ setenv CC mpiicc
 setenv CXX mpiicpc
 setenv FC mpiifort
 setenv CMAKE_Platform hera.intel
+
+# use shared memory and OpenFabrics Alliance (OFA) fabric with Intel MPI to circumvent RDMA-related bug in DAPL.
+setenv I_MPI_FABRICS shm:ofa
diff --git a/tests/compile.sh b/tests/compile.sh
@@ -132,29 +132,26 @@ fi
 set -ex
 
 # Valid applications
-if [[ "${MAKE_OPT}" == *"APP=ATM"* ]]; then
+if [[ "${MAKE_OPT}" == *"APP=ATMAERO"* ]]; then
+    CMAKE_FLAGS="${CMAKE_FLAGS} -DAPP=ATMAERO"
+    unset PYTHONPATH
+elif [[ "${MAKE_OPT}" == *"APP=ATMW"* ]]; then
+    CMAKE_FLAGS="${CMAKE_FLAGS} -DAPP=ATMW"
+elif [[ "${MAKE_OPT}" == *"APP=ATM"* ]]; then
     echo "MAKE_OPT = ${MAKE_OPT}"
     CMAKE_FLAGS="${CMAKE_FLAGS} -DAPP=ATM"
 fi
 
-if [[ "${MAKE_OPT}" == *"APP=ATMW"* ]]; then
-    CMAKE_FLAGS="${CMAKE_FLAGS} -DAPP=ATMW"
-fi
-
-if [[ "${MAKE_OPT}" == *"APP=S2S"* ]]; then
-    CMAKE_FLAGS="${CMAKE_FLAGS} -DAPP=S2S -DMOM6SOLO=ON"
-fi
-
 if [[ "${MAKE_OPT}" == *"APP=S2SW"* ]]; then
     CMAKE_FLAGS="${CMAKE_FLAGS} -DAPP=S2SW -DMOM6SOLO=ON"
-fi
-
-if [[ "${MAKE_OPT}" == *"APP=NG-GODAS"* ]]; then
-    CMAKE_FLAGS="${CMAKE_FLAGS} -DAPP=NG-GODAS"
+elif [[ "${MAKE_OPT}" == *"APP=S2S"* ]]; then
+    CMAKE_FLAGS="${CMAKE_FLAGS} -DAPP=S2S -DMOM6SOLO=ON"
 fi
 
 if [[ "${MAKE_OPT}" == *"APP=NG-GODAS-NEMSDATM"* ]]; then
     CMAKE_FLAGS="${CMAKE_FLAGS} -DAPP=NG-GODAS-NEMSDATM"
+elif [[ "${MAKE_OPT}" == *"APP=NG-GODAS"* ]]; then
+    CMAKE_FLAGS="${CMAKE_FLAGS} -DAPP=NG-GODAS"
 fi
 
 CMAKE_FLAGS=$(trim "${CMAKE_FLAGS}")

diff --git a/tests/default_vars.sh b/tests/default_vars.sh
@@ -514,6 +514,9 @@ export IMFDEEPCNV=2
 export HWRF_SAMFDEEP=.F.
 export RAS=.F.
 
+# Aerosol convective scavenging
+export FSCAV_AERO="'*:0.0'"
+
 # SFC
 export DO_MYJSFC=.F.
 export DO_MYNNSFCLAY=.F.
@@ -533,6 +536,7 @@ export H2O_PHYS=.F.
 
 
 export CPL=.F.
+export CPLCHM=.F.
 export CPLFLX=.F.
 export CPLWAV=.F.
 export CPLWAV2ATM=.F.

diff --git a/tests/fv3_conf/control_run.IN b/tests/fv3_conf/control_run.IN
@@ -142,3 +142,11 @@ if [ $IMP_PHYSICS = 8 ]; then
   cp    @[INPUTDATA_ROOT]/FV3_input_data_gsd/freezeH2O.dat .
   cp    @[INPUTDATA_ROOT]/FV3_input_data_gsd/CCN_ACTIVATE.BIN .
 fi
+
+#prognostic aerosols
+if [ $CPLCHM = .T. ]; then
+  cp      @[INPUTDATA_ROOT]/GOCART/diag_table  .
+  cp      @[INPUTDATA_ROOT]/GOCART/field_table .
+  cp      @[INPUTDATA_ROOT]/GOCART/rc/* .
+  ln -sf  @[INPUTDATA_ROOT]/GOCART/ExtData .
+fi
diff --git a/tests/parm/control.nml.IN b/tests/parm/control.nml.IN
@@ -61,7 +61,7 @@ deflate_level=1
   nwat = 6
   na_init = @[NA_INIT]
   d_ext = 0.
-  dnats = 1
+  dnats = @[DNATS]
   fv_sg_adj = 450
   d2_bg = 0.
   nord = 2
@@ -169,6 +169,7 @@ deflate_level=1
   psautco      = 0.0008,0.0005
   prautco      = 0.00015,0.00015
   lgfdlmprad   = .true.
+  cplchm       = @[CPLCHM]
   cplwav       = @[CPLWAV]
   cplwav2atm   = @[CPLWAV2ATM]
   effr_in      = .true.
@@ -180,6 +181,7 @@ deflate_level=1
   do_skeb      = @[DO_SKEB]
   lndp_type    = @[LNDP_TYPE]
   n_var_lndp   = @[N_VAR_LNDP]
+  fscav_aero   = @[FSCAV_AERO]
 /
 
 &gfdl_cloud_microphysics_nml

diff --git a/tests/parm/fd_nems.yaml b/tests/parm/fd_nems.yaml
@@ -693,6 +693,22 @@
        canonical_units: W m-2
      - standard_name: land_mask
        canonical_units: 1
+     - standard_name: inst_cloud_frac_levels
+       canonical_units: 1
+     - standard_name: inst_ice_nonconv_tendency_levels
+       canonical_units: kg m-2 s-1
+     - standard_name: inst_liq_nonconv_tendency_levels
+       canonical_units: kg m-2 s-1
+     - standard_name: inst_surface_soil_wetness
+       canonical_units: 1
+     - standard_name: lake_fraction
+       canonical_units: 1
+     - standard_name: ice_fraction_in_atm
+       canonical_units: 1
+     - standard_name: ocean_fraction
+       canonical_units: 1
+     - standard_name: surface_snow_area_fraction
+       canonical_units: 1
      #
      #-----------------------------------
      # WW3 import

diff --git a/tests/parm/nems.configure.atm_aerosols.IN b/tests/parm/nems.configure.atm_aerosols.IN
@@ -0,0 +1,34 @@
+#############################################
+####  NEMS Run-Time Configuration File  #####
+#############################################
+
+# EARTH #
+EARTH_component_list: ATM CHM
+EARTH_attributes::
+  Verbosity = 0
+::
+
+# ATM #
+ATM_model:                      @[atm_model]
+ATM_petlist_bounds:             @[atm_petlist_bounds]
+ATM_attributes::
+  Verbosity = 0
+::
+
+# CHM #
+CHM_model:                      @[chm_model]
+CHM_petlist_bounds:             @[chm_petlist_bounds]
+CHM_attributes::
+  Verbosity = 0
+::
+
+# Run Sequence #
+runSeq::
+  @@[coupling_interval_sec]
+    ATM phase1
+    ATM -> CHM
+    CHM
+    CHM -> ATM
+    ATM phase2
+  @
+::
diff --git a/tests/rt.conf b/tests/rt.conf
@@ -187,3 +187,10 @@ RUN     | datm_cdeps_debug_cfsr
 
 COMPILE | APP=ATMW SUITES=FV3_GFS_v16                                                                                           | - wcoss_cray  | fv3 |
 RUN     | control_atmwav                                                                                                        | - wcoss_cray  | fv3 |
+
+###################################################################################################################################################################################
+# ATM-GOCART tests                                                                                                                                                                   #
+###################################################################################################################################################################################
+
+COMPILE | APP=ATMAERO SUITES=FV3_GFS_v16                                                                                        |               | fv3 |
+RUN     | control_atm_aerosols                                                                                                  |               | fv3 |
diff --git a/tests/tests/control_atm_aerosols b/tests/tests/control_atm_aerosols
@@ -0,0 +1,83 @@
+###############################################################################
+#
+#  global control test with prognostic aerosols: GFSv16 FV3-GOCART at C96L127
+#
+###############################################################################
+
+export TEST_DESCR="Compare global results for prognostic aerosols with previous trunk version"
+
+export CNTL_DIR=control_atm_aerosols
+
+export LIST_FILES="sfcf000.nc \
+                   sfcf024.nc \
+                   atmf000.nc \
+                   atmf024.nc \
+                   GFSFLX.GrbF00 \
+                   GFSFLX.GrbF24 \
+                   GFSPRS.GrbF00 \
+                   GFSPRS.GrbF24 \
+                   gocart.inst_aod.20210322_0700z.nc4 \
+                   gocart.inst_aod.20210323_0600z.nc4 \
+                   gocart.tavg_cmass.20210322_0730z.nc4 \
+                   gocart.tavg_cmass.20210323_0430z.nc4 \
+                   RESTART/coupler.res \
+                   RESTART/fv_core.res.nc \
+                   RESTART/fv_core.res.tile1.nc \
+                   RESTART/fv_core.res.tile2.nc \
+                   RESTART/fv_core.res.tile3.nc \
+                   RESTART/fv_core.res.tile4.nc \
+                   RESTART/fv_core.res.tile5.nc \
+                   RESTART/fv_core.res.tile6.nc \
+                   RESTART/fv_srf_wnd.res.tile1.nc \
+                   RESTART/fv_srf_wnd.res.tile2.nc \
+                   RESTART/fv_srf_wnd.res.tile3.nc \
+                   RESTART/fv_srf_wnd.res.tile4.nc \
+                   RESTART/fv_srf_wnd.res.tile5.nc \
+                   RESTART/fv_srf_wnd.res.tile6.nc \
+                   RESTART/fv_tracer.res.tile1.nc \
+                   RESTART/fv_tracer.res.tile2.nc \
+                   RESTART/fv_tracer.res.tile3.nc \
+                   RESTART/fv_tracer.res.tile4.nc \
+                   RESTART/fv_tracer.res.tile5.nc \
+                   RESTART/fv_tracer.res.tile6.nc \
+                   RESTART/phy_data.tile1.nc \
+                   RESTART/phy_data.tile2.nc \
+                   RESTART/phy_data.tile3.nc \
+                   RESTART/phy_data.tile4.nc \
+                   RESTART/phy_data.tile5.nc \
+                   RESTART/phy_data.tile6.nc \
+                   RESTART/sfc_data.tile1.nc \
+                   RESTART/sfc_data.tile2.nc \
+                   RESTART/sfc_data.tile3.nc \
+                   RESTART/sfc_data.tile4.nc \
+                   RESTART/sfc_data.tile5.nc \
+                   RESTART/sfc_data.tile6.nc"
+
+export_fv3
+export NPZ=127
+export NPZP=128
+export DT_ATMOS=720
+export SYEAR=2021
+export SMONTH=03
+export SDAY=22
+export SHOUR=06
+export RESTART_INTERVAL=12
+export OUTPUT_GRID='gaussian_grid'
+export NSTF_NAME='2,0,0,0,0'
+export WRITE_DOPOST=.true.
+export IAER=5111
+export FSCAV_AERO='"*:0.6", "seas1:1.0", "seas2:1.0", "seas3:1.0", "seas4:1.0", "seas5:1.0"'
+export DNATS=3
+
+export CPL=.true.
+export CPLCHM=.T.
+export atm_model='fv3'
+export atm_petlist_bounds="0 149"
+export chm_model='gocart'
+export chm_petlist_bounds="0 143"
+export coupling_interval_sec=${DT_ATMOS}
+export NEMS_CONFIGURE="nems.configure.atm_aerosols.IN"
+
+export FV3_RUN=control_run.IN
+export CCPP_SUITE=FV3_GFS_v16
+export INPUT_NML=control.nml.IN
+2 −2		.gitmodules
+1 −0		CMakeLists.txt
+497 −1,159		atmos_model.F90
+137 −187		ccpp/data/GFS_typedefs.F90
+40 −96		ccpp/data/GFS_typedefs.meta
+0 −155		ccpp/driver/GFS_diagnostics.F90
+1 −0		ccpp/driver/GFS_init.F90
+1 −1		ccpp/physics
+568 −0		cpl/module_block_data.F90
+69 −29		cpl/module_cap_cpl.F90
+19 −6		cpl/module_cplfields.F90
+12 −17		fv3_cap.F90
+14 −9		io/module_write_netcdf_parallel.F90
+7 −18		module_fcst_grid_comp.F90