Update FV3 (Thompson MP and RRTMGP), CMEPS, documentation and CI

.github/workflows/aux.yml

@@ -9,7 +9,7 @@ env:
   base_url: $GITHUB_API_URL/repos/$GITHUB_REPOSITORY/actions/runs
   AUTH: ${{ secrets.GITHUB_TOKEN }}
   aws_instance_id: ${{ secrets.AWS_INSTANCE_ID }}
-  no_instances: 6
+  no_instances: 10
 
 
 jobs:

.gitmodules

@@ -1,7 +1,7 @@
 [submodule "FV3"]
   path = FV3
-  url = https://github.com/NOAA-EMC/fv3atm
-  branch = develop
+  url = https://github.com/climbfuji/fv3atm
+  branch = thompson_ext_diag_dom
 [submodule "NEMS"]
   path = NEMS
   url = https://github.com/NOAA-EMC/NEMS
@@ -20,8 +20,10 @@
   branch = develop
 [submodule "CMEPS"]
   path = CMEPS-interface/CMEPS
-  url = https://github.com/NOAA-EMC/CMEPS.git
-  branch = emc/develop
+  #url = https://github.com/NOAA-EMC/CMEPS.git
+  #branch = emc/develop
+  url = https://github.com/DeniseWorthen/CMEPS.git
+  branch = feature/updcmeps
 [submodule "MOM6"]
   path = MOM6-interface/MOM6
   url = https://github.com/NOAA-EMC/MOM6

CMEPS-interface/CMakeLists.txt

@@ -64,13 +64,9 @@ list(APPEND _mediator_files
   CMEPS/mediator/med_phases_post_rof_mod.F90
   CMEPS/mediator/med_phases_post_wav_mod.F90)
 
-list(APPEND _mediator_defs ESMF_VERSION_MAJOR=${ESMF_VERSION_MAJOR}
-                           ESMF_VERSION_MINOR=${ESMF_VERSION_MINOR})
-
 ### Create target library and set PUBLIC interfaces on the library
 add_library(cmeps STATIC ${_nems_util_files} ${_mediator_files})
 set_target_properties(cmeps PROPERTIES Fortran_MODULE_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/mod)
-target_compile_definitions(cmeps PUBLIC "${_mediator_defs}")
 target_include_directories(cmeps PUBLIC $<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}/mod>
                                         $<INSTALL_INTERFACE:mod>)
 target_link_libraries(cmeps PUBLIC esmf

tests/ci/Dockerfile

@@ -1,4 +1,4 @@
-From noaaemc/ubuntu-hpc:v1.3b
+From noaaemc/ubuntu-hpc:v1.4b
 
 ENV HOME=/home/builder
 COPY --chown=builder:builder . $HOME/ufs-weather-model

tests/ci/ci.sh

@@ -74,14 +74,14 @@ if [ $BUILD = "true" ]; then
 
 elif [ $RUN == "true" ]; then
 
-  docker run -d --rm -v DataVolume:/tmp minsukjinoaa/input-data:20210528 \
-    && docker rmi -f minsukjinoaa/input-data:20210528
+  docker volume rm -f DataVolume >/dev/null &&
+    docker run -d --rm -v DataVolume:/tmp minsukjinoaa/input-data:20210528
 
   docker create -u builder -e "CI_TEST=true" -e "USER=builder" \
                 -e "RT_MACHINE=linux.gnu" -e "RT_COMPILER=gnu" \
                 -w "/home/builder/ufs-weather-model/tests" \
                 -v DataVolume:/home/builder/data/NEMSfv3gfs/input-data-20210528 \
-                --shm-size=512m --name my-container noaaemc/ubuntu-hpc:v1.3b \
+                --shm-size=512m --name my-container noaaemc/ubuntu-hpc:v1.4b \
                 /bin/bash -c "./utest -n ${TEST_NAME} -c ${TEST_CASE} -x"
 
   cd $GITHUB_WORKSPACE

tests/ci/ci.test

@@ -1,4 +1,6 @@
+control
+thr mpi dcp rst bit dbg
 regional_control
 thr dcp
 cpld_control
-rst
+rst dbg

tests/ci/repo_check.sh

@@ -13,8 +13,8 @@ result() {
 }
 
 # Declare variables
-declare -A base fv3 mom6 cice ww3 stoch nems cmeps datm cdeps cmake
-submodules="fv3 mom6 cice ww3 stoch nems cmeps datm cdeps cmake"
+declare -A base fv3 mom6 cice ww3 stoch gocart nems cmeps datm cdeps cmake
+submodules="fv3 mom6 cice ww3 stoch gocart nems cmeps datm cdeps cmake"
 comment=''
 ownerID=$1
 
@@ -43,6 +43,10 @@ stoch[repo]='https://github.com/noaa-psd/stochastic_physics'
 stoch[branch]='master'
 stoch[dir]='stochastic_physics'
 
+gocart[repo]='https://github.com/GEOS-ESM/GOCART'
+gocart[branch]='develop'
+gocart[dir]='GOCART'
+
 nems[repo]='https://github.com/NOAA-EMC/NEMS'
 nems[branch]='develop'
 nems[dir]='NEMS'

tests/default_vars.sh

@@ -473,6 +473,7 @@ export EFFR_IN=.F.
 # Thompson MP
 export LRADAR=.T.
 export LTAEROSOL=.T.
+export EXT_DIAG_THOMPSON=.F.
 
 # GWD
 export LDIAG_UGWP=.F.

tests/fv3_conf/gfs_v16_run_c192L127.IN

@@ -20,4 +20,5 @@ cp    @[INPUTDATA_ROOT]/FV3_input_data_c192L127/*configure .
 
 if [ $DO_RRTMGP = .T. ]; then
   cp @[INPUTDATA_ROOT]/FV3_input_data_c192L127/rrtmgp* .
+  cp @[INPUTDATA_ROOT]/FV3_input_data_RRTMGP/* .
 fi

tests/fv3_conf/regional_run.IN

@@ -44,5 +44,6 @@ elif [[ $POSTAPP = 'hafs' ]]; then
 fi
 
 if [ $DO_RRTMGP = .T. ]; then
+  cp @[INPUTDATA_ROOT]/FV3_input_data_c192L127/rrtmgp* .
   cp @[INPUTDATA_ROOT]/FV3_input_data_RRTMGP/* .
 fi

tests/parm/control_rrtmgp.nml.IN

@@ -120,7 +120,7 @@ deflate_level=1
   fhlwr        = 3600.
   ialb         = 1
   iems         = 1
-  iaer         = 5111
+  iaer         = @[IAER]
   icliq_sw     = 2
   iovr         = 3
   ico2         = 2
@@ -164,7 +164,7 @@ deflate_level=1
   debug        = .false.
   oz_phys      = .F.
   oz_phys_2015 = .T.
-  nstf_name    = 2,0,0,0,0
+  nstf_name    = @[NSTF_NAME]
   nst_anl      = .true.
   psautco      = 0.0008,0.0005
   prautco      = 0.00015,0.00015
@@ -189,8 +189,12 @@ deflate_level=1
   rrtmgp_nGptsLW     = 128
   rrtmgp_nBandsLW    = 16
   rrtmgp_nBandsSW    = 14
-  doG_cldoptics      = .true.
-  rrtmgp_nrghice = 3
+  doGP_cldoptics_LUT = .true.
+  doGP_lwscat        = .true.
+  use_LW_jacobian    = .true.
+  damp_LW_fluxadj    = .true.
+  lfnc_k             = 3125.
+  lfnc_p0            = 60000.
   lsoil_lsm    = 4
   do_mynnedmf  = .false.
   do_mynnsfclay = .false.

tests/parm/control_thompson.nml.IN

@@ -118,6 +118,7 @@ deflate_level=1
   ltaerosol    = @[LTAEROSOL]
   lradar       = @[LRADAR]
   ttendlim     = 0.004
+  ext_diag_thompson = @[EXT_DIAG_THOMPSON]
   pdfcld       = .false.
   fhswr        = 3600.
   fhlwr        = 3600.

tests/parm/regional_rrtmgp.nml.IN

@@ -138,7 +138,7 @@
        fhlwr          = 900.
        ialb           = 1
        iems           = 1
-       IAER           = 111
+       IAER           = 5111
        ico2           = 2
        isubc_sw       = 2
        isubc_lw       = 2
@@ -213,6 +213,9 @@
        doGP_cldoptics_LUT = .true.
        doGP_lwscat        = .true.
        use_LW_jacobian    = .true.
+       damp_LW_fluxadj    = .true.
+       lfnc_k             = 3125.
+       lfnc_p0            = 60000.
 /
 
   &interpolator_nml

tests/rt.conf

@@ -129,6 +129,7 @@ RUN     | control_ras_debug
 COMPILE | -DAPP=ATM -DDEBUG=ON -DCCPP_SUITES=FV3_GFS_v16_thompson -D32BIT=ON                                                      |                                         | fv3 |
 RUN     | control_thompson_debug                                                                                                  |                                         | fv3 |
 RUN     | control_thompson_no_aero_debug                                                                                          |                                         | fv3 |
+RUN     | control_thompson_extdiag_debug                                                                                          |                                         | fv3 |
 
 COMPILE | -DAPP=ATM -DCCPP_SUITES=FV3_GFS_v15_thompson_mynn,FV3_GSD_v0,FV3_RRFS_v1beta,FV3_RRFS_v1alpha -D32BIT=ON -DDEBUG=ON     |                                         | fv3 |
 RUN     | regional_control_debug                                                                                                  |                                         | fv3 |

tests/rt.sh

@@ -414,7 +414,7 @@ if [[ $TESTS_FILE =~ '35d' ]]; then
   TEST_35D=true
 fi
 
-BL_DATE=20210706
+BL_DATE=20210708
 if [[ $MACHINE_ID = hera.* ]] || [[ $MACHINE_ID = orion.* ]] || [[ $MACHINE_ID = cheyenne.* ]] || [[ $MACHINE_ID = gaea.* ]] || [[ $MACHINE_ID = jet.* ]]; then
   RTPWD=${RTPWD:-$DISKNM/NEMSfv3gfs/develop-${BL_DATE}/${RT_COMPILER^^}}
 else

tests/rt_gnu.conf

@@ -42,6 +42,7 @@ RUN     | fv3_rrfs_v1beta_debug
 RUN     | fv3_gsd_debug                                                                                                           |                | fv3         |
 RUN     | control_thompson_debug                                                                                                  |                | fv3         |
 RUN     | control_thompson_no_aero_debug                                                                                          |                | fv3         |
+RUN     | control_thompson_extdiag_debug                                                                                          |                | fv3         |
 RUN     | control_rrtmgp_debug                                                                                                    |                | fv3         |
 RUN     | control_ugwpv1_debug                                                                                                    |                | fv3         |
 RUN     | control_ras_debug                                                                                                       |                | fv3         |

tests/tests/control_thompson_extdiag_debug

@@ -0,0 +1,49 @@
+###############################################################################
+#
+#  Global control with Thompson debug test with extended diagnostics
+#
+###############################################################################
+
+export TEST_DESCR="Compare global control Thompson debug results with extended diagnostics with previous trunk version"
+
+export CNTL_DIR=control_thompson_debug_extdiag
+
+export LIST_FILES="sfcf000.nc \
+                   sfcf001.nc \
+                   atmf000.nc \
+                   atmf001.nc"
+
+export_fv3
+export NPZ=127
+export NPZP=128
+export DT_ATMOS=600
+export SYEAR=2021
+export SMONTH=03
+export SDAY=22
+export SHOUR=06
+export OUTPUT_GRID='gaussian_grid'
+export NSTF_NAME='2,0,0,0,0'
+export FHMAX=1
+
+export IMP_PHYSICS=8
+export DNATS=0
+export DO_SAT_ADJ=.F.
+export LRADAR=.T.
+export LTAEROSOL=.T.
+export EXT_DIAG_THOMPSON=.T.
+export HYBEDMF=.F.
+export SATMEDMF=.T.
+export DO_MYNNEDMF=.F.
+export IMFSHALCNV=2
+export IMFDEEPCNV=2
+export IAER=5111
+export ICLIQ_SW=2
+export IOVR=3
+export LHEATSTRG=.T.
+export DO_TOFD=.T.
+
+export NEW_DIAGTABLE='diag_table_gfsv16_thompson_extdiag'
+
+export FV3_RUN=control_run.IN
+export CCPP_SUITE=FV3_GFS_v16_thompson
+export INPUT_NML=control_thompson.nml.IN

tests/utest

@@ -79,16 +79,14 @@ build_utests() {
         MAKE_OPT=$base_opt
         ;;
       bit)
-        if [[ $base_opt =~ "32BIT=Y" ]]; then
-          MAKE_OPT=$(echo $base_opt | sed -e 's/32BIT=Y/32BIT=N/')
-        elif [[ ${base_opt} =~ "32BIT=N" ]]; then
-          MAKE_OPT=$(echo $base_opt | sed -e 's/32BIT=N/32BIT=Y/')
+        if [[ $base_opt =~ "-D32BIT=ON" ]]; then
+          MAKE_OPT=$(echo $base_opt | sed -e 's/-D32BIT=ON//')
         else
-          MAKE_OPT="$base_opt 32BIT=Y"
+          MAKE_OPT="$base_opt -D32BIT=ON"
         fi
         ;;
       dbg)
-        MAKE_OPT="$base_opt 32BIT=Y DEBUG=Y"
+        MAKE_OPT="$base_opt -DDEBUG=ON"
         ;;
     esac
     MAKE_OPT=$(echo $MAKE_OPT | sed -e 's/^ *//' -e 's/ *$//')

tests/utests/dbg.sh

@@ -2,13 +2,26 @@ set -eu
 source $PATHRT/utests/std.sh
 
 if [[ $application == 'global' ]]; then
+  LIST_FILES="sfcf000.nc sfcf001.nc atmf000.nc atmf001.nc"
+  FHMAX=1
+  DAYS=0.041667
+  NFHOUT=3
+  NFHMAX_HF=12
+  NFHOUT_HF=1
   WLCLK=60
 elif [[ $application == 'regional' ]]; then
   echo "Regional application not yet implemented for debug"
   exit 1
 elif [[ $application == 'cpld' ]]; then
-  echo "Coupled application not yet implemented for debug"
-  exit 1
+  FHMAX=6
+  DAYS=0.25
+  NFHOUT_HF=1
+  RESTART_INTERVAL=${FHMAX}
+  RESTART_N=${FHMAX}
+  LIST_FILES=$(echo -n $LIST_FILES | sed -E "s/sfcf024/sfcf006/g" \
+                                   | sed -E "s/atmf024/atmf006/g" \
+                                   | sed -E "s/2016-10-04-00000/2016-10-03-21600/g" \
+                                   | sed -e "s/^ *//" -e "s/ *$//")
 fi
 
 source $PATHRT/utests/wrt_env.sh

tests/utests/mpi.sh

@@ -22,8 +22,3 @@ fi
 
 (test $CI_TEST == 'true') && source $PATHRT/utests/cmp_proc_bind.sh
 source $PATHRT/utests/wrt_env.sh
-
-cat <<EOF >>${RUNDIR_ROOT}/unit_test${RT_SUFFIX}.env
-export WRITE_GROUP=${WRITE_GROUP}
-export WRTTASK_PER_GROUP=${WRTTASK_PER_GROUP}
-EOF

tests/utests/rst.sh

@@ -45,7 +45,6 @@ export USE_RESTART_TIME=${USE_RESTART_TIME:-}
 export MOM6_RESTART_SETTING=${MOM6_RESTART_SETTING:-}
 export RESTART_FILE_SUFFIX_HRS=${RESTART_FILE_SUFFIX_HRS:-}
 export RESTART_FILE_SUFFIX_SECS=${RESTART_FILE_SUFFIX_SECS:-}
-export RESTART_N=${RESTART_N:-}
 export DEP_RUN=${DEP_RUN:-}
 export WARM_START=${WARM_START}
 export NGGPS_IC=${NGGPS_IC}
@@ -54,5 +53,4 @@ export MAKE_NH=${MAKE_NH}
 export MOUNTAIN=${MOUNTAIN}
 export NA_INIT=${NA_INIT}
 export FDIAG=${FDIAG}
-export LIST_FILES="${LIST_FILES}"
 EOF

tests/utests/std.sh

@@ -4,8 +4,13 @@ if [[ $application == 'global' ]]; then
   if [[ $CI_TEST == 'true' ]]; then
     INPES=3
     JNPES=2
+    WRITE_GROUP=1
+    WRTTASK_PER_GROUP=12
     TASKS=$((INPES*JNPES*6 + WRITE_GROUP*WRTTASK_PER_GROUP))
     RESTART_INTERVAL=$((FHMAX/2))
+    NFHOUT=6
+    NFHMAX_HF=-1
+    NFHOUT_HF=-1
   fi
 elif [[ $application == 'regional' ]]; then
   if [[ $CI_TEST == 'true' ]]; then
@@ -23,6 +28,8 @@ elif [[ $application == 'cpld' ]]; then
     ocn_petlist_bounds="30 39"
     ice_petlist_bounds="40 45"
     TASKS=$((INPES*JNPES*6 + WRITE_GROUP*WRTTASK_PER_GROUP + 10 + 6))
+    RESTART_INTERVAL=12
+    RESTART_N=${RESTART_INTERVAL}
   fi
 fi
 

tests/utests/wrt_env.sh

@@ -4,9 +4,12 @@ export CI_TEST=${CI_TEST}
 export RT_COMPILER=${RT_COMPILER}
 export FHMAX=${FHMAX}
 export DAYS=${DAYS}
-export RESTART_INTERVAL=${RESTART_INTERVAL}
+export RESTART_INTERVAL=${RESTART_INTERVAL:-}
+export RESTART_N=${RESTART_N:-}
 export INPES=${INPES}
 export JNPES=${JNPES}
+export WRITE_GROUP=${WRITE_GROUP}
+export WRTTASK_PER_GROUP=${WRTTASK_PER_GROUP}
 export NPROC_ICE=${NPROC_ICE:-}
 export med_petlist_bounds="${med_petlist_bounds:-}"
 export atm_petlist_bounds="${atm_petlist_bounds:-}"
@@ -18,4 +21,8 @@ export TPN=${TPN}
 export NODES=${NODES}
 export OMP_ENV="${OMP_ENV:-}"
 export MPI_PROC_BIND="${MPI_PROC_BIND:-}"
+export NFHOUT=${NFHOUT}
+export NFHMAX_HF=${NFHMAX_HF}
+export NFHOUT_HF=${NFHOUT_HF}
+export LIST_FILES="${LIST_FILES}"
 EOF