NCAR · climbfuji · Mar 17, 2020 · Mar 16, 2020 · Mar 16, 2020 · Mar 16, 2020
diff --git a/.gitmodules b/.gitmodules
@@ -4,8 +4,10 @@
 	branch = dtc/develop
 [submodule "NEMS"]
 	path = NEMS
-	url = https://github.com/NOAA-EMC/NEMS
-	branch = develop
+	#url = https://github.com/NOAA-EMC/NEMS
+	#branch = develop
+	url = https://github.com/climbfuji/NEMS
+	branch = final_pr_before_merging_to_develop_or_master_20200313
 [submodule "FMS"]
 	path = FMS
 	url = https://github.com/NOAA-GFDL/FMS

diff --git a/FV3 b/FV3
diff --git a/NEMS b/NEMS
diff --git a/cmake/configure_hera.gnu.cmake b/cmake/configure_hera.gnu.cmake
@@ -0,0 +1,35 @@
+message("")
+message("Setting configuration for $ENV{CMAKE_Platform}")
+message("")
+
+get_filename_component (C_COMPILER_NAME ${CMAKE_C_COMPILER} NAME)
+get_filename_component (CXX_COMPILER_NAME ${CMAKE_CXX_COMPILER} NAME)
+get_filename_component (Fortran_COMPILER_NAME ${CMAKE_Fortran_COMPILER} NAME)
+message("C       compiler: ${CMAKE_C_COMPILER_ID} ${CMAKE_C_COMPILER_VERSION} (${C_COMPILER_NAME})")
+message("CXX     compiler: ${CMAKE_CXX_COMPILER_ID} ${CMAKE_CXX_COMPILER_VERSION} (${CXX_COMPILER_NAME})")
+message("Fortran compiler: ${CMAKE_Fortran_COMPILER_ID} ${CMAKE_Fortran_COMPILER_VERSION} (${Fortran_COMPILER_NAME})")
+message("")
+
+option(DEBUG   "Enable DEBUG mode" OFF)
+option(REPRO   "Enable REPRO mode" OFF)
+option(VERBOSE "Enable VERBOSE mode" OFF)
+option(32BIT   "Enable 32BIT (single precision arithmetic in dycore)" OFF)
+option(OPENMP  "Enable OpenMP threading" ON)
+option(AVX2    "Enable AVX2 instruction set" OFF)
+
+option(INLINE_POST "Enable inline post" OFF)
+
+include( cmake/${CMAKE_Fortran_COMPILER_ID}.cmake )
+
+set(NEMSIO_INC $ENV{NEMSIO_INC})
+set(POST_INC $ENV{POST_INC})
+set(NCEP_LIBS $ENV{POST_LIB} $ENV{NEMSIO_LIB} $ENV{G2_LIB4} $ENV{G2TMPL_LIB} $ENV{BACIO_LIB4} $ENV{SP_LIBd} $ENV{W3EMC_LIBd} $ENV{W3NCO_LIBd} $ENV{CRTM_LIB} $ENV{PNG_LIB} $ENV{JASPER_LIB} $ENV{Z_LIB})
+
+set(ESMF_MOD ${ESMF_F90COMPILEPATHS})
+set(ESMF_LIBS "${ESMF_F90ESMFLINKRPATHS} ${ESMF_F90ESMFLINKPATHS} ${ESMF_F90ESMFLINKLIBS}")
+
+set(NETCDF_INC_DIR $ENV{NETCDF}/include)
+set(NETCDF_LIBDIR $ENV{NETCDF}/lib)
+set(NETCDF_LIBS -L$ENV{NETCDF}/lib -lnetcdff -lnetcdf)
+
+message("")
diff --git a/compsets/hera.input b/compsets/hera.input
@@ -31,8 +31,8 @@ platform hera.intel {
     # INPUTS is the input directory, which should contain fix and parm
     # files, plus any restarts or other inputs.
 
-    BASELINE="/scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200313"
-    BASELINE_TEMPLATE="/scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200313"
+    BASELINE="/scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200317/INTEL"
+    BASELINE_TEMPLATE="/scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200317/INTEL"
     INPUTS="@[BASELINE]"
 
     default_resources={

diff --git a/compsets/wcoss.input b/compsets/wcoss.input
@@ -28,8 +28,8 @@ platform wcoss.phase2 {
     LONG_TEST_QUEUE='&LONGQ;'
     BUILD_QUEUE='&BUILDQ;'
     MACHINE_ID='wcoss.phase2'
-    BASELINE="/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200313"
-    BASELINE_TEMPLATE="/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200313"
+    BASELINE="/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200317"
+    BASELINE_TEMPLATE="/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200317"
 
     default_resources={
         TASKS=156
@@ -126,8 +126,8 @@ platform wcoss.phase1 {
     C768_THRD=4
 
     MACHINE_ID='wcoss.phase1'
-    BASELINE="/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200313"
-    BASELINE_TEMPLATE="/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200313"
+    BASELINE="/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200317"
+    BASELINE_TEMPLATE="/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200317"
 
     # WCOSS Phase 2 has been slow of late.
     DEFAULT_TEST_WALLTIME=2700
@@ -213,8 +213,8 @@ platform wcoss_dell_p3 {
     LONG_TEST_QUEUE='&LONGQ;'
     BUILD_QUEUE='&BUILDQ;'
 
-    BASELINE="/gpfs/dell2/emc/modeling/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200313/"
-    BASELINE_TEMPLATE="/gpfs/dell2/emc/modeling/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200313/"
+    BASELINE="/gpfs/dell2/emc/modeling/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200317/"
+    BASELINE_TEMPLATE="/gpfs/dell2/emc/modeling/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200317/"
 
     default_resources={
         TASKS=156
@@ -314,8 +314,8 @@ platform wcoss.cray {
     BUILD_WALLTIME="3600"
     DEFAULT_TEST_WALLTIME=1800
 
-    BASELINE="/gpfs/hps3/emc/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200313"
-    BASELINE_TEMPLATE="/gpfs/hps3/emc/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200313"
+    BASELINE="/gpfs/hps3/emc/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200317"
+    BASELINE_TEMPLATE="/gpfs/hps3/emc/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200317"
 
     execution_time_modules=[[[
       module load alps

diff --git a/conf/configure.fv3.hera.gnu b/conf/configure.fv3.hera.gnu
@@ -0,0 +1,175 @@
+## NEMS configuration file
+##
+## Platform: Hera
+## Compiler: GNU with OpenMPI
+
+SHELL=/bin/sh
+
+################################################################################
+## Include the common configuration parts
+
+ifdef InNemsMakefile
+include $(TOP)/conf/configure.nems.NUOPC
+endif
+
+############
+# commands #
+############
+FC  = mpif90
+CC  = mpicc
+CXX = mpicxx
+LD  = $(FC)
+
+#########
+# flags #
+#########
+# default is 64-bit OpenMP non-hydrostatic build using AVX2
+DEBUG =
+REPRO =
+VERBOSE =
+OPENMP = Y
+AVX2 = Y
+HYDRO = N
+CCPP = N
+STATIC = N
+
+include       $(ESMFMKFILE)
+ESMF_INC    = $(ESMF_F90COMPILEPATHS)
+
+NEMSIOINC = -I$(NEMSIO_INC)
+NCEPLIBS = $(NEMSIO_LIB) $(BACIO_LIB4) $(SP_LIBd) $(W3EMC_LIBd) $(W3NCO_LIBd)
+
+##############################################
+# Need to use at least GNU Make version 3.81 #
+##############################################
+need := 3.81
+ok := $(filter $(need),$(firstword $(sort $(MAKE_VERSION) $(need))))
+ifneq ($(need),$(ok))
+$(error Need at least make version $(need).  Load module gmake/3.81)
+endif
+
+NETCDF_ROOT = $(NETCDF)
+INCLUDE = -I$(NETCDF_ROOT)/include
+NETCDF_INC = -I$(NETCDF_ROOT)/include
+ifneq ($(findstring netcdf/4,$(LOADEDMODULES)),)
+  NETCDF_LIB += -L$(NETCDF)/lib -lnetcdff -lnetcdf
+else
+  NETCDF_LIB += -L$(NETCDF)/lib -lnetcdff -lnetcdf
+endif
+
+FPPFLAGS := -cpp -Wp,-w $(INCLUDE)
+CFLAGS := $(INCLUDE)
+
+FFLAGS := $(INCLUDE) -fcray-pointer -ffree-line-length-none -fno-range-check -fbacktrace
+
+CPPDEFS += -Duse_libMPI -Duse_netCDF -DSPMD -DUSE_LOG_DIAG_FIELD_INFO  -DUSE_GFSL63 -DGFS_PHYS -Duse_WRTCOMP
+CPPDEFS += -DNEW_TAUCTMAX -DINTERNAL_FILE_NML -DNO_INLINE_POST
+
+ifeq ($(HYDRO),Y)
+CPPDEFS +=
+else
+CPPDEFS += -DMOIST_CAPPA -DUSE_COND
+endif
+
+ifeq ($(NAM_phys),Y)
+CPPDEFS += -DNAM_phys
+endif
+
+ifeq ($(32BIT),Y)
+CPPDEFS += -DOVERLOAD_R4 -DOVERLOAD_R8
+FFLAGS +=
+else
+FFLAGS += -fdefault-real-8 -fdefault-double-8
+endif
+
+ifeq ($(AVX2),Y)
+FFLAGS +=
+CFLAGS +=
+else
+FFLAGS +=
+CFLAGS +=
+endif
+
+ifeq ($(MULTI_GASES),Y)
+CPPDEFS += -DMULTI_GASES
+endif
+
+FFLAGS_OPT = -O2 -fno-range-check
+FFLAGS_REPRO = -O2 -g -fbacktrace -fno-range-check
+FFLAGS_DEBUG = -g -O0 -fno-unsafe-math-optimizations -frounding-math -fsignaling-nans -ffpe-trap=invalid,zero,overflow -fbounds-check -fbacktrace -fno-range-check -Wall
+
+TRANSCENDENTALS :=
+FFLAGS_OPENMP = -fopenmp
+FFLAGS_VERBOSE = -v -V
+
+CFLAGS += -D__IFC
+
+CFLAGS_OPT = -O2
+CFLAGS_REPRO = -O2
+CFLAGS_OPENMP = -fopenmp
+CFLAGS_DEBUG = -O0 -g
+
+# Optional Testing compile flags.  Mutually exclusive from DEBUG, REPRO, and OPT
+# *_TEST will match the production if no new option(s) is(are) to be tested.
+FFLAGS_TEST = -O3
+CFLAGS_TEST = -O2
+
+LDFLAGS :=
+LDFLAGS_OPENMP := -fopenmp
+LDFLAGS_VERBOSE := -Wl,-V,--verbose,-cref,-M
+
+# start with blank LIBS
+LIBS :=
+
+ifeq ($(REPRO),Y)
+CPPDEFS += -DREPRO
+CFLAGS += $(CFLAGS_REPRO)
+FFLAGS += $(FFLAGS_REPRO)
+FAST :=
+else ifeq ($(DEBUG),Y)
+CPPDEFS += -DDEBUG
+CFLAGS += $(CFLAGS_DEBUG)
+FFLAGS += $(FFLAGS_DEBUG)
+FAST :=
+else ifeq ($(TEST),Y)
+CFLAGS += $(CFLAGS_TEST)
+FFLAGS += $(FFLAGS_TEST)
+FAST :=
+else
+CFLAGS += $(CFLAGS_OPT)
+FFLAGS += $(FFLAGS_OPT)
+FAST := $(TRANSCENDENTALS)
+endif
+
+ifeq ($(OPENMP),Y)
+CPPDEFS += -DOPENMP
+CFLAGS += $(CFLAGS_OPENMP)
+FFLAGS += $(FFLAGS_OPENMP)
+LDFLAGS += $(LDFLAGS_OPENMP)
+endif
+
+ifeq ($(VERBOSE),Y)
+CFLAGS += $(CFLAGS_VERBOSE)
+FFLAGS += $(FFLAGS_VERBOSE)
+LDFLAGS += $(LDFLAGS_VERBOSE)
+endif
+
+ifeq ($(CCPP),Y)
+CPPDEFS += -DCCPP
+CFLAGS += -I$(PATH_CCPP)/include
+FFLAGS += -I$(PATH_CCPP)/include
+ifeq ($(STATIC),Y)
+CPPDEFS += -DSTATIC
+LDFLAGS += -L$(PATH_CCPP)/lib -lccppphys -lccpp $(NCEPLIBS) -lxml2
+else
+LDFLAGS += -L$(PATH_CCPP)/lib -lccpp
+endif
+endif
+
+LDFLAGS += $(LIBS)
+
+ifdef InNemsMakefile
+FFLAGS += $(ESMF_INC)
+CPPFLAGS += -cpp -traditional
+EXTLIBS = $(NCEPLIBS) $(ESMF_LIB) $(LDFLAGS) $(NETCDF_LIB)
+endif
diff --git a/modulefiles/hera.gnu/fv3 b/modulefiles/hera.gnu/fv3
@@ -0,0 +1,42 @@
+#%Module######################################################################
+##
+##    NEMS FV3 Prerequisites: Hera/GNU
+
+proc ModulesHelp {} {
+  puts stderr "\tcit - loads modules required for building and running FV3 under NEMS on Hera/GNU"
+}
+
+module-whatis "loads NEMS FV3 prerequisites for Hera/GNU"
+
+# NOTE: the "module purge" and loading of the module command are
+# handled by the module-setup.sh (or .csh) script.
+setenv NCEPLIBS /scratch2/NCEPDEV/nwprod/NCEPLIBS
+##
+## load contrib environment
+## load slurm utils (arbitrary.pl  layout.pl)
+##
+module load contrib sutils
+
+##
+## load programming environment
+## this typically includes compiler, MPI and job scheduler
+##
+module load gnu/9.2.0
+module load openmpi/3.1.4
+module load netcdf/4.7.2
+
+##
+## use pre-compiled EMSF library and NCEP libraries for above compiler / MPI combination
+##
+module use -a /scratch1/BMC/gmtb/software/modulefiles/gnu-9.2.0/openmpi-3.1.4
+module load NCEPlibs/1.0.0
+
+##
+## load cmake
+##
+module use -a /scratch1/BMC/gmtb/software/modulefiles/generic
+module load cmake/3.16.3
+setenv CMAKE_C_COMPILER mpicc
+setenv CMAKE_CXX_COMPILER mpicxx
+setenv CMAKE_Fortran_COMPILER mpif90
+setenv CMAKE_Platform hera.gnu
diff --git a/tests/rt.sh b/tests/rt.sh
@@ -315,7 +315,7 @@ mkdir -p ${STMP}/${USER}
 
 # Different own baseline directories for different compilers on Theia/Cheyenne
 NEW_BASELINE=${STMP}/${USER}/FV3_RT/REGRESSION_TEST
-if [[ $MACHINE_ID = theia.* ]] || [[ $MACHINE_ID = cheyenne.* ]] || [[ $MACHINE_ID = jet.* ]] || [[ $MACHINE_ID = gaea.* ]]; then
+if [[ $MACHINE_ID = hera.* ]] || [[ $MACHINE_ID = cheyenne.* ]]; then
     NEW_BASELINE=${NEW_BASELINE}_${COMPILER^^}
 fi
 
@@ -381,10 +381,13 @@ while getopts ":cfsl:mkreh" opt; do
   esac
 done
 
-if [[ $MACHINE_ID = cheyenne.* ]]; then
-  RTPWD=${RTPWD:-$DISKNM/develop-20200313/${COMPILER^^}}
+# Fix me - make those definitions and DISKNM consistent
+if [[ $MACHINE_ID = hera.* ]]; then
+  RTPWD=${RTPWD:-$DISKNM/NEMSfv3gfs/develop-20200317/${COMPILER^^}}
+elif [[ $MACHINE_ID = cheyenne.* ]]; then
+  RTPWD=${RTPWD:-$DISKNM/develop-20200317/${COMPILER^^}}
 else
-  RTPWD=${RTPWD:-$DISKNM/NEMSfv3gfs/develop-20200313}
+  RTPWD=${RTPWD:-$DISKNM/NEMSfv3gfs/develop-20200317}
 fi
 
 shift $((OPTIND-1))
@@ -552,11 +555,11 @@ while read -r line; do
       APP=''
       NEMS_VER=$(echo $line | cut -d'|' -f2 | sed -e 's/^ *//' -e 's/ *$//')
       SET=$(     echo $line | cut -d'|' -f3)
-      MACHINES=$(echo $line | cut -d'|' -f4 | sed -e 's/^ *//' -e 's/ *$//')
+      MACHINES=$(echo $line | cut -d'|' -f4)
       CB=$(      echo $line | cut -d'|' -f5)
 
       [[ $SET_ID != ' ' && $SET != *${SET_ID}* ]] && continue
-      [[ $MACHINES != ' ' && $MACHINES != "${MACHINE_ID}" ]] && continue
+      [[ $MACHINES != ' ' && $MACHINES != *${MACHINE_ID}* ]] && continue
       [[ $CREATE_BASELINE == true && $CB != *fv3* ]] && continue
 
       (( COMPILE_NR += 1 ))
@@ -591,11 +594,11 @@ while read -r line; do
 
       APP=$(     echo $line | cut -d'|' -f2 | sed -e 's/^ *//' -e 's/ *$//')
       SET=$(     echo $line | cut -d'|' -f3)
-      MACHINES=$(echo $line | cut -d'|' -f4 | sed -e 's/^ *//' -e 's/ *$//')
+      MACHINES=$(echo $line | cut -d'|' -f4)
       CB=$(      echo $line | cut -d'|' -f5)
 
       [[ $SET_ID != ' ' && $SET != *${SET_ID}* ]] && continue
-      [[ $MACHINES != ' ' && $MACHINES != "${MACHINE_ID}" ]] && continue
+      [[ $MACHINES != ' ' && $MACHINES != *${MACHINE_ID}* ]] && continue
       [[ $CREATE_BASELINE == true && $CB != *fv3* ]] && continue
       [[ ${ROCOTO} == true || ${ECFLOW} == true ]] && continue
+11 −8		atmos_model.F90
+7 −1		ccpp/build_ccpp.sh
+1 −1		ccpp/physics
+7 −0		ccpp/set_compilers.sh
+42 −50		gfsphysics/GFS_layer/GFS_physics_driver.F90
+1 −1		gfsphysics/GFS_layer/GFS_typedefs.F90
+57 −31		io/FV3GFS_io.F90
+1 −1		src/conf/module-setup.sh.inc
+7 −0		src/incmake/env/rdhpcs/hera.gnu.mk
+1 −1		src/module_NEMS_Rusage.F90