Merge pull request ufs-community#50 from climbfuji/update_dtc_develop…

…_from_ncar_master_20200606

Update dtc develop from ncar master 2020/06/06

.gitmodules

@@ -17,4 +17,4 @@
 [submodule "stochastic_physics"]
 	path = stochastic_physics
 	url = https://github.com/noaa-psd/stochastic_physics
-
+	branch = master

CCPP.appBuilder

@@ -4,7 +4,7 @@
 
 COMPONENTS=( CCPP FMS FV3 )
 
-FV3_MAKEOPT="CCPP=Y STATIC=Y SUITES=FV3_GFS_2017"
+FV3_MAKEOPT="CCPP=Y"
 
 # The modules.nems and configure.nems are selected by
 # conf/before_appbuilder_file.mk.

CCPP_repro.appBuilder

@@ -4,7 +4,7 @@
 
 COMPONENTS=( CCPP FMS FV3 )
 
-FV3_MAKEOPT="REPRO=Y CCPP=Y STATIC=Y SUITES=FV3_GFS_2017"
+FV3_MAKEOPT="REPRO=Y CCPP=Y"
 
 # The modules.nems and configure.nems are selected by
 # conf/before_appbuilder_file.mk.

CMakeLists.txt

@@ -1,4 +1,4 @@
-cmake_minimum_required(VERSION 3.0)
+cmake_minimum_required(VERSION 3.15)
 
 foreach(env_var IN ITEMS
          CMAKE_C_COMPILER CMAKE_CXX_COMPILER CMAKE_Fortran_COMPILER
@@ -23,13 +23,6 @@ find_package(ESMF REQUIRED)
 
 include(${PROJECT_SOURCE_DIR}/cmake/configure_${CMAKE_Platform}.cmake)
 
-if(NOT DEFINED PHYS)
-  set(PHYS gfs)
-endif()
-message("")
-message("Selected physics package: ${PHYS}")
-message("")
-
 add_definitions(-Duse_libMPI)
 add_definitions(-Duse_netCDF)
 add_definitions(-Duse_WRTCOMP)
@@ -43,6 +36,62 @@ add_definitions(-DUSE_COND)
 add_definitions(-DNEW_TAUCTMAX)
 add_definitions(-DINTERNAL_FILE_NML)
 
+if (CMAKE_Fortran_COMPILER_ID MATCHES "GNU" AND CMAKE_Fortran_COMPILER_VERSION VERSION_LESS 9.0.0)        
+  message(FATAL_ERROR "GNU Compiler >= 9 is required")
+endif()
+
+if(CCPP)
+
+    find_package(Python 3 QUIET COMPONENTS Interpreter)
+    if (NOT Python_Interpreter_FOUND)
+      find_package(Python 2.7 QUIET REQUIRED COMPONENTS Interpreter)
+    endif()
+    message("Found Python: ${Python_EXECUTABLE}")
+
+    if(DEFINED CCPP_SUITES)
+        message("Calling CCPP code generator (ccpp_prebuild.py) for SUITES = ${CCPP_SUITES}")
+        execute_process(COMMAND FV3/ccpp/framework/scripts/ccpp_prebuild.py
+                                "--config=FV3/ccpp/config/ccpp_prebuild_config.py"
+                                "--suites=${CCPP_SUITES}"
+                                "--builddir=${PROJECT_BINARY_DIR}/FV3"
+                        WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}
+                        OUTPUT_FILE ${PROJECT_BINARY_DIR}/ccpp_prebuild.out
+                        ERROR_FILE ${PROJECT_BINARY_DIR}/ccpp_prebuild.err
+                        RESULT_VARIABLE RC
+                       )
+    else()
+        message("Calling CCPP code generator (ccpp_prebuild.py) ...")
+        execute_process(COMMAND FV3/ccpp/framework/scripts/ccpp_prebuild.py
+                                "--config=FV3/ccpp/config/ccpp_prebuild_config.py"
+                                "--builddir=${PROJECT_BINARY_DIR}/FV3"
+                        WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}
+                        OUTPUT_FILE ${PROJECT_BINARY_DIR}/ccpp_prebuild.out
+                        ERROR_FILE ${PROJECT_BINARY_DIR}/ccpp_prebuild.err
+                        RESULT_VARIABLE RC
+                       )
+    endif()
+
+    # Check return code from ccpp_prebuild.py
+    if(RC EQUAL 0)
+        message("")
+    else()
+        message(FATAL_ERROR "An error occured while running ccpp_prebuild.py, check ${PROJECT_BINARY_DIR}/ccpp_prebuild.{out,err}")
+    endif()
+
+    # this should not be necessary; including CCPP_*.cmake here and passing
+    # SCHEMES, CAPS and TYPEDEFS via environment variables to CCPP build.
+    # CCPP should be able to directly include those three .cmake files.
+
+    include(${PROJECT_BINARY_DIR}/FV3/ccpp/physics/CCPP_SCHEMES.cmake)
+    include(${PROJECT_BINARY_DIR}/FV3/ccpp/physics/CCPP_CAPS.cmake)
+    include(${PROJECT_BINARY_DIR}/FV3/ccpp/physics/CCPP_TYPEDEFS.cmake)
+
+    set(ENV{CCPP_SCHEMES} "${SCHEMES}")
+    set(ENV{CCPP_CAPS} "${CAPS}")
+    set(ENV{CCPP_TYPEDEFS} "${TYPEDEFS}")
+
+endif()
+
 ###############################################################################
 ### FMS
 ###############################################################################
@@ -154,6 +203,9 @@ if(WW3)
   if(${CMAKE_Platform} STREQUAL "hera.intel")
     set(WW3_COMP "hera")
   endif()
+  if(${CMAKE_Platform} STREQUAL "orion.intel")
+    set(WW3_COMP "orion")
+  endif()
   message("Build WW3:")
   message("   run: ${CMAKE_BUILD_TOOL} WW3_PARCOMPN=4 WW3_COMP=${WW3_COMP} ww3_nems")
   message("   in:  ${PROJECT_SOURCE_DIR}/WW3/model/esmf")
@@ -223,7 +275,7 @@ target_link_libraries(NEMS.exe
                       fv3core
                       io
                       ${IPD_LIBRARIES}
-                      ${PHYS}physics
+                      gfsphysics
                       ${CCPP_LIBRARIES}
                       fv3cpl
                       stochastic_physics

build.sh

@@ -23,18 +23,7 @@ rm -rf ${BUILD_DIR}
 mkdir ${BUILD_DIR}
 
 CCPP_SUITES="${CCPP_SUITES:-FV3_GFS_2017_gfdlmp}"
-
-./FV3/ccpp/framework/scripts/ccpp_prebuild.py \
-    --config=FV3/ccpp/config/ccpp_prebuild_config.py \
-    --static \
-    --suites=${CCPP_SUITES} \
-    --builddir=${BUILD_DIR}/FV3 > ${BUILD_DIR}/ccpp_prebuild.log 2>&1
-
-source ${BUILD_DIR}/FV3/ccpp/physics/CCPP_SCHEMES.sh
-source ${BUILD_DIR}/FV3/ccpp/physics/CCPP_CAPS.sh
-source ${BUILD_DIR}/FV3/ccpp/physics/CCPP_STATIC_API.sh
-
-CMAKE_FLAGS+=" -DCCPP=ON -DSTATIC=ON -DSUITES=${CCPP_SUITES} -DNETCDF_DIR=${NETCDF}"
+CMAKE_FLAGS+=" -DCCPP=ON -DSUITES=${CCPP_SUITES} -DNETCDF_DIR=${NETCDF}"
 
 cd ${BUILD_DIR}
 cmake .. ${CMAKE_FLAGS}

cmake/configure_orion.intel.cmake

@@ -0,0 +1,35 @@
+message("")
+message("Setting configuration for $ENV{CMAKE_Platform}")
+message("")
+
+get_filename_component (C_COMPILER_NAME ${CMAKE_C_COMPILER} NAME)
+get_filename_component (CXX_COMPILER_NAME ${CMAKE_CXX_COMPILER} NAME)
+get_filename_component (Fortran_COMPILER_NAME ${CMAKE_Fortran_COMPILER} NAME)
+message("C       compiler: ${CMAKE_C_COMPILER_ID} ${CMAKE_C_COMPILER_VERSION} (${C_COMPILER_NAME})")
+message("CXX     compiler: ${CMAKE_CXX_COMPILER_ID} ${CMAKE_CXX_COMPILER_VERSION} (${CXX_COMPILER_NAME})")
+message("Fortran compiler: ${CMAKE_Fortran_COMPILER_ID} ${CMAKE_Fortran_COMPILER_VERSION} (${Fortran_COMPILER_NAME})")
+message("")
+
+option(DEBUG   "Enable DEBUG mode" OFF)
+option(REPRO   "Enable REPRO mode" OFF)
+option(VERBOSE "Enable VERBOSE mode" OFF)
+option(32BIT   "Enable 32BIT (single precision arithmetic in dycore)" OFF)
+option(OPENMP  "Enable OpenMP threading" ON)
+option(AVX2    "Enable AVX2 instruction set" ON)
+
+option(INLINE_POST "Enable inline post" ON)
+
+include( cmake/${CMAKE_Fortran_COMPILER_ID}.cmake )
+
+set(NEMSIO_INC $ENV{NEMSIO_INC})
+set(POST_INC $ENV{POST_INC})
+set(NCEP_LIBS $ENV{POST_LIB} $ENV{NEMSIO_LIB} $ENV{G2_LIB4} $ENV{G2TMPL_LIB} $ENV{BACIO_LIB4} $ENV{SP_LIBd} $ENV{W3EMC_LIBd} $ENV{W3NCO_LIBd} $ENV{CRTM_LIB} $ENV{PNG_LIB} $ENV{JASPER_LIB} $ENV{Z_LIB})
+
+set(ESMF_MOD ${ESMF_F90COMPILEPATHS})
+set(ESMF_LIBS "${ESMF_F90ESMFLINKRPATHS} ${ESMF_F90ESMFLINKPATHS} ${ESMF_F90ESMFLINKLIBS}")
+
+set(NETCDF_INC_DIR $ENV{NETCDF}/include)
+set(NETCDF_LIBDIR $ENV{NETCDF}/lib)
+set(NETCDF_LIBS -L$ENV{NETCDF}/lib -lnetcdff -lnetcdf)
+
+message("")

cmake/configure_stampede.intel.cmake

@@ -0,0 +1,35 @@
+message("")
+message("Setting configuration for $ENV{CMAKE_Platform}")
+message("")
+
+get_filename_component (C_COMPILER_NAME ${CMAKE_C_COMPILER} NAME)
+get_filename_component (CXX_COMPILER_NAME ${CMAKE_CXX_COMPILER} NAME)
+get_filename_component (Fortran_COMPILER_NAME ${CMAKE_Fortran_COMPILER} NAME)
+message("C       compiler: ${CMAKE_C_COMPILER_ID} ${CMAKE_C_COMPILER_VERSION} (${C_COMPILER_NAME})")
+message("CXX     compiler: ${CMAKE_CXX_COMPILER_ID} ${CMAKE_CXX_COMPILER_VERSION} (${CXX_COMPILER_NAME})")
+message("Fortran compiler: ${CMAKE_Fortran_COMPILER_ID} ${CMAKE_Fortran_COMPILER_VERSION} (${Fortran_COMPILER_NAME})")
+message("")
+
+option(DEBUG   "Enable DEBUG mode" OFF)
+option(REPRO   "Enable REPRO mode" OFF)
+option(VERBOSE "Enable VERBOSE mode" OFF)
+option(32BIT   "Enable 32BIT (single precision arithmetic in dycore)" OFF)
+option(OPENMP  "Enable OpenMP threading" ON)
+option(AVX2    "Enable AVX2 instruction set" ON)
+
+option(INLINE_POST "Enable inline post" OFF)
+
+include( cmake/${CMAKE_Fortran_COMPILER_ID}.cmake )
+
+set(NEMSIO_INC $ENV{NEMSIO_INC})
+set(POST_INC $ENV{POST_INC})
+set(NCEP_LIBS $ENV{POST_LIB} $ENV{NEMSIO_LIB} $ENV{G2_LIB4} $ENV{G2TMPL_LIB} $ENV{BACIO_LIB4} $ENV{SP_LIBd} $ENV{W3EMC_LIBd} $ENV{W3NCO_LIBd} $ENV{CRTM_LIB} $ENV{PNG_LIB} $ENV{JASPER_LIB} $ENV{Z_LIB})
+
+set(ESMF_MOD ${ESMF_F90COMPILEPATHS})
+set(ESMF_LIBS "${ESMF_F90ESMFLINKRPATHS} ${ESMF_F90ESMFLINKPATHS} ${ESMF_F90ESMFLINKLIBS}")
+
+set(NETCDF_INC_DIR $ENV{NETCDF}/include)
+set(NETCDF_LIBDIR $ENV{NETCDF}/lib)
+set(NETCDF_LIBS -L$ENV{NETCDF}/lib -lnetcdff -lnetcdf)
+
+message("")

compsets/fv3.input

@@ -92,7 +92,7 @@ build fv3.exe {
 }
 
 build fv3_ccpp_control.exe {
-    # This block builds the FV3 with CCPP in static mode for the control setup.
+    # This block builds the FV3 with CCPP for the control setup.
     # It is otherwise identical to the fv3.exe block.
     use plat
 
@@ -106,7 +106,7 @@ build fv3_ccpp_control.exe {
     # build: script to build the NEMS.x.  For now, this is an embedded
     # bash script.
     build=compile.sh(fv3="@[target]",modules="@[modules.nems]",md5="@[md5sum]",
-                     MAKE_OPTS="\'CCPP=Y STATIC=Y SUITES=FV3_GFS_2017\'")
+                     MAKE_OPTS="\'CCPP=Y SUITES=FV3_GFS_2017\'")
 }
 
 build fv3_32bit.exe {
@@ -266,6 +266,10 @@ fv3_defaults = {
     IAU_INC_FILES="''"
     IAU_DRYMASSFIXER='.false.'
 
+    DO_CA='.F.'
+    CA_SGS='.F.'
+    CA_GLOBAL='.F.'
+
     CPLFLX='.F.'
     CPLWAV='.F.'
     CPLWAV2ATM='.F.'
@@ -1422,6 +1426,19 @@ test fv3_wrtGauss_netcdf: fv3.exe {
     }
 }
 
+test fv3_wrtGlatlon_netcdf: fv3.exe {
+    use fv3_wrtGauss_netcdf
+
+    TEST_DESCR="Compare FV3 global latlon grid netcdf output results with previous trunk version"
+    CNTL_NAME="fv3_wrtGlatlon_netcdf"
+
+    OUTPUT_GRID="'global_latlon'"
+
+    COM="@[plat%COMrt]/@[TEST_NAME]"          # Test result area
+    RUNDIR="@[plat%TMPrt]/@[TEST_NAME]"       # Test work area
+    CNTL="@[plat%BASELINE]/@[CNTL_NAME]"      # Control baseline area
+}
+
 ########################################################################
 
 test fv3_satmedmf: fv3.exe {
@@ -1432,6 +1449,7 @@ test fv3_satmedmf: fv3.exe {
 
     SATMEDMF='.true.'
     HYBEDMF='.false.'
+    DT_ATMOS=1200
     OUTPUT_GRID="'gaussian_grid'"
     OUTPUT_FILE="'nemsio'"
     WRITE_NEMSIOFLIP='.true.'

compsets/hera.input

@@ -31,8 +31,8 @@ platform hera.intel {
     # INPUTS is the input directory, which should contain fix and parm
     # files, plus any restarts or other inputs.
 
-    BASELINE="/scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200323/INTEL"
-    BASELINE_TEMPLATE="/scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200323/INTEL"
+    BASELINE="/scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200424/INTEL"
+    BASELINE_TEMPLATE="/scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200424/INTEL"
     INPUTS="@[BASELINE]"
 
     default_resources={

compsets/wcoss.input

@@ -28,8 +28,8 @@ platform wcoss.phase2 {
     LONG_TEST_QUEUE='&LONGQ;'
     BUILD_QUEUE='&BUILDQ;'
     MACHINE_ID='wcoss.phase2'
-    BASELINE="/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200323"
-    BASELINE_TEMPLATE="/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200323"
+    BASELINE="/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200424"
+    BASELINE_TEMPLATE="/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200424"
 
     default_resources={
         TASKS=156
@@ -126,8 +126,8 @@ platform wcoss.phase1 {
     C768_THRD=4
 
     MACHINE_ID='wcoss.phase1'
-    BASELINE="/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200323"
-    BASELINE_TEMPLATE="/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200323"
+    BASELINE="/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200424"
+    BASELINE_TEMPLATE="/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200424"
 
     # WCOSS Phase 2 has been slow of late.
     DEFAULT_TEST_WALLTIME=2700
@@ -213,8 +213,8 @@ platform wcoss_dell_p3 {
     LONG_TEST_QUEUE='&LONGQ;'
     BUILD_QUEUE='&BUILDQ;'
 
-    BASELINE="/gpfs/dell2/emc/modeling/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200323/"
-    BASELINE_TEMPLATE="/gpfs/dell2/emc/modeling/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200323/"
+    BASELINE="/gpfs/dell2/emc/modeling/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200424/"
+    BASELINE_TEMPLATE="/gpfs/dell2/emc/modeling/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200424/"
 
     default_resources={
         TASKS=156
@@ -314,8 +314,8 @@ platform wcoss.cray {
     BUILD_WALLTIME="3600"
     DEFAULT_TEST_WALLTIME=1800
 
-    BASELINE="/gpfs/hps3/emc/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200323"
-    BASELINE_TEMPLATE="/gpfs/hps3/emc/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200323"
+    BASELINE="/gpfs/hps3/emc/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200424"
+    BASELINE_TEMPLATE="/gpfs/hps3/emc/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200424"
 
     execution_time_modules=[[[
       module load alps

conf/before_components.mk

@@ -16,23 +16,8 @@
 
 CHOSEN_MODULE=$(BUILD_TARGET)/fv3
 
-ifneq (,$(findstring INTEL16=Y,$(FV3_MAKEOPT)))
-  ifeq ($(CHOSEN_MODULE),gaea.intel/fv3)
-    override CHOSEN_MODULE=$(BUILD_TARGET)/fv3.intel-16.0.3.210
-    $(warning Overriding CHOSEN_MODULE with $(CHOSEN_MODULE) as requested per MAKEOPT)
-  endif
-endif
-
 CONFIGURE_NEMS_FILE=configure.fv3.$(BUILD_TARGET)
 
-# ----------------------------------------------------------------------
-# Exit for systems that are currently not supported
-ifeq ($(BUILD_TARGET),theia.pgi)
-  $(error NEMSfv3gfs currently not supported on $(BUILD_TARGET))
-else ifeq ($(BUILD_TARGET),cheyenne.pgi)
-  $(error NEMSfv3gfs currently not supported on $(BUILD_TARGET))
-endif
-
 # ----------------------------------------------------------------------
 # Copy the executable and modules.nems files into the tests/ directory
 # if a TEST_BUILD_NAME is specified.

conf/configure.fv3.cheyenne.gnu

@@ -55,7 +55,6 @@ OPENMP = Y
 AVX2 = Y
 HYDRO = N
 CCPP = N
-STATIC = N
 SION = N
 
 include       $(ESMFMKFILE)
@@ -183,12 +182,7 @@ ifeq ($(CCPP),Y)
 CPPDEFS += -DCCPP
 CFLAGS += -I$(PATH_CCPP)/include
 FFLAGS += -I$(PATH_CCPP)/include
-ifeq ($(STATIC),Y)
-CPPDEFS += -DSTATIC
 LDFLAGS += -L$(PATH_CCPP)/lib -lccppphys -lccpp $(NCEPLIBS) -lxml2
-else
-LDFLAGS += -L$(PATH_CCPP)/lib -lccpp
-endif
 endif
 
 ifeq ($(SION),Y)