Skip to content

Commit

Permalink
Update EMC develop from NCAR dtc/develop 2020/03/17 (NOAA-EMC#84)
Browse files Browse the repository at this point in the history
* hera.gnu build target with regression test config rt_gnu.conf (same as used on Cheyenne); this initial version uses the UFS public release v1.0.0 NCEPLIBS
* re-enable IPD REPRO, CCPP REPRO, CCPP PROD regression tests; IPD REPRO and CCPP REPRO are bit-for-bit identical
* bugfixes for some CCPP regression tests (IAU dry mass fixer)
* addition of two-way coupled CCPP regression tests, removal of old IPD Thompson test, addition of several debug tests (Thompson non-aero and GSD suite) to cover more physics options in DEBUG mode
* change logic in rt.sh that COMPILE or APPBUILD lines without a machine name are executed on all machines (similar to what is done for the RUN lines)
  • Loading branch information
climbfuji authored Mar 19, 2020
1 parent 0a88706 commit 44604a0
Show file tree
Hide file tree
Showing 42 changed files with 145,092 additions and 22,913 deletions.
35 changes: 35 additions & 0 deletions cmake/configure_hera.gnu.cmake
Original file line number Diff line number Diff line change
@@ -0,0 +1,35 @@
message("")
message("Setting configuration for $ENV{CMAKE_Platform}")
message("")

get_filename_component (C_COMPILER_NAME ${CMAKE_C_COMPILER} NAME)
get_filename_component (CXX_COMPILER_NAME ${CMAKE_CXX_COMPILER} NAME)
get_filename_component (Fortran_COMPILER_NAME ${CMAKE_Fortran_COMPILER} NAME)
message("C compiler: ${CMAKE_C_COMPILER_ID} ${CMAKE_C_COMPILER_VERSION} (${C_COMPILER_NAME})")
message("CXX compiler: ${CMAKE_CXX_COMPILER_ID} ${CMAKE_CXX_COMPILER_VERSION} (${CXX_COMPILER_NAME})")
message("Fortran compiler: ${CMAKE_Fortran_COMPILER_ID} ${CMAKE_Fortran_COMPILER_VERSION} (${Fortran_COMPILER_NAME})")
message("")

option(DEBUG "Enable DEBUG mode" OFF)
option(REPRO "Enable REPRO mode" OFF)
option(VERBOSE "Enable VERBOSE mode" OFF)
option(32BIT "Enable 32BIT (single precision arithmetic in dycore)" OFF)
option(OPENMP "Enable OpenMP threading" ON)
option(AVX2 "Enable AVX2 instruction set" OFF)

option(INLINE_POST "Enable inline post" OFF)

include( cmake/${CMAKE_Fortran_COMPILER_ID}.cmake )

set(NEMSIO_INC $ENV{NEMSIO_INC})
set(POST_INC $ENV{POST_INC})
set(NCEP_LIBS $ENV{POST_LIB} $ENV{NEMSIO_LIB} $ENV{G2_LIB4} $ENV{G2TMPL_LIB} $ENV{BACIO_LIB4} $ENV{SP_LIBd} $ENV{W3EMC_LIBd} $ENV{W3NCO_LIBd} $ENV{CRTM_LIB} $ENV{PNG_LIB} $ENV{JASPER_LIB} $ENV{Z_LIB})

set(ESMF_MOD ${ESMF_F90COMPILEPATHS})
set(ESMF_LIBS "${ESMF_F90ESMFLINKRPATHS} ${ESMF_F90ESMFLINKPATHS} ${ESMF_F90ESMFLINKLIBS}")

set(NETCDF_INC_DIR $ENV{NETCDF}/include)
set(NETCDF_LIBDIR $ENV{NETCDF}/lib)
set(NETCDF_LIBS -L$ENV{NETCDF}/lib -lnetcdff -lnetcdf)

message("")
1 change: 0 additions & 1 deletion compsets/all.input
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,6 @@ run fv3_gfdlmprad @ fv3, standard, baseline, ww3, plat==hera.intel
run fv3_gfdlmprad_atmwav @ fv3, standard, baseline, ww3, plat==hera.intel,plat==wcoss_dell_p3,plat==wcoss.cray
run fv3_gfdlmprad_gwd @ fv3, standard, baseline
run fv3_gfdlmprad_noahmp @ fv3, standard, baseline
run fv3_thompson @ fv3, standard, baseline
run fv3_wsm6 @ fv3, standard, baseline
run fv3_csawmgshoc @ fv3, standard, baseline
run fv3_rasmgshoc @ fv3, standard, baseline
Expand Down
4 changes: 2 additions & 2 deletions compsets/hera.input
Original file line number Diff line number Diff line change
Expand Up @@ -31,8 +31,8 @@ platform hera.intel {
# INPUTS is the input directory, which should contain fix and parm
# files, plus any restarts or other inputs.

BASELINE="/scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200227"
BASELINE_TEMPLATE="/scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200227"
BASELINE="/scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200317/INTEL"
BASELINE_TEMPLATE="/scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200317/INTEL"
INPUTS="@[BASELINE]"

default_resources={
Expand Down
16 changes: 8 additions & 8 deletions compsets/wcoss.input
Original file line number Diff line number Diff line change
Expand Up @@ -28,8 +28,8 @@ platform wcoss.phase2 {
LONG_TEST_QUEUE='&LONGQ;'
BUILD_QUEUE='&BUILDQ;'
MACHINE_ID='wcoss.phase2'
BASELINE="/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200227"
BASELINE_TEMPLATE="/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200227"
BASELINE="/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200317"
BASELINE_TEMPLATE="/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200317"

default_resources={
TASKS=156
Expand Down Expand Up @@ -126,8 +126,8 @@ platform wcoss.phase1 {
C768_THRD=4

MACHINE_ID='wcoss.phase1'
BASELINE="/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200227"
BASELINE_TEMPLATE="/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200227"
BASELINE="/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200317"
BASELINE_TEMPLATE="/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200317"

# WCOSS Phase 2 has been slow of late.
DEFAULT_TEST_WALLTIME=2700
Expand Down Expand Up @@ -213,8 +213,8 @@ platform wcoss_dell_p3 {
LONG_TEST_QUEUE='&LONGQ;'
BUILD_QUEUE='&BUILDQ;'

BASELINE="/gpfs/dell2/emc/modeling/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200227/"
BASELINE_TEMPLATE="/gpfs/dell2/emc/modeling/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200227/"
BASELINE="/gpfs/dell2/emc/modeling/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200317/"
BASELINE_TEMPLATE="/gpfs/dell2/emc/modeling/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200317/"

default_resources={
TASKS=156
Expand Down Expand Up @@ -314,8 +314,8 @@ platform wcoss.cray {
BUILD_WALLTIME="3600"
DEFAULT_TEST_WALLTIME=1800

BASELINE="/gpfs/hps3/emc/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200227"
BASELINE_TEMPLATE="/gpfs/hps3/emc/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200227"
BASELINE="/gpfs/hps3/emc/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200317"
BASELINE_TEMPLATE="/gpfs/hps3/emc/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200317"

execution_time_modules=[[[
module load alps
Expand Down
175 changes: 175 additions & 0 deletions conf/configure.fv3.hera.gnu
Original file line number Diff line number Diff line change
@@ -0,0 +1,175 @@
## NEMS configuration file
##
## Platform: Hera
## Compiler: GNU with OpenMPI

SHELL=/bin/sh

################################################################################
## Include the common configuration parts

ifdef InNemsMakefile
include $(TOP)/conf/configure.nems.NUOPC
endif

############
# commands #
############
FC = mpif90
CC = mpicc
CXX = mpicxx
LD = $(FC)

#########
# flags #
#########
# default is 64-bit OpenMP non-hydrostatic build using AVX2
DEBUG =
REPRO =
VERBOSE =
OPENMP = Y
AVX2 = Y
HYDRO = N
CCPP = N
STATIC = N

include $(ESMFMKFILE)
ESMF_INC = $(ESMF_F90COMPILEPATHS)

NEMSIOINC = -I$(NEMSIO_INC)
NCEPLIBS = $(NEMSIO_LIB) $(BACIO_LIB4) $(SP_LIBd) $(W3EMC_LIBd) $(W3NCO_LIBd)

##############################################
# Need to use at least GNU Make version 3.81 #
##############################################
need := 3.81
ok := $(filter $(need),$(firstword $(sort $(MAKE_VERSION) $(need))))
ifneq ($(need),$(ok))
$(error Need at least make version $(need). Load module gmake/3.81)
endif

NETCDF_ROOT = $(NETCDF)
INCLUDE = -I$(NETCDF_ROOT)/include
NETCDF_INC = -I$(NETCDF_ROOT)/include
ifneq ($(findstring netcdf/4,$(LOADEDMODULES)),)
NETCDF_LIB += -L$(NETCDF)/lib -lnetcdff -lnetcdf
else
NETCDF_LIB += -L$(NETCDF)/lib -lnetcdff -lnetcdf
endif

FPPFLAGS := -cpp -Wp,-w $(INCLUDE)
CFLAGS := $(INCLUDE)

FFLAGS := $(INCLUDE) -fcray-pointer -ffree-line-length-none -fno-range-check -fbacktrace

CPPDEFS += -Duse_libMPI -Duse_netCDF -DSPMD -DUSE_LOG_DIAG_FIELD_INFO -DUSE_GFSL63 -DGFS_PHYS -Duse_WRTCOMP
CPPDEFS += -DNEW_TAUCTMAX -DINTERNAL_FILE_NML -DNO_INLINE_POST

ifeq ($(HYDRO),Y)
CPPDEFS +=
else
CPPDEFS += -DMOIST_CAPPA -DUSE_COND
endif

ifeq ($(NAM_phys),Y)
CPPDEFS += -DNAM_phys
endif

ifeq ($(32BIT),Y)
CPPDEFS += -DOVERLOAD_R4 -DOVERLOAD_R8
FFLAGS +=
else
FFLAGS += -fdefault-real-8 -fdefault-double-8
endif

ifeq ($(AVX2),Y)
FFLAGS +=
CFLAGS +=
else
FFLAGS +=
CFLAGS +=
endif

ifeq ($(MULTI_GASES),Y)
CPPDEFS += -DMULTI_GASES
endif

FFLAGS_OPT = -O2 -fno-range-check
FFLAGS_REPRO = -O2 -g -fbacktrace -fno-range-check
FFLAGS_DEBUG = -g -O0 -fno-unsafe-math-optimizations -frounding-math -fsignaling-nans -ffpe-trap=invalid,zero,overflow -fbounds-check -fbacktrace -fno-range-check -Wall

TRANSCENDENTALS :=
FFLAGS_OPENMP = -fopenmp
FFLAGS_VERBOSE = -v -V

CFLAGS += -D__IFC

CFLAGS_OPT = -O2
CFLAGS_REPRO = -O2
CFLAGS_OPENMP = -fopenmp
CFLAGS_DEBUG = -O0 -g

# Optional Testing compile flags. Mutually exclusive from DEBUG, REPRO, and OPT
# *_TEST will match the production if no new option(s) is(are) to be tested.
FFLAGS_TEST = -O3
CFLAGS_TEST = -O2

LDFLAGS :=
LDFLAGS_OPENMP := -fopenmp
LDFLAGS_VERBOSE := -Wl,-V,--verbose,-cref,-M

# start with blank LIBS
LIBS :=

ifeq ($(REPRO),Y)
CPPDEFS += -DREPRO
CFLAGS += $(CFLAGS_REPRO)
FFLAGS += $(FFLAGS_REPRO)
FAST :=
else ifeq ($(DEBUG),Y)
CPPDEFS += -DDEBUG
CFLAGS += $(CFLAGS_DEBUG)
FFLAGS += $(FFLAGS_DEBUG)
FAST :=
else ifeq ($(TEST),Y)
CFLAGS += $(CFLAGS_TEST)
FFLAGS += $(FFLAGS_TEST)
FAST :=
else
CFLAGS += $(CFLAGS_OPT)
FFLAGS += $(FFLAGS_OPT)
FAST := $(TRANSCENDENTALS)
endif

ifeq ($(OPENMP),Y)
CPPDEFS += -DOPENMP
CFLAGS += $(CFLAGS_OPENMP)
FFLAGS += $(FFLAGS_OPENMP)
LDFLAGS += $(LDFLAGS_OPENMP)
endif

ifeq ($(VERBOSE),Y)
CFLAGS += $(CFLAGS_VERBOSE)
FFLAGS += $(FFLAGS_VERBOSE)
LDFLAGS += $(LDFLAGS_VERBOSE)
endif

ifeq ($(CCPP),Y)
CPPDEFS += -DCCPP
CFLAGS += -I$(PATH_CCPP)/include
FFLAGS += -I$(PATH_CCPP)/include
ifeq ($(STATIC),Y)
CPPDEFS += -DSTATIC
LDFLAGS += -L$(PATH_CCPP)/lib -lccppphys -lccpp $(NCEPLIBS) -lxml2
else
LDFLAGS += -L$(PATH_CCPP)/lib -lccpp
endif
endif

LDFLAGS += $(LIBS)

ifdef InNemsMakefile
FFLAGS += $(ESMF_INC)
CPPFLAGS += -cpp -traditional
EXTLIBS = $(NCEPLIBS) $(ESMF_LIB) $(LDFLAGS) $(NETCDF_LIB)
endif
Loading

0 comments on commit 44604a0

Please sign in to comment.