Skip to content

Commit

Permalink
Merge pull request #195 from DavidHuber-NOAA/port_2_s4
Browse files Browse the repository at this point in the history
GitHub Issue #177 Update build/run scripts and disable regression tests for S4
  • Loading branch information
MichaelLueken authored Aug 11, 2021
2 parents c5c0a9e + cca6d55 commit 99346e4
Show file tree
Hide file tree
Showing 7 changed files with 28 additions and 323 deletions.
2 changes: 1 addition & 1 deletion cmake/Modules/platforms/S4.cmake
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ macro (setS4)
message("Setting paths for S4")
option(FIND_HDF5 "Try to Find HDF5 libraries" OFF)
option(FIND_HDF5_HL "Try to Find HDF5 libraries" OFF)
set(HOST_FLAG "-xHOST" CACHE INTERNAL "Host Flag")
set(HOST_FLAG "-march=ivybridge" CACHE INTERNAL "Host Flag")
set(MKL_FLAG "-mkl" CACHE INTERNAL "MKL Flag")
set(GSI_Intel_Platform_FLAGS "-DPOUND_FOR_STRINGIFY -O3 -fp-model source -assume byterecl -convert big_endian -g -traceback -D_REAL8_ ${OpenMP_Fortran_FLAGS} ${MPI_Fortran_COMPILE_FLAGS}" CACHE INTERNAL "GSI Fortran Flags")
set(ENKF_Platform_FLAGS "-O3 ${HOST_FLAG} -warn all -implicitnone -traceback -fp-model strict -convert big_endian -DGFS -D_REAL8_ ${OpenMP_Fortran_FLAGS}" CACHE INTERNAL "ENKF Fortran Flags")
Expand Down
2 changes: 1 addition & 1 deletion cmake/Modules/setHOST.cmake
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,7 @@ macro( setHOST )
set( HOST-Jet "TRUE" )
setJet()
elseif( HOST-S4 )
option(BUILD_CORELIBS "Build the Core libraries " ON)
option(BUILD_CORELIBS "Build the Core libraries " OFF)
set( host "S4" )
set( HOST-S4 "TRUE" )
setS4()
Expand Down
48 changes: 23 additions & 25 deletions modulefiles/modulefile.ProdGSI.s4
Original file line number Diff line number Diff line change
Expand Up @@ -2,32 +2,30 @@
## NOAA-EMC/GSI
##_____________________________________________________

setenv CRTM_FIX /data/prod/hpc-stack/fix/crtm/2.3.0

module use -a /home/mpotts/s4-cardinal/spack/share/spack/modules/linux-centos7-x86_64
module load license_intel/S4
module load intel/18.0.3
module load intel-mpi/18.0.3-intel-18.0.3
module load license_intel/S4
module use /data/prod/hpc-stack/modulefiles/stack
module load hpc/1.1.0

module load cmake/3.12.2-intel-18.0.3
# Load intel compiler and mpi
module load hpc-intel/18.0.4
module load hpc-impi/18.0.4
# Load python
module load miniconda/3.8-s4

module load hdf5/1.10.4-intel-18.0.3-impi
module load netcdf/4.6.2-intel-18.0.3-impi
module load netcdf-fortran/4.4.4-intel-18.0.3-impi
module load sp/v2.0.2-intel-18.0.3

module load w3emc/v2.2.0-intel-18.0.3-impi

module load bacio/v2.1.0-intel-18.0.3
module load sfcio/v1.1.0-intel-18.0.3
module load w3nco/v2.0.6-intel-18.0.3-impi

module load bufr/v11.2.0-intel-18.0.3
module load nemsio/v2.2.3-intel-18.0.3-impi
module load sigio/v2.1.0-intel-18.0.3-impi
module load crtm/v2.3.0
module load prod-util/1.0.18-intel-18.0.3-impi

echo "done loading modules"

# Loading production utilities (ndate)
# Load production utilities
module load prod_util/1.2.2

# Load nceplibs
module load bufr/11.4.0
module load ip/3.3.3
module load nemsio/2.5.2
module load sfcio/1.4.1
module load sigio/2.3.2
module load sp/2.3.3
module load w3nco/2.4.1
module load w3emc/2.7.3
module load bacio/2.4.1
module load crtm/2.3.0
module load netcdf/4.7.4
61 changes: 0 additions & 61 deletions regression/regression_param.sh
Original file line number Diff line number Diff line change
Expand Up @@ -24,9 +24,6 @@ case $machine in
;;
Discover)
sub_cmd="sub_discover"
;;
s4)
sub_cmd="sub_s4"
;;
Cheyenne)
sub_cmd="sub_ncar -a p48503002 -q economy -d $PWD"
Expand All @@ -42,7 +39,6 @@ esac
# wcoss_d : 128 Gb / 28 cores = 4.57 Gb / core
# theia : 64 Gb / 24 cores = 2.67 Gb / core
# discover:
# s4 :
# cheyenne:
# Select minimim memory per core for regression tests
export memnode=${memnode:-64}
Expand All @@ -68,9 +64,6 @@ case $regtest in
elif [[ "$machine" = "WCOSS_D" ]]; then
topts[1]="0:30:00" ; popts[1]="28/2/" ; ropts[1]="/1"
topts[2]="0:30:00" ; popts[2]="28/4/" ; ropts[2]="/2"
elif [[ "$machine" = "s4" ]]; then
topts[1]="1:45:00" ; popts[1]="20/4" ; ropts[1]="/1"
topts[2]="1:45:00" ; popts[2]="40/2" ; ropts[2]="/2"
elif [[ "$machine" = "Discover" ]]; then
topts[1]="0:30:00" ; popts[1]="36/2" ; ropts[1]="/1"
topts[2]="0:30:00" ; popts[2]="72/3" ; ropts[2]="/2"
Expand Down Expand Up @@ -104,9 +97,6 @@ case $regtest in
elif [[ "$machine" = "Discover" ]]; then
topts[1]="0:30:00" ; popts[1]="16/1" ; ropts[1]="/1"
topts[2]="0:30:00" ; popts[2]="16/2" ; ropts[2]="/2"
elif [[ "$machine" = "s4" ]]; then
topts[1]="0:25:00" ; popts[1]="16/2/" ; ropts[1]="/1"
topts[2]="0:25:00" ; popts[2]="16/4/" ; ropts[2]="/2"
fi

if [ "$debug" = ".true." ] ; then
Expand All @@ -131,9 +121,6 @@ case $regtest in
elif [[ "$machine" = "WCOSS_C" ]]; then
topts[1]="1:35:00" ; popts[1]="48/12/" ; ropts[1]="1024/1" # sub_wcoss_c popts are "#tasks/#nodes/"
topts[2]="1:25:00" ; popts[2]="60/15/" ; ropts[2]="1024/2"
elif [[ "$machine" = "s4" ]]; then
topts[1]="0:55:00" ; popts[1]="16/2/" ; ropts[1]="/1"
topts[2]="0:45:00" ; popts[2]="16/4/" ; ropts[2]="/2"
elif [[ "$machine" = "WCOSS_D" ]]; then
topts[1]="0:15:00" ; popts[1]="28/2/" ; ropts[1]="/1"
topts[2]="0:15:00" ; popts[2]="28/4/" ; ropts[2]="/2"
Expand Down Expand Up @@ -175,9 +162,6 @@ case $regtest in
elif [[ "$machine" = "WCOSS_C" ]]; then
topts[1]="0:15:00" ; popts[1]="48/8/" ; ropts[1]="1024/1" # sub_wcoss_c popts are "#tasks/#nodes/"
topts[2]="0:15:00" ; popts[2]="60/10/" ; ropts[2]="1024/2"
elif [[ "$machine" = "s4" ]]; then
topts[1]="0:25:00" ; popts[1]="20/4/" ; ropts[1]="/1"
topts[2]="0:25:00" ; popts[2]="40/4/" ; ropts[2]="/2"
elif [[ "$machine" = "WCOSS_D" ]]; then
topts[1]="0:15:00" ; popts[1]="28/2/" ; ropts[1]="/1"
topts[2]="0:15:00" ; popts[2]="28/4/" ; ropts[2]="/2"
Expand Down Expand Up @@ -304,9 +288,6 @@ case $regtest in
elif [[ "$machine" = "Discover" ]]; then
topts[1]="0:30:00" ; popts[1]="48/2" ; ropts[1]="/1"
topts[2]="0:30:00" ; popts[2]="60/3" ; ropts[2]="/2"
elif [[ "$machine" = "s4" ]]; then
topts[1]="0:30:00" ; popts[1]="16/2/" ; ropts[1]="/1"
topts[2]="0:30:00" ; popts[2]="16/4/" ; ropts[2]="/2"
fi

if [ "$debug" = ".true." ] ; then
Expand All @@ -331,9 +312,6 @@ case $regtest in
elif [[ "$machine" = "WCOSS_C" ]]; then
topts[1]="0:30:00" ; popts[1]="48/8/" ; ropts[1]="1024/1" # sub_wcoss_c popts are "#tasks/#nodes/"
topts[2]="0:30:00" ; popts[2]="60/10/" ; ropts[2]="1024/2"
elif [[ "$machine" = "s4" ]]; then
topts[1]="0:25:00" ; popts[1]="16/2/" ; ropts[1]="/1"
topts[2]="0:25:00" ; popts[2]="16/4/" ; ropts[2]="/2"
elif [[ "$machine" = "WCOSS_D" ]]; then
topts[1]="0:15:00" ; popts[1]="28/2/" ; ropts[1]="/1"
topts[2]="0:15:00" ; popts[2]="28/4/" ; ropts[2]="/2"
Expand Down Expand Up @@ -370,9 +348,6 @@ case $regtest in
elif [[ "$machine" = "Discover" ]]; then
topts[1]="0:30:00" ; popts[1]="16/1" ; ropts[1]="/1"
topts[2]="0:30:00" ; popts[2]="20/2" ; ropts[2]="/2"
elif [[ "$machine" = "s4" ]]; then
topts[1]="0:25:00" ; popts[1]="16/1/" ; ropts[1]="/1"
topts[2]="0:25:00" ; popts[2]="16/2/" ; ropts[2]="/1"
fi

if [ "$debug" = ".true." ] ; then
Expand All @@ -397,9 +372,6 @@ case $regtest in
elif [[ "$machine" = "WCOSS_D" ]]; then
topts[1]="0:15:00" ; popts[1]="28/1/" ; ropts[1]="/1"
topts[2]="0:15:00" ; popts[2]="28/2/" ; ropts[2]="/1"
elif [[ "$machine" = "s4" ]]; then
topts[1]="0:25:00" ; popts[1]="16/1/" ; ropts[1]="/1"
topts[2]="0:25:00" ; popts[2]="16/2/" ; ropts[2]="/1"
fi

if [ "$debug" = ".true." ] ; then
Expand All @@ -424,9 +396,6 @@ case $regtest in
elif [[ "$machine" = "WCOSS_C" ]]; then
topts[1]="0:30:00" ; popts[1]="48/8/" ; ropts[1]="1024/1" # sub_wcoss_c popts are "#tasks/#nodes/"
topts[2]="0:30:00" ; popts[2]="60/10/" ; ropts[2]="1024/2"
elif [[ "$machine" = "s4" ]]; then
topts[1]="0:50:00" ; popts[1]="7/12/" ; ropts[1]="/1"
topts[2]="0:50:00" ; popts[2]="9/12/" ; ropts[2]="/2"
elif [[ "$machine" = "WCOSS_D" ]]; then
topts[1]="0:30:00" ; popts[1]="7/24/" ; ropts[1]="/1"
topts[2]="0:30:00" ; popts[2]="9/24/" ; ropts[2]="/2"
Expand Down Expand Up @@ -457,9 +426,6 @@ case $regtest in
elif [[ "$machine" = "WCOSS_C" ]]; then
topts[1]="0:15:00" ; popts[1]="8/2/" ; ropts[1]="1024/1" # sub_wcoss_c popts are "#tasks/#nodes/"
topts[2]="0:15:00" ; popts[2]="16/4/" ; ropts[2]="1024/2"
elif [[ "$machine" = "s4" ]]; then
topts[1]="0:25:00" ; popts[1]="8/1/" ; ropts[1]="/1"
topts[2]="0:25:00" ; popts[2]="16/1/" ; ropts[2]="/2"
elif [[ "$machine" = "WCOSS_D" ]]; then
topts[1]="0:15:00" ; popts[1]="14/1/" ; ropts[1]="/1"
topts[2]="0:15:00" ; popts[2]="28/2/" ; ropts[2]="/2"
Expand Down Expand Up @@ -490,9 +456,6 @@ case $regtest in
elif [[ "$machine" = "WCOSS_C" ]]; then
topts[1]="1:30:00" ; popts[1]="72/9/" ; ropts[1]="1024/1" # sub_wcoss_c popts are "#tasks/#nodes/"
topts[2]="1:30:00" ; popts[2]="96/12/" ; ropts[2]="1024/2"
elif [[ "$machine" = "s4" ]]; then
topts[1]="0:50:00" ; popts[1]="7/10/" ; ropts[1]="/1"
topts[2]="0:50:00" ; popts[2]="9/10/" ; ropts[2]="/2"
elif [[ "$machine" = "WCOSS_D" ]]; then
topts[1]="0:15:00" ; popts[1]="7/14/" ; ropts[1]="/1"
topts[2]="0:15:00" ; popts[2]="9/14/" ; ropts[2]="/2"
Expand Down Expand Up @@ -520,9 +483,6 @@ case $regtest in
elif [[ "$machine" = "WCOSS_C" ]]; then
topts[1]="1:15:00" ; popts[1]="48/6/" ; ropts[1]="1024/1" # sub_wcoss_c popts are "#tasks/#nodes/"
topts[2]="1:15:00" ; popts[2]="64/8/" ; ropts[2]="1024/1"
elif [[ "$machine" = "s4" ]]; then
topts[1]="0:25:00" ; popts[1]="8/6/" ; ropts[1]="/1"
topts[2]="0:25:00" ; popts[2]="8/8/" ; ropts[2]="/1"
elif [[ "$machine" = "WCOSS_D" ]]; then
topts[1]="0:30:00" ; popts[1]="14/8/" ; ropts[1]="/1"
topts[2]="0:30:00" ; popts[2]="14/14/" ; ropts[2]="/2"
Expand Down Expand Up @@ -556,9 +516,6 @@ case $regtest in
elif [[ "$machine" = "WCOSS_D" ]]; then
topts[1]="0:15:00" ; popts[1]="10/10/" ; ropts[1]="/1"
topts[2]="0:15:00" ; popts[2]="14/14/" ; ropts[2]="/2"
elif [[ "$machine" = "s4" ]]; then
topts[1]="0:40:00" ; popts[1]="6/6/" ; ropts[1]="/1"
topts[2]="0:40:00" ; popts[2]="8/8/" ; ropts[2]="/1"
fi

if [ "$debug" = ".true." ] ; then
Expand All @@ -583,9 +540,6 @@ case $regtest in
elif [[ "$machine" = "WCOSS_C" ]]; then
topts[1]="0:25:00" ; popts[1]="20/4/" ; ropts[1]="1024/1" # sub_wcoss_c popts are "#tasks/#nodes/"
topts[2]="0:25:00" ; popts[2]="20/5/" ; ropts[2]="1024/2"
elif [[ "$machine" = "s4" ]]; then
topts[1]="0:25:00" ; popts[1]="32/2/" ; ropts[1]="/1"
topts[2]="0:25:00" ; popts[2]="32/4/" ; ropts[2]="/2"
elif [[ "$machine" = "WCOSS_D" ]]; then
topts[1]="0:15:00" ; popts[1]="16/2/" ; ropts[1]="/1"
topts[2]="0:15:00" ; popts[2]="16/4/" ; ropts[2]="/2"
Expand Down Expand Up @@ -619,9 +573,6 @@ case $regtest in
elif [[ "$machine" = "Discover" ]]; then
topts[1]="0:30:00" ; popts[1]="20/4" ; ropts[1]="/1"
topts[2]="0:30:00" ; popts[2]="40/2" ; ropts[2]="/2"
elif [[ "$machine" = "s4" ]]; then
topts[1]="0:25:00" ; popts[1]="36/2/" ; ropts[1]="/1"
topts[2]="0:25:00" ; popts[2]="72/3/" ; ropts[2]="/2"
fi

if [ "$debug" = ".true." ] ; then
Expand Down Expand Up @@ -652,9 +603,6 @@ case $regtest in
elif [[ "$machine" = "Discover" ]]; then
topts[1]="0:30:00" ; popts[1]="20/4" ; ropts[1]="/1"
topts[2]="0:30:00" ; popts[2]="40/2" ; ropts[2]="/2"
elif [[ "$machine" = "s4" ]]; then
topts[1]="0:25:00" ; popts[1]="36/2/" ; ropts[1]="/1"
topts[2]="0:25:00" ; popts[2]="72/3/" ; ropts[2]="/2"
fi

if [ "$debug" = ".true." ] ; then
Expand Down Expand Up @@ -723,15 +671,6 @@ elif [[ "$machine" = "WCOSS_D" ]]; then
export KMP_STACKSIZE=2G
export FORT_BUFFERED=true
export APRUN="mpirun"
elif [[ "$machine" = "s4" ]]; then
export APRUN="srun"
export MPI_BUFS_PER_PROC=2048
export MPI_BUFS_PER_HOST=2048
export MPI_GROUP_MAX=256
export MPI_MEMMAP_OFF=1
export MP_STDOUTMODE=ORDERED
export KMP_STACKSIZE=512MB
export KMP_AFFINITY=scatter
elif [[ "$machine" = "Discover" ]]; then
export APRUN="mpiexec_mpt -np \$SLURM_NTASKS"
fi
20 changes: 0 additions & 20 deletions regression/regression_var.sh
Original file line number Diff line number Diff line change
Expand Up @@ -42,8 +42,6 @@ if [ -z ${machine+x} ]; then
export machine="WCOSS_C"
elif [ -d /gpfs/dell1/ptmp ]; then # venus or mars
export machine="WCOSS_D"
elif [ -d /data/users ]; then # S4
export machine="s4"
elif [ -d /discover/nobackup ]; then # NCCS Discover
export machine="Discover"
fi
Expand Down Expand Up @@ -157,24 +155,6 @@ case $machine in

export accnt=""
;;
s4)
export noscrub="/data/users/$LOGNAME"
if [[ "$cmaketest" = "false" ]]; then
export basedir="/home/$LOGNAME/gsi"
fi
export group="dev"
export queue="dev"
export NWPROD="/usr/local/jcsda/nwprod_gdas_2014"
export ptmp="/scratch/short/$LOGNAME/$ptmpName"

export fixcrtm="/home/mpotts/gsi/trunk/lib/CRTM_REL-2.2.3/fix_update"
export casesdir="/data/users/mpotts/CASES"
export ndate="$NWPROD/util/exec/ndate"

export check_resource="no"

export accnt="star"
;;
Discover)
if [[ "$cmaketest" = "false" ]]; then
echo "Regression tests on Discover need to be run via ctest"
Expand Down
6 changes: 3 additions & 3 deletions ush/build_all_cmake.sh
Original file line number Diff line number Diff line change
Expand Up @@ -31,8 +31,8 @@ elif [[ -d /ioddev_dell ]]; then
elif [[ -d /scratch1 ]] ; then
. /apps/lmod/lmod/init/sh
target=hera
elif [[ -d /carddata ]] ; then
. /opt/apps/lmod/3.1.9/init/sh
elif [[ -d /data/prod ]] ; then
. /usr/share/lmod/lmod/init/sh
target=s4
elif [[ -d /jetmon ]] ; then
. $MODULESHOME/init/sh
Expand Down Expand Up @@ -78,7 +78,7 @@ if [ $target = wcoss_d ]; then
elif [ $target = wcoss -o $target = gaea ]; then
module purge
module load $dir_modules/modulefile.ProdGSI.$target
elif [ $target = hera -o $target = orion ]; then
elif [ $target = hera -o $target = orion -o $target = s4 ]; then
module purge
module use $dir_modules
module load modulefile.ProdGSI.$target
Expand Down
Loading

0 comments on commit 99346e4

Please sign in to comment.