Update Narwhal Port (#776)

* Update narwhal port, add narwhal intelhpcx
* Update comparelog scripts and narwhal aprun launch, add -q

cicecore/cicedynB/infrastructure/ice_domain.F90

@@ -234,6 +234,7 @@ subroutine init_domain_blocks
 #if (defined CESMCOUPLED)
       nprocs = get_num_procs()
 #else
+      write(nu_diag,*) subname,'ERROR: nprocs, get_num_procs = ',nprocs,get_num_procs()
       call abort_ice(subname//'ERROR: Input nprocs not same as system request')
 #endif
    else if (nghost < 1) then

configuration/scripts/cice.launch.csh

@@ -11,7 +11,7 @@ set jobfile = $1
 source ${ICE_SCRIPTS}/setup_machparams.csh
 
 #==========================================
-if (${ICE_MACHINE} =~ cheyenne*) then
+if (${ICE_MACHCOMP} =~ cheyenne*) then
 if (${ICE_COMMDIR} =~ serial*) then
 cat >> ${jobfile} << EOFR
 ./cice >&! \$ICE_RUNLOG_FILE
@@ -23,7 +23,7 @@ EOFR
 endif
 
 #=======
-else if (${ICE_MACHINE} =~ hobart* || ${ICE_MACHINE} =~ izumi*) then
+else if (${ICE_MACHCOMP} =~ hobart* || ${ICE_MACHCOMP} =~ izumi*) then
 if (${ICE_COMMDIR} =~ serial*) then
 cat >> ${jobfile} << EOFR
 ./cice >&! \$ICE_RUNLOG_FILE
@@ -35,7 +35,7 @@ EOFR
 endif
 
 #=======
-else if (${ICE_MACHINE} =~ gaffney* || ${ICE_MACHINE} =~ koehr* || ${ICE_MACHINE} =~ mustang*) then
+else if (${ICE_MACHCOMP} =~ gaffney* || ${ICE_MACHCOMP} =~ koehr* || ${ICE_MACHCOMP} =~ mustang*) then
 if (${ICE_COMMDIR} =~ serial*) then
 cat >> ${jobfile} << EOFR
 ./cice >&! \$ICE_RUNLOG_FILE
@@ -47,7 +47,7 @@ EOFR
 endif
 
 #=======
-else if (${ICE_MACHINE} =~ nrlssc*) then
+else if (${ICE_MACHCOMP} =~ nrlssc*) then
 if (${ICE_COMMDIR} =~ serial*) then
 cat >> ${jobfile} << EOFR
 ./cice >&! \$ICE_RUNLOG_FILE
@@ -59,13 +59,19 @@ EOFR
 endif
 
 #=======
-else if (${ICE_MACHINE} =~ onyx* || ${ICE_MACHINE} =~ narwhal) then
+else if (${ICE_MACHCOMP} =~ narwhal_*hpcx*) then
 cat >> ${jobfile} << EOFR
-aprun -n ${ntasks} -N ${taskpernodelimit} -d ${nthrds} ./cice >&! \$ICE_RUNLOG_FILE
+mpirun -np ${ntasks} -hostfile \$PBS_NODEFILE \${EXTRA_OMPI_SETTINGS} ./cice >&! \$ICE_RUNLOG_FILE
+EOFR
+
+#=======
+else if (${ICE_MACHCOMP} =~ onyx* || ${ICE_MACHCOMP} =~ narwhal*) then
+cat >> ${jobfile} << EOFR
+aprun -q -n ${ntasks} -N ${taskpernodelimit} -d ${nthrds} ./cice >&! \$ICE_RUNLOG_FILE
 EOFR
 
 #=======
-else if (${ICE_MACHINE} =~ cori*) then
+else if (${ICE_MACHCOMP} =~ cori*) then
 if (${ICE_COMMDIR} =~ serial*) then
 cat >> ${jobfile} << EOFR
 ./cice >&! \$ICE_RUNLOG_FILE
@@ -77,7 +83,7 @@ EOFR
 endif
 
 #=======
-else if (${ICE_MACHINE} =~ compy*) then
+else if (${ICE_MACHCOMP} =~ compy*) then
 if (${ICE_COMMDIR} =~ serial*) then
 cat >> ${jobfile} << EOFR
 ./cice >&! \$ICE_RUNLOG_FILE
@@ -89,7 +95,7 @@ EOFR
 endif
 
 #=======
-else if (${ICE_MACHINE} =~ badger*) then
+else if (${ICE_MACHCOMP} =~ badger*) then
 if (${ICE_COMMDIR} =~ serial*) then
 cat >> ${jobfile} << EOFR
 ./cice >&! \$ICE_RUNLOG_FILE
@@ -101,7 +107,7 @@ EOFR
 endif
 
 #=======
-else if (${ICE_MACHINE} =~ fram*) then
+else if (${ICE_MACHCOMP} =~ fram*) then
 if (${ICE_COMMDIR} =~ serial*) then
 cat >> ${jobfile} << EOFR
 ./cice >&! \$ICE_RUNLOG_FILE
@@ -113,7 +119,7 @@ EOFR
 endif
 
 #=======
-else if (${ICE_MACHINE} =~ cesium*) then
+else if (${ICE_MACHCOMP} =~ cesium*) then
 if (${ICE_COMMDIR} =~ serial*) then
 cat >> ${jobfile} << EOFR
 ./cice >&! \$ICE_RUNLOG_FILE
@@ -125,7 +131,7 @@ EOFR
 endif
 
 #=======
-else if (${ICE_MACHINE} =~ millikan*) then
+else if (${ICE_MACHCOMP} =~ millikan*) then
 if (${ICE_COMMDIR} =~ serial*) then
 cat >> ${jobfile} << EOFR
 ./cice >&! \$ICE_RUNLOG_FILE
@@ -137,7 +143,7 @@ EOFR
 endif
 
 #=======
-else if (${ICE_MACHINE} =~ daley* || ${ICE_MACHINE} =~ banting*) then
+else if (${ICE_MACHCOMP} =~ daley* || ${ICE_MACHCOMP} =~ banting*) then
 if (${ICE_COMMDIR} =~ serial*) then
 cat >> ${jobfile} << EOFR
 ./cice >&! \$ICE_RUNLOG_FILE
@@ -149,7 +155,7 @@ EOFR
 endif
 
 #=======
-else if (${ICE_MACHINE} =~ ppp5* || ${ICE_MACHINE} =~ ppp6* || ${ICE_MACHINE} =~ robert* || ${ICE_MACHINE} =~ underhill*) then
+else if (${ICE_MACHCOMP} =~ ppp5* || ${ICE_MACHCOMP} =~ ppp6* || ${ICE_MACHCOMP} =~ robert* || ${ICE_MACHCOMP} =~ underhill*) then
 if (${ICE_COMMDIR} =~ serial*) then
 cat >> ${jobfile} << EOFR
 ./cice >&! \$ICE_RUNLOG_FILE
@@ -161,7 +167,7 @@ EOFR
 endif
 
 #=======
-else if (${ICE_MACHINE} =~ ppp3*) then
+else if (${ICE_MACHCOMP} =~ ppp3*) then
 if (${ICE_COMMDIR} =~ serial*) then
 cat >> ${jobfile} << EOFR
 ./cice >&! \$ICE_RUNLOG_FILE
@@ -173,7 +179,7 @@ EOFR
 endif
 
 #=======
-else if (${ICE_MACHINE} =~ gpsc3*) then
+else if (${ICE_MACHCOMP} =~ gpsc3*) then
 if (${ICE_COMMDIR} =~ serial*) then
 cat >> ${jobfile} << EOFR
 ./cice >&! \$ICE_RUNLOG_FILE
@@ -186,7 +192,7 @@ endif
 
 
 #=======
-else if (${ICE_MACHINE} =~ freya*) then
+else if (${ICE_MACHCOMP} =~ freya*) then
 if (${ICE_COMMDIR} =~ serial*) then
 cat >> ${jobfile} << EOFR
 aprun -n 1 -N 1 -d 1 ./cice >&! \$ICE_RUNLOG_FILE
@@ -198,45 +204,45 @@ EOFR
 endif
 
 #=======
-else if (${ICE_MACHINE} =~ gaea*) then
+else if (${ICE_MACHCOMP} =~ gaea*) then
 cat >> ${jobfile} << EOFR
 srun -n ${ntasks} -c ${nthrds} ./cice >&! \$ICE_RUNLOG_FILE
 EOFR
 
 #=======
-else if (${ICE_MACHINE} =~ hera*) then
+else if (${ICE_MACHCOMP} =~ hera*) then
 cat >> ${jobfile} << EOFR
 srun -n ${ntasks} -c ${nthrds} ./cice >&! \$ICE_RUNLOG_FILE
 EOFR
 
 #=======
-else if (${ICE_MACHINE} =~ orion*) then
+else if (${ICE_MACHCOMP} =~ orion*) then
 cat >> ${jobfile} << EOFR
 srun -n ${ntasks} -c ${nthrds} ./cice >&! \$ICE_RUNLOG_FILE
 EOFR
 
 #=======
-else if (${ICE_MACHINE} =~ high_Sierra*) then
+else if (${ICE_MACHCOMP} =~ high_Sierra*) then
 cat >> ${jobfile} << EOFR
 mpirun -np ${ntasks} ./cice >&! \$ICE_RUNLOG_FILE
 #./cice >&! \$ICE_RUNLOG_FILE
 EOFR
 
 #=======
-else if (${ICE_MACHINE} =~ phase3*) then
+else if (${ICE_MACHCOMP} =~ phase3*) then
 cat >> ${jobfile} << EOFR
 mpirun -np ${ntasks} ./cice >&! \$ICE_RUNLOG_FILE
 #./cice >&! \$ICE_RUNLOG_FILE
 EOFR
 
 #=======
-else if (${ICE_MACHINE} =~ testmachine*) then
+else if (${ICE_MACHCOMP} =~ testmachine*) then
 cat >> ${jobfile} << EOFR
 ./cice >&! \$ICE_RUNLOG_FILE
 EOFR
 
 #=======
-else if (${ICE_MACHINE} =~ travisCI*) then
+else if (${ICE_MACHCOMP} =~ travisCI*) then
 if (${ICE_COMMDIR} =~ serial*) then
 cat >> ${jobfile} << EOFR
 ./cice >&! \$ICE_RUNLOG_FILE
@@ -251,7 +257,7 @@ endif
 #EOFR
 
 #=======
-else if (${ICE_MACHINE} =~ conda*) then
+else if (${ICE_MACHCOMP} =~ conda*) then
 if (${ICE_COMMDIR} =~ serial*) then
 cat >> ${jobfile} << EOFR
 ./cice >&! \$ICE_RUNLOG_FILE
@@ -265,7 +271,7 @@ endif
 
 #=======
 else
-  echo "${0} ERROR ${ICE_MACHINE} unknown"
+  echo "${0} ERROR ${ICE_MACHCOMP} unknown"
   exit -1
 endif
 #=======

configuration/scripts/machines/Macros.narwhal_aocc

@@ -49,7 +49,7 @@ INCLDIR := $(INCLDIR)
 
 ifeq ($(ICE_THREADED), true) 
    LDFLAGS += -mp
-   CFLAGS += -mp
+#   CFLAGS += -mp
    FFLAGS += -mp
 else
    LDFLAGS += -nomp

configuration/scripts/machines/Macros.narwhal_intelhpcx

@@ -0,0 +1,57 @@
+#==============================================================================
+# Macros file for NAVYDSRC narwhal, intel compiler with hpcx hcoll openmpi
+#==============================================================================
+
+CPP        := fpp
+CPPDEFS    := -DFORTRANUNDERSCORE ${ICE_CPPDEFS}
+CFLAGS     := -c -O2 -fp-model precise -fcommon
+
+FIXEDFLAGS := -132
+FREEFLAGS  := -FR
+FFLAGS     := -fp-model precise -convert big_endian -assume byterecl -ftz -traceback
+# -mcmodel medium -shared-intel
+FFLAGS_NOOPT:= -O0
+
+ifeq ($(ICE_BLDDEBUG), true)
+  FFLAGS     += -O0 -g -check uninit -check bounds -check pointers -fpe0 -check noarg_temp_created
+#  FFLAGS     += -O0 -g -check all -fpe0 -ftrapuv -fp-model except -check noarg_temp_created -init=snan,arrays
+else
+  FFLAGS     += -O2
+endif
+
+SCC   := cc 
+SFC   := ftn
+MPICC := mpicc
+MPIFC := mpif90
+
+ifeq ($(ICE_COMMDIR), mpi)
+  FC := $(MPIFC)
+  CC := $(MPICC)
+else
+  FC := $(SFC)
+  CC := $(SCC)
+endif
+LD:= $(FC)
+
+# defined by module
+NETCDF_PATH := $(NETCDF_DIR)
+#PNETCDF_PATH := $(PNETCDF)
+#PNETCDF_PATH := /glade/apps/opt/pnetcdf/1.3.0/intel/default
+#LAPACK_LIBDIR := /glade/apps/opt/lapack/3.4.2/intel/12.1.5/lib
+
+#PIO_CONFIG_OPTS:= --enable-filesystem-hints=gpfs 
+
+INCLDIR := $(INCLDIR)
+INCLDIR += -I$(NETCDF_PATH)/include
+
+LIB_NETCDF := $(NETCDF_PATH)/lib
+#LIB_PNETCDF := $(PNETCDF_PATH)/lib
+#LIB_MPI := $(IMPILIBDIR)
+SLIBS   := $(HCOLL_LIBS) -L$(NETCDF_PATH)/lib -lnetcdf -lnetcdff
+
+ifeq ($(ICE_THREADED), true) 
+   LDFLAGS += -qopenmp
+   CFLAGS += -qopenmp
+   FFLAGS += -qopenmp
+endif
+

configuration/scripts/machines/env.narwhal_aocc

@@ -14,21 +14,18 @@ module unload PrgEnv-cray
 module unload PrgEnv-gnu
 module unload PrgEnv-intel
 module unload PrgEnv-nvidia
-module load PrgEnv-aocc/8.1.0
-module load cray-pals/1.0.17
-module load bct-env/0.1
-module unload aocc
-module load aocc/2.2.0.1
-module unload cray-mpich
-module load cray-mpich/8.1.5
-
+module load PrgEnv-aocc
+module load cpe/22.03
+module load aocc/3.0.0 
+module unload cray-pals
+module load cray-pals/1.2.2
 module unload cray-hdf5
 module unload cray-hdf5-parallel
 module unload cray-netcdf-hdf5parallel
 module unload cray-parallel-netcdf
 module unload netcdf
-module load cray-netcdf/4.7.4.4
-module load cray-hdf5/1.12.0.4
+module load cray-netcdf/4.8.1.1
+module load cray-hdf5/1.12.1.1
 
 setenv NETCDF_PATH ${NETCDF_DIR}
 limit coredumpsize unlimited
@@ -41,7 +38,7 @@ endif
 setenv ICE_MACHINE_MACHNAME narwhal
 setenv ICE_MACHINE_MACHINFO "Cray EX AMD EPYC 7H12 2.6GHz, Slingshot-10 Interconnect"
 setenv ICE_MACHINE_ENVNAME intel
-setenv ICE_MACHINE_ENVINFO "aocc_3.0.0-Build#78 2020_12_10 clang/flang 12.0.0, cray-mpich/8.1.5, netcdf/4.7.4.4"
+setenv ICE_MACHINE_ENVINFO "aocc_3.0.0-Build#78 2020_12_10 clang/flang 12.0.0, cray-mpich/8.1.14, netcdf/4.8.1.1"
 setenv ICE_MACHINE_MAKE gmake
 setenv ICE_MACHINE_WKDIR $WORKDIR/CICE_RUNS
 setenv ICE_MACHINE_INPUTDATA /p/work1/projects/RASM/cice_consortium

configuration/scripts/machines/env.narwhal_cray

@@ -14,21 +14,17 @@ module unload PrgEnv-cray
 module unload PrgEnv-gnu
 module unload PrgEnv-intel
 module unload PrgEnv-nvidia
-module load PrgEnv-cray/8.1.0
-module load cray-pals/1.0.17
-module load bct-env/0.1
-module unload cce 
-module load cce/12.0.3
-module unload cray-mpich
-module load cray-mpich/8.1.9
-
+module load PrgEnv-cray
+module load cpe/22.03
+module unload cray-pals
+module load cray-pals/1.2.2
 module unload cray-hdf5
 module unload cray-hdf5-parallel
 module unload cray-netcdf-hdf5parallel
 module unload cray-parallel-netcdf
 module unload netcdf
-module load cray-netcdf/4.7.4.4
-module load cray-hdf5/1.12.0.4
+module load cray-netcdf/4.8.1.1
+module load cray-hdf5/1.12.1.1
 
 setenv NETCDF_PATH ${NETCDF_DIR}
 limit coredumpsize unlimited
@@ -39,9 +35,9 @@ setenv OMP_WAIT_POLICY PASSIVE
 endif
 
 setenv ICE_MACHINE_MACHNAME narwhal
-setenv ICE_MACHINE_MACHINFO "Cray EX AMD EPYC 7H12"
+setenv ICE_MACHINE_MACHINFO "Cray EX AMD EPYC 7H12 2.6GHz, Slingshot-10 Interconnect"
 setenv ICE_MACHINE_ENVNAME cray
-setenv ICE_MACHINE_ENVINFO "cce 12.0.3, cray-mpich/8.1.9, netcdf/4.7.4.4"
+setenv ICE_MACHINE_ENVINFO "cce 13.0.2, cray-mpich/8.1.14, netcdf/4.8.1.1"
 setenv ICE_MACHINE_MAKE gmake
 setenv ICE_MACHINE_WKDIR $WORKDIR/CICE_RUNS
 setenv ICE_MACHINE_INPUTDATA /p/work1/projects/RASM/cice_consortium