Skip to content

Commit

Permalink
Merge branch 'rljacob/testsuite/add-blues' into next (PR #150)
Browse files Browse the repository at this point in the history 
Enable blues.lcrc.anl.gov to run acme_developer on 3 compilers (pgi, gnu, intel) and 2 mpi
libraries (openmpi and mpich) as well as the built-in mpi-serial.

First attempt to run acme_developer uncovered problems with the mvapich library so
machine file changes were made to remove mvapich and add openmpi and mpich as options.
Additional changes to machine files were made to enable mpi-serial mostly in how the PNETCDF
library root was defined. Finally, the default baseline root was changed.

All but 3 acme_developer tests pass.

SEG-29
  • Loading branch information
@rljacob
rljacob committed Mar 23, 2015
2 parents 17a1007 + 4796c30 commit b90c649
Show file tree
Hide file tree
Showing 5 changed files with 111 additions and 15 deletions.
27 changes: 18 additions & 9 deletions scripts/ccsm_utils/Machines/config_compilers.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -818,25 +818,34 @@ for mct, etc.
</compiler>

<compiler COMPILER="pgi" MACH="blues">
<NETCDF_PATH> /soft/netcdf/4.3.1-serial/pgi-13.9/</NETCDF_PATH>
<MPI_PATH MPILIB="mvapich2">/soft/mvapich2/2.0-pgi-13.9-psm</MPI_PATH>
<MPI_LIB_NAME MPILIB="mvapich2">mpich</MPI_LIB_NAME>
<PNETCDF_PATH>$(PNETCDFROOT)</PNETCDF_PATH>
<NETCDF_PATH>$(NETCDFROOT)</NETCDF_PATH>
<MPI_PATH MPILIB="openmpi">/soft/openmpi/1.8.2/pgi-13.9</MPI_PATH>
<MPI_PATH MPILIB="mpich">/soft/mpich2/1.4.1-pgi-13.9/</MPI_PATH>
<MPI_LIB_NAME MPILIB="openmpi"> mpi</MPI_LIB_NAME>
<MPI_LIB_NAME MPILIB="mpich">mpich</MPI_LIB_NAME>
<ADD_SLIBS>$(shell $(NETCDF_PATH)/bin/nc-config --flibs) -llapack -lblas</ADD_SLIBS>
<PIO_FILESYSTEM_HINTS>gpfs </PIO_FILESYSTEM_HINTS>
</compiler>

<compiler COMPILER="intel" MACH="blues">
<NETCDF_PATH>/soft/netcdf/4.3.1-serial/intel-13.1</NETCDF_PATH>
<MPI_PATH MPILIB="mvapich2">/soft/mvapich2/2.0-intel-13.1-psm</MPI_PATH>
<MPI_LIB_NAME MPILIB="mvapich2">mpich</MPI_LIB_NAME>
<PNETCDF_PATH>$(PNETCDFROOT)</PNETCDF_PATH>
<NETCDF_PATH>$(NETCDFROOT)</NETCDF_PATH>
<MPI_PATH MPILIB="openmpi">/soft/openmpi/1.8.2/intel-13.1</MPI_PATH>
<MPI_PATH MPILIB="mpich">/soft/mpich2/1.4.1-intel-13.1</MPI_PATH>
<MPI_LIB_NAME MPILIB="openmpi"> mpi</MPI_LIB_NAME>
<MPI_LIB_NAME MPILIB="mpich">mpich</MPI_LIB_NAME>
<ADD_SLIBS>$(shell $(NETCDF_PATH)/bin/nc-config --flibs) -llapack -lblas</ADD_SLIBS>
<PIO_FILESYSTEM_HINTS>gpfs </PIO_FILESYSTEM_HINTS>
</compiler>

<compiler COMPILER="gnu" MACH="blues">
<NETCDF_PATH>/soft/netcdf/4.3.1-serial/gcc-4.7.2</NETCDF_PATH>
<MPI_PATH MPILIB="mvapich2">/soft/mvapich2/2.0-gcc-4.7.2-psm</MPI_PATH>
<MPI_LIB_NAME MPILIB="mvapich2">mpich</MPI_LIB_NAME>
<PNETCDF_PATH>$(PNETCDFROOT)</PNETCDF_PATH>
<NETCDF_PATH>$(NETCDFROOT)</NETCDF_PATH>
<MPI_PATH MPILIB="openmpi">/soft/openmpi/1.6.5-gcc-4.7.2</MPI_PATH>
<MPI_PATH MPILIB="mpich">/soft/mpich2/1.4.1-gcc-4.7.2/</MPI_PATH>
<MPI_LIB_NAME MPILIB="openmpi"> mpi</MPI_LIB_NAME>
<MPI_LIB_NAME MPILIB="mpich">mpich</MPI_LIB_NAME>
<ADD_SLIBS>$(shell $(NETCDF_PATH)/bin/nc-config --flibs) -llapack -lblas</ADD_SLIBS>
<PIO_FILESYSTEM_HINTS>gpfs </PIO_FILESYSTEM_HINTS>
</compiler>
Expand Down
4 changes: 2 additions & 2 deletions scripts/ccsm_utils/Machines/config_machines.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -558,14 +558,14 @@
<machine MACH="blues">
<DESC>ANL/LCRC Linux Cluster</DESC>
<COMPILERS>intel,gnu,pgi</COMPILERS>
<MPILIBS>mvapich2</MPILIBS>
<MPILIBS>openmpi,mpich,mpi-serial</MPILIBS>
<RUNDIR>/lcrc/project/$PROJECT/$CCSMUSER/$CASE/run</RUNDIR>
<EXEROOT>/lcrc/project/$PROJECT/$CCSMUSER/$CASE/bld</EXEROOT>
<DIN_LOC_ROOT>/home/ccsm-data/inputdata</DIN_LOC_ROOT>
<DIN_LOC_ROOT_CLMFORC>/home/ccsm-data/inputdata/atm/datm7</DIN_LOC_ROOT_CLMFORC>
<DOUT_S_ROOT>/lcrc/project/$PROJECT/$CCSMUSER/archive/$CASE</DOUT_S_ROOT>
<DOUT_L_MSROOT>UNSET</DOUT_L_MSROOT>
<CCSM_BASELINE>/home/ccsm-data/ccsm_baselines</CCSM_BASELINE>
<CCSM_BASELINE>/lcrc/group/earthscience/acme_baselines</CCSM_BASELINE>
<CCSM_CPRNC>/gpfs/home/projects/ccsm/tools/cprnc/cprnc</CCSM_CPRNC>
<OS>LINUX</OS>
<BATCHQUERY>qstat</BATCHQUERY>
Expand Down
44 changes: 41 additions & 3 deletions scripts/ccsm_utils/Machines/env_mach_specific.blues
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,20 +6,58 @@ setenv P4_GLOBMEMSIZE 500000000

soft add +cmake-2.8.12

#
# Tried mvapich2 1.8.1, 1.9 and 2.0 and they
# all failed an ERS test. So mvapich2 is not supported.
#
#
if ( $COMPILER == "intel" ) then
soft add +intel-13.1
soft add +netcdf-4.3.1-serial-intel
soft add +mvapich2-2.0-intel-13.1-psm
setenv NETCDFROOT /soft/netcdf/4.3.1-serial/intel-13.1
if ( $MPILIB == "openmpi") then
soft add +openmpi-1.8.2-intel-13.1-psm
else if ( $MPILIB == "mpich") then
soft add +mpich2-1.4.1-intel-13.1
endif
if ( $MPILIB == "mpi-serial") then
setenv PNETCDFROOT ""
else
soft add +pnetcdf-1.5.0-mpich2-intel-13.1
setenv PNETCDFROOT /soft/pnetcdf/1.5.0/intel-13.1/mpich2-1.4.1
endif
endif

if ( $COMPILER == "pgi" ) then
soft add +pgi-13.9
soft add +mvapich2-2.0-pgi-13.9-psm
soft add +netcdf-4.3.1-serial-pgi
setenv NETCDFROOT /soft/netcdf/4.3.1-serial/pgi-13.9
if ( $MPILIB == "openmpi") then
soft add +openmpi-1.8.2-pgi-13.9-psm
else if ( $MPILIB == "mpich") then
soft add +mpich2-1.4.1-pgi-13.9
endif
if ( $MPILIB == "mpi-serial") then
setenv PNETCDFROOT ""
else
soft add +pnetcdf-1.5.0-mpich2-pgi-13.9
setenv PNETCDFROOT /soft/pnetcdf/1.5.0/pgi-13.9/mpich2-1.4.1
endif
endif

if ( $COMPILER == "gnu" ) then
soft add +gcc-4.7.2
soft add +mvapich2-2.0-gcc-4.7.2-psm
soft add +netcdf-4.3.1-serial-gcc
setenv NETCDFROOT /soft/netcdf/4.3.1-serial/gcc-4.7.2
if ( $MPILIB == "openmpi") then
soft add +openmpi-1.6.5-gnu-4.7.2-psm
else if ( $MPILIB == "mpich") then
soft add +mpich2-1.4.1-gcc-4.7.2
endif
if ( $MPILIB == "mpi-serial") then
setenv PNETCDFROOT ""
else
soft add +pnetcdf-1.5.0-mpich2-gcc-4.7.2
setenv PNETCDFROOT /soft/pnetcdf/1.5.0/gcc-4.7.2/mpich2-1.4.1
endif
endif
6 changes: 5 additions & 1 deletion scripts/ccsm_utils/Machines/mkbatch.blues
View file
Original file line number Diff line number Diff line change
Expand Up @@ -55,7 +55,11 @@ cd \$RUNDIR
echo "\`date\` -- CSM EXECUTION BEGINS HERE"
setenv OMP_NUM_THREADS ${maxthrds}
mpiexec -n ${maxtasks} \${EXEROOT}/cesm.exe >&! cesm.log.\$LID
if ( "\$MPILIB" == "mpi-serial" ) then
\$EXEROOT/cesm.exe >&! cesm.log.\$LID
else
mpiexec -n ${maxtasks} \${EXEROOT}/cesm.exe >&! cesm.log.\$LID
endif
wait
echo "\`date\` -- CSM EXECUTION HAS FINISHED"
Expand Down
45 changes: 45 additions & 0 deletions scripts/ccsm_utils/Testlistxml/testlist.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -61,6 +61,9 @@
<machine compiler="ibm" testtype="prerelease">intrepid</machine>
</test>
<test name="ERS">
<machine compiler="gnu" testtype="acme_developer">blues</machine>
<machine compiler="intel" testtype="acme_developer">blues</machine>
<machine compiler="pgi" testtype="acme_developer">blues</machine>
<machine compiler="pgi" testtype="acme_developer">bluewaters</machine>
<machine compiler="intel" testtype="acme_developer">cascade</machine>
<machine compiler="nag" testtype="acme_developer">cascade</machine>
Expand Down Expand Up @@ -103,6 +106,9 @@
<machine compiler="intel" testtype="aux_drv">yellowstone</machine>
</test>
<test name="ERS_IOP">
<machine compiler="gnu" testtype="acme_developer">blues</machine>
<machine compiler="intel" testtype="acme_developer">blues</machine>
<machine compiler="pgi" testtype="acme_developer">blues</machine>
<machine compiler="pgi" testtype="acme_developer">bluewaters</machine>
<machine compiler="pgi" testtype="acme_integration">bluewaters</machine>
<machine compiler="intel" testtype="acme_developer">cascade</machine>
Expand Down Expand Up @@ -177,6 +183,9 @@
<machine compiler="pgi" testtype="prebeta">yellowstone</machine>
</test>
<test name="ERS_IOP4c">
<machine compiler="gnu" testtype="acme_developer">blues</machine>
<machine compiler="intel" testtype="acme_developer">blues</machine>
<machine compiler="pgi" testtype="acme_developer">blues</machine>
<machine compiler="pgi" testtype="acme_developer">bluewaters</machine>
<machine compiler="pgi" testtype="acme_integration">bluewaters</machine>
<machine compiler="intel" testtype="acme_developer">cascade</machine>
Expand Down Expand Up @@ -249,6 +258,9 @@
<machine compiler="intel" testtype="prebeta">yellowstone</machine>
</test>
<test name="ERS_IOP4p">
<machine compiler="gnu" testtype="acme_developer">blues</machine>
<machine compiler="intel" testtype="acme_developer">blues</machine>
<machine compiler="pgi" testtype="acme_developer">blues</machine>
<machine compiler="pgi" testtype="acme_developer">bluewaters</machine>
<machine compiler="pgi" testtype="acme_integration">bluewaters</machine>
<machine compiler="intel" testtype="acme_developer">cascade</machine>
Expand Down Expand Up @@ -334,6 +346,9 @@
<machine compiler="intel" testtype="prerelease">titan</machine>
</test>
<test name="NCK">
<machine compiler="gnu" testtype="acme_developer">blues</machine>
<machine compiler="intel" testtype="acme_developer">blues</machine>
<machine compiler="pgi" testtype="acme_developer">blues</machine>
<machine compiler="pgi" testtype="acme_developer">bluewaters</machine>
<machine compiler="pgi" testtype="acme_integration">bluewaters</machine>
<machine compiler="intel" testtype="acme_developer">cascade</machine>
Expand Down Expand Up @@ -468,6 +483,9 @@
<machine compiler="ibm" testtype="prerelease">intrepid</machine>
</test>
<test name="PEA_P1_M">
<machine compiler="gnu" testtype="acme_developer">blues</machine>
<machine compiler="intel" testtype="acme_developer">blues</machine>
<machine compiler="pgi" testtype="acme_developer">blues</machine>
<machine compiler="pgi" testtype="acme_developer">bluewaters</machine>
<machine compiler="pgi" testtype="acme_integration">bluewaters</machine>
<machine compiler="intel" testtype="acme_developer">cascade</machine>
Expand Down Expand Up @@ -595,6 +613,9 @@
<machine compiler="pgi" testtype="prerelease">titan</machine>
</test>
<test name="SMS">
<machine compiler="gnu" testtype="acme_developer">blues</machine>
<machine compiler="intel" testtype="acme_developer">blues</machine>
<machine compiler="pgi" testtype="acme_developer">blues</machine>
<machine compiler="pgi" testtype="acme_developer">bluewaters</machine>
<machine compiler="pgi" testtype="acme_integration">bluewaters</machine>
<machine compiler="intel" testtype="acme_developer">cascade</machine>
Expand Down Expand Up @@ -679,6 +700,9 @@
</grid>
<grid name="ne30_g16_rx1">
<test name="ERS">
<machine compiler="gnu" testtype="acme_developer">blues</machine>
<machine compiler="intel" testtype="acme_developer">blues</machine>
<machine compiler="pgi" testtype="acme_developer">blues</machine>
<machine compiler="pgi" testtype="acme_developer">bluewaters</machine>
<machine compiler="intel" testtype="acme_developer">cascade</machine>
<machine compiler="nag" testtype="acme_developer">cascade</machine>
Expand Down Expand Up @@ -726,6 +750,9 @@
<machine compiler="ibm" testtype="prebeta">mira</machine>
</test>
<test name="ERS_IOP">
<machine compiler="gnu" testtype="acme_developer">blues</machine>
<machine compiler="intel" testtype="acme_developer">blues</machine>
<machine compiler="pgi" testtype="acme_developer">blues</machine>
<machine compiler="pgi" testtype="acme_developer">bluewaters</machine>
<machine compiler="pgi" testtype="acme_integration">bluewaters</machine>
<machine compiler="intel" testtype="acme_developer">cascade</machine>
Expand Down Expand Up @@ -800,6 +827,9 @@
<machine compiler="pgi" testtype="prebeta">yellowstone</machine>
</test>
<test name="ERS_IOP4c">
<machine compiler="gnu" testtype="acme_developer">blues</machine>
<machine compiler="intel" testtype="acme_developer">blues</machine>
<machine compiler="pgi" testtype="acme_developer">blues</machine>
<machine compiler="pgi" testtype="acme_developer">bluewaters</machine>
<machine compiler="pgi" testtype="acme_integration">bluewaters</machine>
<machine compiler="intel" testtype="acme_developer">cascade</machine>
Expand Down Expand Up @@ -872,6 +902,9 @@
<machine compiler="intel" testtype="prebeta">yellowstone</machine>
</test>
<test name="ERS_IOP4p">
<machine compiler="gnu" testtype="acme_developer">blues</machine>
<machine compiler="intel" testtype="acme_developer">blues</machine>
<machine compiler="pgi" testtype="acme_developer">blues</machine>
<machine compiler="pgi" testtype="acme_developer">bluewaters</machine>
<machine compiler="pgi" testtype="acme_integration">bluewaters</machine>
<machine compiler="intel" testtype="acme_developer">cascade</machine>
Expand Down Expand Up @@ -1273,6 +1306,9 @@
<machine compiler="intel" testtype="aux_cice">yellowstone</machine>
</test>
<test name="ERS">
<machine compiler="gnu" testtype="acme_developer">blues</machine>
<machine compiler="intel" testtype="acme_developer">blues</machine>
<machine compiler="pgi" testtype="acme_developer">blues</machine>
<machine compiler="pgi" testtype="acme_developer">bluewaters</machine>
<machine compiler="pgi" testtype="acme_integration">bluewaters</machine>
<machine compiler="intel" testtype="acme_developer">cascade</machine>
Expand Down Expand Up @@ -2543,6 +2579,9 @@
<machine compiler="pgi" testtype="prebeta">yellowstone</machine>
</test>
<test name="ERS">
<machine compiler="gnu" testtype="acme_developer">blues</machine>
<machine compiler="intel" testtype="acme_developer">blues</machine>
<machine compiler="pgi" testtype="acme_developer">blues</machine>
<machine compiler="pgi" testtype="acme_developer">bluewaters</machine>
<machine compiler="intel" testtype="acme_developer">cascade</machine>
<machine compiler="nag" testtype="acme_developer">cascade</machine>
Expand Down Expand Up @@ -2582,6 +2621,9 @@
<machine compiler="pgi" testtype="acme_developer">yellowstone</machine>
</test>
<test name="ERS_IOP">
<machine compiler="gnu" testtype="acme_developer">blues</machine>
<machine compiler="intel" testtype="acme_developer">blues</machine>
<machine compiler="pgi" testtype="acme_developer">blues</machine>
<machine compiler="pgi" testtype="acme_developer">bluewaters</machine>
<machine compiler="pgi" testtype="acme_integration">bluewaters</machine>
<machine compiler="intel" testtype="acme_developer">cascade</machine>
Expand Down Expand Up @@ -5621,6 +5663,9 @@
<compset name="TG">
<grid name="f09_g16">
<test name="ERS_Ly21">
<machine compiler="gnu" testtype="acme_developer">blues</machine>
<machine compiler="intel" testtype="acme_developer">blues</machine>
<machine compiler="pgi" testtype="acme_developer">blues</machine>
<machine compiler="pgi" testtype="acme_developer">bluewaters</machine>
<machine compiler="pgi" testtype="acme_integration">bluewaters</machine>
<machine compiler="intel" testtype="acme_developer">cascade</machine>
Expand Down

0 comments on commit b90c649

Please sign in to comment.