Add support for LANL machine files (Mustang and Wolf)

scripts/ccsm_utils/Machines/config_compilers.xml

@@ -1029,4 +1029,64 @@ for mct, etc.
   <TRILINOS_PATH>$(TRILINOS_PATH)</TRILINOS_PATH>
 </compiler>
 
+<compiler MACH="mustang" COMPILER="pgi">
+	<MPICC>mpicc</MPICC>
+	<MPIFC>mpif90</MPIFC>
+	<MPICXX>mpic++</MPICXX>
+	<SFC>pgfortran</SFC>
+	<SCC>pgcc</SCC>
+	<SCXX>pgc++</SCXX>
+	<ADD_SLIBS>$(shell $(NETCDF_PATH)/bin/nf-config --flibs) -llapack -lblas</ADD_SLIBS>
+</compiler>
+
+<compiler MACH="mustang" COMPILER="intel">
+	<MPICC>mpicc</MPICC>
+	<MPIFC>mpif90</MPIFC>
+	<MPICXX>mpic++</MPICXX>
+	<SFC>ifort</SFC>
+	<SCC>icc</SCC>
+	<SCXX>icpc</SCXX>
+	<ADD_SLIBS>$(shell $(NETCDF_PATH)/bin/nf-config --flibs) -llapack -lblas</ADD_SLIBS>
+</compiler>
+
+<compiler MACH="mustang" COMPILER="gnu">
+	<MPICC>mpicc</MPICC>
+	<MPIFC>mpif90</MPIFC>
+	<MPICXX>mpic++</MPICXX>
+	<SFC>gfortran</SFC>
+	<SCC>gcc</SCC>
+	<SCXX>g++</SCXX>
+	<ADD_SLIBS>$(shell $(NETCDF_PATH)/bin/nf-config --flibs) -llapack -lblas</ADD_SLIBS>
+</compiler>
+
+<compiler MACH="wolf" COMPILER="pgi">
+	<MPICC>mpicc</MPICC>
+	<MPIFC>mpif90</MPIFC>
+	<MPICXX>mpic++</MPICXX>
+	<SFC>pgfortran</SFC>
+	<SCC>pgcc</SCC>
+	<SCXX>pgc++</SCXX>
+	<ADD_SLIBS>$(shell $(NETCDF_PATH)/bin/nf-config --flibs) -llapack -lblas</ADD_SLIBS>
+</compiler>
+
+<compiler MACH="wolf" COMPILER="intel">
+	<MPICC>mpicc</MPICC>
+	<MPIFC>mpif90</MPIFC>
+	<MPICXX>mpic++</MPICXX>
+	<SFC>ifort</SFC>
+	<SCC>icc</SCC>
+	<SCXX>icpc</SCXX>
+	<ADD_SLIBS>$(shell $(NETCDF_PATH)/bin/nf-config --flibs) -llapack -lblas</ADD_SLIBS>
+</compiler>
+
+<compiler MACH="wolf" COMPILER="gnu">
+	<MPICC>mpicc</MPICC>
+	<MPIFC>mpif90</MPIFC>
+	<MPICXX>mpic++</MPICXX>
+	<SFC>gfortran</SFC>
+	<SCC>gcc</SCC>
+	<SCXX>g++</SCXX>
+	<ADD_SLIBS>$(shell $(NETCDF_PATH)/bin/nf-config --flibs) -llapack -lblas</ADD_SLIBS>
+</compiler>
+
 </config_compilers>

scripts/ccsm_utils/Machines/config_machines.xml

@@ -808,4 +808,42 @@
         <MAX_TASKS_PER_NODE>32</MAX_TASKS_PER_NODE>
 </machine>
 
+<machine MACH="mustang">
+         <DESC>LANL Linux Cluster, 24 pes/node, batch system Moab</DESC>
+         <COMPILERS>intel,gnu</COMPILERS>
+         <MPILIBS>openmpi,mvapich</MPILIBS>
+         <OS>LINUX</OS>
+         <RUNDIR>/lustre/scratch1/turquoise/$ENV{USER}/ACME/cases/$CASE/run</RUNDIR>
+         <EXEROOT>/lustre/scratch1/turquoise/$ENV{USER}/ACME/cases/$CASE/bld</EXEROOT>
+         <DIN_LOC_ROOT>/lustre/scratch1/turquoise/$ENV{USER}/ACME/input_data</DIN_LOC_ROOT>
+         <DIN_LOC_ROOT_CLMFORC>/lustre/scratch1/turquoise/$ENV{USER}/ACME/input_data/atm/datm7</DIN_LOC_ROOT_CLMFORC>
+         <DOUT_S_ROOT>/lustre/scratch1/turquoise/$ENV{USER}/ACME/archive/$CASE</DOUT_S_ROOT>
+         <DOUT_L_MSROOT>UNSET</DOUT_L_MSROOT>
+         <CCSM_BASELINE>/lustre/scratch1/turquoise/$ENV{USER}/ACME/input_data/ccsm_baselines</CCSM_BASELINE>
+         <BATCHQUERY>mshow</BATCHQUERY>
+         <BATCHSUBMIT>msub</BATCHSUBMIT>
+         <GMAKE_J>4</GMAKE_J>
+         <MAX_TASKS_PER_NODE>24</MAX_TASKS_PER_NODE>
+		 <SUPPORTED_BY>jacobsen.douglas -at- gmail.com</SUPPORTED_BY>
+</machine>
+
+<machine MACH="wolf">
+         <DESC>LANL Linux Cluster, 16 pes/node, batch system Moab</DESC>
+         <COMPILERS>intel,gnu</COMPILERS>
+         <MPILIBS>openmpi,mvapich</MPILIBS>
+         <OS>LINUX</OS>
+         <RUNDIR>/lustre/scratch1/turquoise/$ENV{USER}/ACME/cases/$CASE/run</RUNDIR>
+         <EXEROOT>/lustre/scratch1/turquoise/$ENV{USER}/ACME/cases/$CASE/bld</EXEROOT>
+         <DIN_LOC_ROOT>/lustre/scratch1/turquoise/$ENV{USER}/ACME/input_data</DIN_LOC_ROOT>
+         <DIN_LOC_ROOT_CLMFORC>/lustre/scratch1/turquoise/$ENV{USER}/ACME/input_data/atm/datm7</DIN_LOC_ROOT_CLMFORC>
+         <DOUT_S_ROOT>/lustre/scratch1/turquoise/$ENV{USER}/ACME/archive/$CASE</DOUT_S_ROOT>
+         <DOUT_L_MSROOT>UNSET</DOUT_L_MSROOT>
+         <CCSM_BASELINE>/lustre/scratch1/turquoise/$ENV{USER}/ACME/input_data/ccsm_baselines</CCSM_BASELINE>
+         <BATCHQUERY>mshow</BATCHQUERY>
+         <BATCHSUBMIT>msub</BATCHSUBMIT>
+         <GMAKE_J>4</GMAKE_J>
+         <MAX_TASKS_PER_NODE>16</MAX_TASKS_PER_NODE>
+		 <SUPPORTED_BY>jacobsen.douglas -at- gmail.com</SUPPORTED_BY>
+</machine>
+
 </config_machines>

scripts/ccsm_utils/Machines/env_mach_specific.mustang

@@ -0,0 +1,35 @@
+#! /bin/csh -f
+
+#-------------------------------------------------------------------------------
+# Modules and library paths
+#-------------------------------------------------------------------------------
+source /usr/share/Modules/init/csh
+module purge
+module use /usr/projects/climate/SHARED_CLIMATE/modulefiles/all
+
+if ( $COMPILER == "intel" ) then
+	module load intel/13.1.3
+endif
+
+if ( $COMPILER == "pgi" ) then
+	# 13.10 is too old to support
+	# PGI support is currently disabled until we have access to a newer version
+	# (i.e. 14.10)
+	module load pgi/13.10
+endif
+
+if ( $COMPILER == "gnu" ) then
+	module load gcc/4.8.2
+endif
+
+if ( $MPILIB == "openmpi" ) then
+	module load openmpi/1.6.5
+endif
+
+if ( $MPILIB == "mvapich" ) then
+	module load mvapich2/1.8
+endif
+
+module load netcdf/4.4.0
+module load parallel-netcdf/1.5.0
+module load cmake/3.0.0

scripts/ccsm_utils/Machines/env_mach_specific.wolf

@@ -0,0 +1,35 @@
+#! /bin/csh -f
+
+#-------------------------------------------------------------------------------
+# Modules and library paths
+#-------------------------------------------------------------------------------
+source /usr/share/Modules/init/csh
+module purge
+module use /usr/projects/climate/SHARED_CLIMATE/modulefiles/all
+
+if ( $COMPILER == "intel" ) then
+	module load intel/13.1.3
+endif
+
+if ( $COMPILER == "pgi" ) then
+	# 13.10 is too old to support
+	# PGI support is currently disabled until we have access to a newer version
+	# (i.e. 14.10)
+	module load pgi/13.10
+endif
+
+if ( $COMPILER == "gnu" ) then
+	module load gcc/4.8.2
+endif
+
+if ( $MPILIB == "openmpi" ) then
+	module load openmpi/1.6.5
+endif
+
+if ( $MPILIB == "mvapich" ) then
+	module load mvapich2/1.8
+endif
+
+module load netcdf/4.4.0
+module load parallel-netcdf/1.5.0
+module load cmake/3.0.0

scripts/ccsm_utils/Machines/mkbatch.mustang

@@ -0,0 +1,106 @@
+#! /bin/csh -f
+
+#################################################################################
+if ($PHASE == set_batch) then
+#################################################################################
+
+source ./Tools/ccsm_getenv || exit -1
+
+set ntasks  = `${CASEROOT}/Tools/taskmaker.pl -sumtasks`
+set maxthrds = `${CASEROOT}/Tools/taskmaker.pl -maxthrds`
+@ nodes = $ntasks / ${MAX_TASKS_PER_NODE}
+if ( $ntasks % ${MAX_TASKS_PER_NODE} > 0) then
+	@ nodes = $nodes + 1
+	@ ntasks = $nodes * ${MAX_TASKS_PER_NODE}
+endif
+@ taskpernode = ${MAX_TASKS_PER_NODE} / ${maxthrds}
+
+#--- Job name is first fifteen characters of case name ---
+set jobname = `echo ${CASE} | cut -c1-15`
+
+set walltime = "01:00:00"
+if ($CCSM_ESTCOST > 1) set walltime = "02:00:00"
+if ($CCSM_ESTCOST > 3) set walltime = "10:00:00"
+
+if ($?TESTMODE) then
+	set file = $CASEROOT/${CASE}.test 
+else
+	set file = $CASEROOT/${CASE}.run 
+endif
+
+if ( "${PROJECT}" == "PROJECT_UNSET" ) then
+	set account = ""
+else
+	set account = "#MSUB -A ${PROJECT}"
+endif
+
+cat >! $file << EOF1
+#!/bin/csh
+#MSUB -N ${jobname}
+#MSUB -l nodes=${nodes}:ppn=${MAX_TASKS_PER_NODE}
+#MSUB -l walltime=${walltime}
+#MSUB -o $CASEROOT/log.o
+#MSUB -e $CASEROOT/log.e
+${account}
+
+cd $CASEROOT
+source ./Tools/ccsm_getenv || exit -1
+
+EOF1
+
+#################################################################################
+else if ($PHASE == set_exe) then
+#################################################################################
+
+source ./Tools/ccsm_getenv || exit -1
+
+set maxthrds = `${CASEROOT}/Tools/taskmaker.pl -maxthrds`
+set ntasks_tot = `${CASEROOT}/Tools/taskmaker.pl -sumtasks`
+
+cat >> $CASEROOT/${CASE}.run << EOF1
+# -------------------------------------------------------------------------
+# Run the model
+# -------------------------------------------------------------------------
+cd \$RUNDIR
+echo "\`date\` -- CSM EXECUTION BEGINS HERE" 
+
+setenv MP_LABELIO yes
+sleep 10
+setenv OMP_NUM_THREADS ${maxthrds}
+if ( "\$MPILIB" == "openmpi" ) then
+	mpirun -np ${ntasks_tot}  $EXEROOT/cesm.exe >&! cesm.log.\$LID
+endif
+
+if ( "\$MPILIB" == "mvapich" ) then
+	srun -n ${ntasks_tot}  $EXEROOT/cesm.exe >&! cesm.log.\$LID
+endif
+wait
+
+wait
+echo "\`date\` -- CSM EXECUTION HAS FINISHED" 
+EOF1
+
+
+#################################################################################
+else if ($PHASE == set_larch) then
+#################################################################################
+
+cat >! $CASEROOT/${CASE}.l_archive << EOF1
+#! /bin/csh -f
+
+echo 'sorry, no archiving script'
+
+EOF1
+chmod 775 ${CASEROOT}/${CASE}.l_archive
+
+#################################################################################
+else
+#################################################################################
+
+    echo "  PHASE setting of $PHASE is not an accepted value"
+    echo "  accepted values are set_batch, set_exe and set_larch"
+    exit 1
+
+#################################################################################
+endif
+#################################################################################

scripts/ccsm_utils/Machines/mkbatch.wolf

@@ -0,0 +1,106 @@
+#! /bin/csh -f
+
+#################################################################################
+if ($PHASE == set_batch) then
+#################################################################################
+
+source ./Tools/ccsm_getenv || exit -1
+
+set ntasks  = `${CASEROOT}/Tools/taskmaker.pl -sumtasks`
+set maxthrds = `${CASEROOT}/Tools/taskmaker.pl -maxthrds`
+@ nodes = $ntasks / ${MAX_TASKS_PER_NODE}
+if ( $ntasks % ${MAX_TASKS_PER_NODE} > 0) then
+	@ nodes = $nodes + 1
+	@ ntasks = $nodes * ${MAX_TASKS_PER_NODE}
+endif
+@ taskpernode = ${MAX_TASKS_PER_NODE} / ${maxthrds}
+
+#--- Job name is first fifteen characters of case name ---
+set jobname = `echo ${CASE} | cut -c1-15`
+
+set walltime = "01:00:00"
+if ($CCSM_ESTCOST > 1) set walltime = "02:00:00"
+if ($CCSM_ESTCOST > 3) set walltime = "10:00:00"
+
+if ($?TESTMODE) then
+	set file = $CASEROOT/${CASE}.test 
+else
+	set file = $CASEROOT/${CASE}.run 
+endif
+
+if ( "${PROJECT}" == "PROJECT_UNSET" ) then
+	set account = ""
+else
+	set account = "#MSUB -A ${PROJECT}"
+endif
+
+cat >! $file << EOF1
+#!/bin/csh
+#MSUB -N ${jobname}
+#MSUB -l nodes=${nodes}:ppn=${MAX_TASKS_PER_NODE}
+#MSUB -l walltime=${walltime}
+#MSUB -o $CASEROOT/log.o
+#MSUB -e $CASEROOT/log.e
+${account}
+
+cd $CASEROOT
+source ./Tools/ccsm_getenv || exit -1
+
+EOF1
+
+#################################################################################
+else if ($PHASE == set_exe) then
+#################################################################################
+
+source ./Tools/ccsm_getenv || exit -1
+
+set maxthrds = `${CASEROOT}/Tools/taskmaker.pl -maxthrds`
+set ntasks_tot = `${CASEROOT}/Tools/taskmaker.pl -sumtasks`
+
+cat >> $CASEROOT/${CASE}.run << EOF1
+# -------------------------------------------------------------------------
+# Run the model
+# -------------------------------------------------------------------------
+cd \$RUNDIR
+echo "\`date\` -- CSM EXECUTION BEGINS HERE" 
+
+setenv MP_LABELIO yes
+sleep 10
+setenv OMP_NUM_THREADS ${maxthrds}
+if ( "\$MPILIB" == "openmpi" ) then
+	mpirun -np ${ntasks_tot}  $EXEROOT/cesm.exe >&! cesm.log.\$LID
+endif
+
+if ( "\$MPILIB" == "mvapich" ) then
+	srun -n ${ntasks_tot}  $EXEROOT/cesm.exe >&! cesm.log.\$LID
+endif
+wait
+
+wait
+echo "\`date\` -- CSM EXECUTION HAS FINISHED" 
+EOF1
+
+
+#################################################################################
+else if ($PHASE == set_larch) then
+#################################################################################
+
+cat >! $CASEROOT/${CASE}.l_archive << EOF1
+#! /bin/csh -f
+
+echo 'sorry, no archiving script'
+
+EOF1
+chmod 775 ${CASEROOT}/${CASE}.l_archive
+
+#################################################################################
+else
+#################################################################################
+
+    echo "  PHASE setting of $PHASE is not an accepted value"
+    echo "  accepted values are set_batch, set_exe and set_larch"
+    exit 1
+
+#################################################################################
+endif
+#################################################################################