From 515c55db683c3d311c496986ea2bf0fcf56d5c82 Mon Sep 17 00:00:00 2001
From: Balwinder Singh <balwindersingh@gmail.com>
Date: Thu, 23 Feb 2017 11:58:16 -0800
Subject: [PATCH 1/2] Adds machine files for PNNL-EMSL cluster cascade

[BFB] - Bit-For-Bit
---
 cime_config/acme/machines/config_compilers.xml | 9 ++-------
 cime_config/acme/machines/config_machines.xml  | 4 ++--
 2 files changed, 4 insertions(+), 9 deletions(-)

diff --git a/cime_config/acme/machines/config_compilers.xml b/cime_config/acme/machines/config_compilers.xml
index 237cd880b31..e15715f7bc3 100644
--- a/cime_config/acme/machines/config_compilers.xml
+++ b/cime_config/acme/machines/config_compilers.xml
@@ -1101,19 +1101,14 @@ for mct, etc.
 
 
 <compiler COMPILER="intel" MACH="cascade">
-  <NETCDF_PATH> $(NETCDF_LIB)/..</NETCDF_PATH>
+  <NETCDF_PATH> $(NETCDF_HOME)</NETCDF_PATH>
   <PNETCDF_PATH>$(PNETCDFROOT)</PNETCDF_PATH>
   <PIO_FILESYSTEM_HINTS>lustre</PIO_FILESYSTEM_HINTS>
   <CONFIG_ARGS> --host=Linux --enable-filesystem-hints=lustre</CONFIG_ARGS>
   <ADD_CPPDEFS> -DLINUX </ADD_CPPDEFS>
   <ADD_FFLAGS DEBUG="TRUE"> -g -traceback  -O0 -fpe0 -check  all -check noarg_temp_created -ftrapuv </ADD_FFLAGS>
-  <SLIBS> -L$(NETCDF_PATH)/lib -lnetcdf -lnetcdff -lpmi -L$(MKL_PATH) -lmkl_rt </SLIBS>
-  <ADD_SLIBS MPILIB="mpich"> -mkl=cluster </ADD_SLIBS>
+  <SLIBS> -L$(NETCDF_PATH)/lib -lnetcdf -lnetcdff -lpmi -L$(MKL_PATH)/lib/intel64 -lmkl_rt </SLIBS>
   <ADD_SLIBS MPILIB="mpich2"> -mkl=cluster </ADD_SLIBS>
-  <ADD_SLIBS MPILIB="mpt"> -mkl=cluster </ADD_SLIBS>
-  <ADD_SLIBS MPILIB="openmpi"> -mkl=cluster </ADD_SLIBS>
-  <ADD_SLIBS MPILIB="mvapich"> -mkl=cluster </ADD_SLIBS>
-  <ADD_SLIBS MPILIB="impi"> -mkl=cluster </ADD_SLIBS>
   <ADD_SLIBS MPILIB="mpi-serial"> -mkl </ADD_SLIBS>
 </compiler>
 
diff --git a/cime_config/acme/machines/config_machines.xml b/cime_config/acme/machines/config_machines.xml
index 9a634aaea4b..4de26fc4ce5 100644
--- a/cime_config/acme/machines/config_machines.xml
+++ b/cime_config/acme/machines/config_machines.xml
@@ -1279,8 +1279,8 @@
          <DIN_LOC_ROOT_CLMFORC>/dtemp/sing201/inputdata/CAM/CSMDATA_CAM/aerocom/csmdata/atm/datm7</DIN_LOC_ROOT_CLMFORC>
          <DOUT_S_ROOT>/dtemp/$USER/archive/$CASE</DOUT_S_ROOT>
          <DOUT_L_MSROOT>csm/$CASE</DOUT_L_MSROOT>
-         <BASELINE_ROOT>/dtemp/sing201/acme_testing/acme_baselines/</BASELINE_ROOT>
-             <CIME_OUTPUT_ROOT>/dtemp/$USER/csmruns</CIME_OUTPUT_ROOT>
+         <CCSM_BASELINE>/dtemp/sing201/acme_testing/acme_baselines/</CCSM_BASELINE>
+             <CESMSCRATCHROOT>/dtemp/$USER/csmruns</CESMSCRATCHROOT>
          <CCSM_CPRNC>/home/sing201/CAM/cprnc/cprnc</CCSM_CPRNC>
 	 <BATCH_SYSTEM>slurm</BATCH_SYSTEM>
          <SUPPORTED_BY>balwinder.singh at pnnl dot gov</SUPPORTED_BY>

From 963cdae233574aa6b32f4350c75747004e2ae062 Mon Sep 17 00:00:00 2001
From: Balwinder Singh <balwindersingh@gmail.com>
Date: Fri, 14 Apr 2017 15:34:36 -0700
Subject: [PATCH 2/2] Modified batch system to slurm and fixed an environment
 issue.

[BFB] - Bit-For-Bit
---
 cime_config/acme/machines/config_batch.xml    | 10 ++
 cime_config/acme/machines/config_machines.xml | 91 ++++++++++++++-----
 cime_config/acme/machines/config_pio.xml      |  1 +
 3 files changed, 80 insertions(+), 22 deletions(-)

diff --git a/cime_config/acme/machines/config_batch.xml b/cime_config/acme/machines/config_batch.xml
index 48893de566c..d83de75e727 100644
--- a/cime_config/acme/machines/config_batch.xml
+++ b/cime_config/acme/machines/config_batch.xml
@@ -224,6 +224,16 @@
         <queue walltimemax="01:00:00" jobmin="1" jobmax="65536" default="true">default</queue>
       </queues>
     </batch_system>
+    
+    <batch_system MACH="cascade" type="slurm">
+      <directives>
+	<directive>--mail-type=END</directive>
+	<directive>--mail-user=email@pnnl.gov</directive>
+      </directives>
+      <queues>
+	<queue walltimemax="00:30:00" jobmin="1" jobmax="9999" default="true">small</queue>
+      </queues>
+    </batch_system>
 
    <batch_system MACH="constance" type="slurm">
     <directives>
diff --git a/cime_config/acme/machines/config_machines.xml b/cime_config/acme/machines/config_machines.xml
index 4de26fc4ce5..2537acdda17 100644
--- a/cime_config/acme/machines/config_machines.xml
+++ b/cime_config/acme/machines/config_machines.xml
@@ -1266,28 +1266,75 @@
     </environment_variables>
   </machine>
 
-<machine MACH="cascade">
-         <DESC>PNL cluster, os is Linux (pgi), batch system is SLURM</DESC>
-         <NODENAME_REGEX>cascade</NODENAME_REGEX>
-         <TESTS>acme_developer</TESTS>
-         <OS>LINUX</OS>
-         <COMPILERS>intel,nag</COMPILERS>
-         <MPILIBS>mpich</MPILIBS>
-         <RUNDIR>/dtemp/$USER/csmruns/$CASE/run</RUNDIR>
-         <EXEROOT>/dtemp/$USER/csmruns/$CASE/bld</EXEROOT>
-         <DIN_LOC_ROOT>/dtemp/sing201/inputdata/CAM/CSMDATA_CAM/aerocom/csmdata</DIN_LOC_ROOT>
-         <DIN_LOC_ROOT_CLMFORC>/dtemp/sing201/inputdata/CAM/CSMDATA_CAM/aerocom/csmdata/atm/datm7</DIN_LOC_ROOT_CLMFORC>
-         <DOUT_S_ROOT>/dtemp/$USER/archive/$CASE</DOUT_S_ROOT>
-         <DOUT_L_MSROOT>csm/$CASE</DOUT_L_MSROOT>
-         <CCSM_BASELINE>/dtemp/sing201/acme_testing/acme_baselines/</CCSM_BASELINE>
-             <CESMSCRATCHROOT>/dtemp/$USER/csmruns</CESMSCRATCHROOT>
-         <CCSM_CPRNC>/home/sing201/CAM/cprnc/cprnc</CCSM_CPRNC>
-	 <BATCH_SYSTEM>slurm</BATCH_SYSTEM>
-         <SUPPORTED_BY>balwinder.singh at pnnl dot gov</SUPPORTED_BY>
-         <GMAKE_J>8</GMAKE_J>
-         <MAX_TASKS_PER_NODE>16</MAX_TASKS_PER_NODE>
-</machine>
-
+  <machine MACH="cascade">
+    <DESC>PNL Intel KNC cluster, OS is Linux, batch system is SLURM</DESC>
+    <OS>LINUX</OS>
+    <COMPILERS>intel</COMPILERS>
+    <MPILIBS>mvapich2</MPILIBS>
+    <NODENAME_REGEX>glogin</NODENAME_REGEX>
+    <CESMSCRATCHROOT>/dtemp/$USER</CESMSCRATCHROOT>
+    <RUNDIR>/dtemp/$USER/csmruns/$CASE/run</RUNDIR>
+    <EXEROOT>/dtemp/$USER/csmruns/$CASE/bld</EXEROOT>
+    <CIME_OUTPUT_ROOT>/dtemp/$USER</CIME_OUTPUT_ROOT>
+    <DIN_LOC_ROOT>/dtemp/sing201/acme/inputdata/</DIN_LOC_ROOT>
+    <DIN_LOC_ROOT_CLMFORC>/dtemp/sing201/acme/inputdata/atm/datm7</DIN_LOC_ROOT_CLMFORC>
+    <DOUT_S_ROOT>/dtemp/sing201/$USER/archive/$CASE</DOUT_S_ROOT>
+    <DOUT_L_MSROOT>UNSET</DOUT_L_MSROOT>
+    <CCSM_BASELINE>/dtemp/sing201/acme/acme_baselines</CCSM_BASELINE>
+    <CCSM_CPRNC>/dtemp/sing201/acme/acme_baselines/cprnc/cprnc</CCSM_CPRNC>
+    <SAVE_TIMING_DIR>/dtemp/sing201/acme/timing_acme</SAVE_TIMING_DIR>
+    <BATCH_SYSTEM>slurm</BATCH_SYSTEM>
+    <SUPPORTED_BY>balwinder.singh -at- pnnl.gov</SUPPORTED_BY>
+    <GMAKE_J>8</GMAKE_J>
+    <MAX_TASKS_PER_NODE>16</MAX_TASKS_PER_NODE>
+    <PES_PER_NODE>16</PES_PER_NODE>
+    <PROJECT_REQUIRED>TRUE</PROJECT_REQUIRED>
+    <mpirun mpilib="mpi-serial">
+      <executable></executable>
+    </mpirun>
+    <mpirun mpilib="mvapich2">
+      <executable>srun</executable>
+      <arguments>
+	<arg name="mpi">--mpi=none</arg>
+	<arg name="num_tasks">--ntasks=$TOTALPES</arg>
+	<arg name="cpu_bind">--cpu_bind=sockets --cpu_bind=verbose</arg>
+	<arg name="kill-on-bad-exit">--kill-on-bad-exit</arg>
+      </arguments>
+    </mpirun>
+    <module_system type="module">
+      <init_path lang="python">/opt/lmod/5.0.1/init/env_modules_python.py</init_path>
+      <init_path lang="csh">/etc/profile.d/modules.csh</init_path>
+      <init_path lang="sh">/etc/profile.d/modules.sh</init_path>
+      <cmd_path lang="python">/opt/lmod/lmod/libexec/lmod python</cmd_path>
+      <cmd_path lang="sh">module</cmd_path>
+      <cmd_path lang="csh">module</cmd_path>
+      <modules>
+	<command name="purge"></command>
+      </modules>
+      <modules>
+	<command name="load">python/2.7.9</command>
+      </modules>
+      <modules compiler="intel">
+	<command name="load">intel/15.0.090</command>
+	<command name="load">mkl/14.0</command>
+      </modules>
+      <modules mpilib="mvapich2">
+	<command name="load">mvapich2/1.9</command>
+      </modules>
+      <modules>
+	<command name="load">netcdf/4.3.0</command>
+      </modules>
+    </module_system>
+    <environment_variables>
+      <env name="OMP_STACKSIZE">64M</env>
+      <env name="NETCDF_HOME">$ENV{NETCDF_ROOT}</env>
+    </environment_variables>
+    <environment_variables compiler="intel">
+      <env name="MKL_PATH">$ENV{MLIBHOME}</env>
+      <env name="COMPILER">intel</env>
+    </environment_variables>
+  </machine>
+  
 <machine MACH="constance">
          <DESC>PNL Haswell cluster, OS is Linux, batch system is SLURM</DESC>
          <OS>LINUX</OS>
diff --git a/cime_config/acme/machines/config_pio.xml b/cime_config/acme/machines/config_pio.xml
index e1716e68c5b..e342715251d 100644
--- a/cime_config/acme/machines/config_pio.xml
+++ b/cime_config/acme/machines/config_pio.xml
@@ -48,6 +48,7 @@
       <value mach="userdefined">netcdf</value>
       <value mach="eastwind">netcdf</value>
       <value mach="constance">netcdf</value>
+      <value mach="cascade">netcdf</value>
       <value mach="sooty">netcdf</value>
       <value mach="pleiades.*">netcdf</value>
       <value mach="hobart" compiler="pgi">netcdf</value>