Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add machine files for PNNL-EMSL cluster Cascade #1285

Merged
merged 2 commits into from
Apr 15, 2017
Merged

Add machine files for PNNL-EMSL cluster Cascade #1285

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
1b9cff6
Adds machine files for PNNL-EMSL cluster cascade
singhbalwinder Feb 23, 2017
07559ae
Modified batch system to slurm and fixed an environment issue.
singhbalwinder Apr 14, 2017
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 10 additions & 0 deletions cime/cime_config/acme/machines/config_batch.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -224,6 +224,16 @@
<queue walltimemax="01:00:00" jobmin="1" jobmax="65536" default="true">default</queue>
</queues>
</batch_system>

<batch_system MACH="cascade" type="slurm">
<directives>
<directive>--mail-type=END</directive>
<directive>--mail-user=email@pnnl.gov</directive>
</directives>
<queues>
<queue walltimemax="00:30:00" jobmin="1" jobmax="9999" default="true">small</queue>
</queues>
</batch_system>

<batch_system MACH="constance" type="slurm">
<directives>
Expand Down
9 changes: 2 additions & 7 deletions cime/cime_config/acme/machines/config_compilers.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1101,19 +1101,14 @@ for mct, etc.


<compiler COMPILER="intel" MACH="cascade">
<NETCDF_PATH> $(NETCDF_LIB)/..</NETCDF_PATH>
<NETCDF_PATH> $(NETCDF_HOME)</NETCDF_PATH>
<PNETCDF_PATH>$(PNETCDFROOT)</PNETCDF_PATH>
<PIO_FILESYSTEM_HINTS>lustre</PIO_FILESYSTEM_HINTS>
<CONFIG_ARGS> --host=Linux --enable-filesystem-hints=lustre</CONFIG_ARGS>
<ADD_CPPDEFS> -DLINUX </ADD_CPPDEFS>
<ADD_FFLAGS DEBUG="TRUE"> -g -traceback -O0 -fpe0 -check all -check noarg_temp_created -ftrapuv </ADD_FFLAGS>
<SLIBS> -L$(NETCDF_PATH)/lib -lnetcdf -lnetcdff -lpmi -L$(MKL_PATH) -lmkl_rt </SLIBS>
<ADD_SLIBS MPILIB="mpich"> -mkl=cluster </ADD_SLIBS>
<SLIBS> -L$(NETCDF_PATH)/lib -lnetcdf -lnetcdff -lpmi -L$(MKL_PATH)/lib/intel64 -lmkl_rt </SLIBS>
<ADD_SLIBS MPILIB="mpich2"> -mkl=cluster </ADD_SLIBS>
<ADD_SLIBS MPILIB="mpt"> -mkl=cluster </ADD_SLIBS>
<ADD_SLIBS MPILIB="openmpi"> -mkl=cluster </ADD_SLIBS>
<ADD_SLIBS MPILIB="mvapich"> -mkl=cluster </ADD_SLIBS>
<ADD_SLIBS MPILIB="impi"> -mkl=cluster </ADD_SLIBS>
<ADD_SLIBS MPILIB="mpi-serial"> -mkl </ADD_SLIBS>
</compiler>

Expand Down
91 changes: 69 additions & 22 deletions cime/cime_config/acme/machines/config_machines.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1266,28 +1266,75 @@
</environment_variables>
</machine>

<machine MACH="cascade">
<DESC>PNL cluster, os is Linux (pgi), batch system is SLURM</DESC>
<NODENAME_REGEX>cascade</NODENAME_REGEX>
<TESTS>acme_developer</TESTS>
<OS>LINUX</OS>
<COMPILERS>intel,nag</COMPILERS>
<MPILIBS>mpich</MPILIBS>
<RUNDIR>/dtemp/$USER/csmruns/$CASE/run</RUNDIR>
<EXEROOT>/dtemp/$USER/csmruns/$CASE/bld</EXEROOT>
<DIN_LOC_ROOT>/dtemp/sing201/inputdata/CAM/CSMDATA_CAM/aerocom/csmdata</DIN_LOC_ROOT>
<DIN_LOC_ROOT_CLMFORC>/dtemp/sing201/inputdata/CAM/CSMDATA_CAM/aerocom/csmdata/atm/datm7</DIN_LOC_ROOT_CLMFORC>
<DOUT_S_ROOT>/dtemp/$USER/archive/$CASE</DOUT_S_ROOT>
<DOUT_L_MSROOT>csm/$CASE</DOUT_L_MSROOT>
<BASELINE_ROOT>/dtemp/sing201/acme_testing/acme_baselines/</BASELINE_ROOT>
<CIME_OUTPUT_ROOT>/dtemp/$USER/csmruns</CIME_OUTPUT_ROOT>
<CCSM_CPRNC>/home/sing201/CAM/cprnc/cprnc</CCSM_CPRNC>
<BATCH_SYSTEM>slurm</BATCH_SYSTEM>
<SUPPORTED_BY>balwinder.singh at pnnl dot gov</SUPPORTED_BY>
<GMAKE_J>8</GMAKE_J>
<MAX_TASKS_PER_NODE>16</MAX_TASKS_PER_NODE>
</machine>

<machine MACH="cascade">
<DESC>PNL Intel KNC cluster, OS is Linux, batch system is SLURM</DESC>
<OS>LINUX</OS>
<COMPILERS>intel</COMPILERS>
<MPILIBS>mvapich2</MPILIBS>
<NODENAME_REGEX>glogin</NODENAME_REGEX>
<CESMSCRATCHROOT>/dtemp/$USER</CESMSCRATCHROOT>
<RUNDIR>/dtemp/$USER/csmruns/$CASE/run</RUNDIR>
<EXEROOT>/dtemp/$USER/csmruns/$CASE/bld</EXEROOT>
<CIME_OUTPUT_ROOT>/dtemp/$USER</CIME_OUTPUT_ROOT>
<DIN_LOC_ROOT>/dtemp/sing201/acme/inputdata/</DIN_LOC_ROOT>
<DIN_LOC_ROOT_CLMFORC>/dtemp/sing201/acme/inputdata/atm/datm7</DIN_LOC_ROOT_CLMFORC>
<DOUT_S_ROOT>/dtemp/sing201/$USER/archive/$CASE</DOUT_S_ROOT>
<DOUT_L_MSROOT>UNSET</DOUT_L_MSROOT>
<CCSM_BASELINE>/dtemp/sing201/acme/acme_baselines</CCSM_BASELINE>
<CCSM_CPRNC>/dtemp/sing201/acme/acme_baselines/cprnc/cprnc</CCSM_CPRNC>
<SAVE_TIMING_DIR>/dtemp/sing201/acme/timing_acme</SAVE_TIMING_DIR>
<BATCH_SYSTEM>slurm</BATCH_SYSTEM>
<SUPPORTED_BY>balwinder.singh -at- pnnl.gov</SUPPORTED_BY>
<GMAKE_J>8</GMAKE_J>
<MAX_TASKS_PER_NODE>16</MAX_TASKS_PER_NODE>
<PES_PER_NODE>16</PES_PER_NODE>
<PROJECT_REQUIRED>TRUE</PROJECT_REQUIRED>
<mpirun mpilib="mpi-serial">
<executable></executable>
</mpirun>
<mpirun mpilib="mvapich2">
<executable>srun</executable>
<arguments>
<arg name="mpi">--mpi=none</arg>
<arg name="num_tasks">--ntasks=$TOTALPES</arg>
<arg name="cpu_bind">--cpu_bind=sockets --cpu_bind=verbose</arg>
<arg name="kill-on-bad-exit">--kill-on-bad-exit</arg>
</arguments>
</mpirun>
<module_system type="module">
<init_path lang="python">/opt/lmod/5.0.1/init/env_modules_python.py</init_path>
<init_path lang="csh">/etc/profile.d/modules.csh</init_path>
<init_path lang="sh">/etc/profile.d/modules.sh</init_path>
<cmd_path lang="python">/opt/lmod/lmod/libexec/lmod python</cmd_path>
<cmd_path lang="sh">module</cmd_path>
<cmd_path lang="csh">module</cmd_path>
<modules>
<command name="purge"></command>
</modules>
<modules>
<command name="load">python/2.7.9</command>
</modules>
<modules compiler="intel">
<command name="load">intel/15.0.090</command>
<command name="load">mkl/14.0</command>
</modules>
<modules mpilib="mvapich2">
<command name="load">mvapich2/1.9</command>
</modules>
<modules>
<command name="load">netcdf/4.3.0</command>
</modules>
</module_system>
<environment_variables>
<env name="OMP_STACKSIZE">64M</env>
<env name="NETCDF_HOME">$ENV{NETCDF_ROOT}</env>
</environment_variables>
<environment_variables compiler="intel">
<env name="MKL_PATH">$ENV{MLIBHOME}</env>
<env name="COMPILER">intel</env>
</environment_variables>
</machine>

<machine MACH="constance">
<DESC>PNL Haswell cluster, OS is Linux, batch system is SLURM</DESC>
<OS>LINUX</OS>
Expand Down
1 change: 1 addition & 0 deletions cime/cime_config/acme/machines/config_pio.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,7 @@
<value mach="userdefined">netcdf</value>
<value mach="eastwind">netcdf</value>
<value mach="constance">netcdf</value>
<value mach="cascade">netcdf</value>
<value mach="sooty">netcdf</value>
<value mach="pleiades.*">netcdf</value>
<value mach="hobart" compiler="pgi">netcdf</value>
Expand Down