Skip to content

Commit

Permalink
Merge branch 'lcrc/machines/bebop' (PR #1662)
Browse files Browse the repository at this point in the history 
Adding support for ANL Bebop cluster.

[BFB]

Conflicts:
	cime/config/acme/machines/config_batch.xml
  • Loading branch information
@jayeshkrishna
jayeshkrishna committed Aug 1, 2017
2 parents e302ced + dee3a81 commit 2644a4c
Show file tree
Hide file tree
Showing 3 changed files with 97 additions and 2 deletions.
8 changes: 8 additions & 0 deletions config/acme/machines/config_batch.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -201,6 +201,14 @@
</queues>
</batch_system>

<batch_system MACH="bebop" type="slurm" >
<queues>
<queue walltimemax="00:30:00" jobmin="1" jobmax="2304" strict="true">debug</queue>
<queue walltimemax="01:00:00" jobmin="1" jobmax="21888" default="true">bdw</queue>
<queue walltimemax="01:00:00" jobmin="1" jobmax="18432">knl</queue>
</queues>
</batch_system>

<!-- eos is PBS -->
<batch_system MACH="eos" type="pbs" >
<directives>
Expand Down
13 changes: 13 additions & 0 deletions config/acme/machines/config_compilers.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1112,6 +1112,19 @@ for mct, etc.
<PIO_FILESYSTEM_HINTS>gpfs </PIO_FILESYSTEM_HINTS>
</compiler>

<compiler COMPILER="intel" MACH="bebop">
<ADD_FFLAGS DEBUG="FALSE"> -O2 -debug minimal -qno-opt-dynamic-align</ADD_FFLAGS>
<ADD_CFLAGS compile_threaded="true"> -qopenmp </ADD_CFLAGS>
<ADD_FFLAGS compile_threaded="true"> -qopenmp </ADD_FFLAGS>
<ADD_FFLAGS_NOOPT compile_threaded="true"> -qopenmp </ADD_FFLAGS_NOOPT>
<ADD_LDFLAGS compile_threaded="true"> -qopenmp </ADD_LDFLAGS>
<ADD_SLIBS> $(shell $(NETCDF_PATH)/bin/nf-config --flibs) -mkl</ADD_SLIBS>
<GPTL_CPPDEFS> -DHAVE_SLASHPROC </GPTL_CPPDEFS>
<MPIFC> mpiifort </MPIFC>
<MPICC> mpiicc </MPICC>
<MPICXX> mpiicpc </MPICXX>
</compiler>

<compiler COMPILER="pgi" MACH="eastwind">
<ADD_CFLAGS DEBUG="FALSE"> -O2 </ADD_CFLAGS>
<ADD_FFLAGS DEBUG="FALSE"> -O2 </ADD_FFLAGS>
Expand Down
78 changes: 76 additions & 2 deletions config/acme/machines/config_machines.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -828,7 +828,7 @@

<machine MACH="blues">
<DESC>ANL/LCRC Linux Cluster</DESC>
<NODENAME_REGEX>b.*.lcrc.anl.gov</NODENAME_REGEX>
<NODENAME_REGEX>blogin.*.lcrc.anl.gov</NODENAME_REGEX>
<TESTS>acme_integration</TESTS>
<COMPILERS>gnu,pgi,intel,nag</COMPILERS>
<MPILIBS>mvapich,mpich,openmpi,mpi-serial</MPILIBS>
Expand Down Expand Up @@ -903,7 +903,7 @@

<machine MACH="anvil">
<DESC>ANL/LCRC Linux Cluster</DESC>
<NODENAME_REGEX>b.*.lcrc.anl.gov</NODENAME_REGEX>
<NODENAME_REGEX>blogin.*.lcrc.anl.gov</NODENAME_REGEX>
<TESTS>acme_integration</TESTS>
<COMPILERS>intel,gnu,pgi</COMPILERS>
<MPILIBS>mvapich,openmpi,mpi-serial</MPILIBS>
Expand Down Expand Up @@ -1006,6 +1006,80 @@
</environment_variables>
</machine>

<machine MACH="bebop">
<DESC>ANL/LCRC Cluster, Cray CS400, 352-nodes Xeon Phi 7230 KNLs 64C/1.3GHz + 672-nodes Xeon E5-2695v4 Broadwells 36C/2.10GHz, Intel Omni-Path network, SLURM batch system, Lmod module environment.</DESC>
<NODENAME_REGEX>beboplogin.*</NODENAME_REGEX>
<TESTS>acme_developer</TESTS>
<COMPILERS>intel,gnu</COMPILERS>
<MPILIBS>impi,mpich,mvapich,openmpi,mpi-serial</MPILIBS>
<CIME_OUTPUT_ROOT>/lcrc/group/acme/$USER/acme_scratch/bebop</CIME_OUTPUT_ROOT>
<RUNDIR>$CIME_OUTPUT_ROOT/$CASE/run</RUNDIR>
<EXEROOT>$CIME_OUTPUT_ROOT/$CASE/bld</EXEROOT>
<DIN_LOC_ROOT>/home/ccsm-data/inputdata</DIN_LOC_ROOT>
<DIN_LOC_ROOT_CLMFORC>/home/ccsm-data/inputdata/atm/datm7</DIN_LOC_ROOT_CLMFORC>
<DOUT_S_ROOT>$CIME_OUTPUT_ROOT/archive/$CASE</DOUT_S_ROOT>
<DOUT_L_MSROOT>csm/$CASE</DOUT_L_MSROOT>
<BASELINE_ROOT>/lcrc/group/acme/acme_baselines</BASELINE_ROOT>
<CCSM_CPRNC>/lcrc/group/acme/tools/cprnc/cprnc</CCSM_CPRNC>
<OS>LINUX</OS>
<BATCH_SYSTEM>slurm</BATCH_SYSTEM>
<SUPPORTED_BY>acme</SUPPORTED_BY>
<GMAKE_J>8</GMAKE_J>
<MAX_TASKS_PER_NODE>36</MAX_TASKS_PER_NODE>
<PES_PER_NODE>36</PES_PER_NODE>
<PROJECT>acme</PROJECT>
<PROJECT_REQUIRED>TRUE</PROJECT_REQUIRED>
<PIO_CONFIG_OPTS> -D PIO_BUILD_TIMING:BOOL=ON </PIO_CONFIG_OPTS>
<mpirun mpilib="default">
<executable>mpirun</executable>
<arguments>
<arg name="num_tasks" > -n $TOTALPES</arg>
</arguments>
</mpirun>
<module_system type="module">
<init_path lang="sh">/home/software/spack-0.10.1/opt/spack/linux-centos7-x86_64/gcc-4.8.5/lmod-7.4.9-ic63herzfgw5u3na5mdtvp3nwxy6oj2z/lmod/lmod/init/sh</init_path>
<init_path lang="csh">/home/software/spack-0.10.1/opt/spack/linux-centos7-x86_64/gcc-4.8.5/lmod-7.4.9-ic63herzfgw5u3na5mdtvp3nwxy6oj2z/lmod/lmod/init/csh</init_path>
<init_path lang="perl">/home/software/spack-0.10.1/opt/spack/linux-centos7-x86_64/gcc-4.8.5/lmod-7.4.9-ic63herzfgw5u3na5mdtvp3nwxy6oj2z/lmod/lmod/init/perl</init_path>
<init_path lang="python">/home/software/spack-0.10.1/opt/spack/linux-centos7-x86_64/gcc-4.8.5/lmod-7.4.9-ic63herzfgw5u3na5mdtvp3nwxy6oj2z/lmod/lmod/init/env_modules_python.py</init_path>
<cmd_path lang="perl">/home/software/spack-0.10.1/opt/spack/linux-centos7-x86_64/gcc-4.8.5/lmod-7.4.9-ic63herzfgw5u3na5mdtvp3nwxy6oj2z/lmod/lmod/libexec/lmod perl</cmd_path>
<cmd_path lang="python">/home/software/spack-0.10.1/opt/spack/linux-centos7-x86_64/gcc-4.8.5/lmod-7.4.9-ic63herzfgw5u3na5mdtvp3nwxy6oj2z/lmod/lmod/libexec/lmod python</cmd_path>
<cmd_path lang="sh">module</cmd_path>
<cmd_path lang="csh">module</cmd_path>
<modules>
<command name="purge"></command>
</modules>
<modules compiler="intel">
<command name="load">intel/17.0.4-74uvhji</command>
<command name="load">intel-mkl/2017.3.196-jyjmyut</command>
</modules>
<modules compiler="gnu">
<command name="load">gcc/7.1.0-4bgguyp</command>
</modules>
<modules mpilib="impi">
<command name="load">intel-mpi/2017.3-dfphq6k</command>
</modules>
<modules>
<command name="load">cmake</command>
<command name="load">netcdf/4.4.1.1-prsuusl</command>
<command name="load">netcdf-fortran/4.4.4-ojwazvy</command>
<command name="load">parallel-netcdf/1.8.1</command>
</modules>
</module_system>
<environment_variables>
<env name="NETCDF_PATH">$SHELL{which nf-config | xargs dirname | xargs dirname}</env>
<env name="PNETCDF_PATH" mpilib="!mpi-serial">$SHELL{which pnetcdf_version | xargs dirname | xargs dirname}</env>
<env name="PATH">/lcrc/group/acme/soft/perl/5.26.0/bin:$ENV{PATH}</env>
</environment_variables>
<environment_variables SMP_PRESENT="TRUE">
<env name="OMP_STACKSIZE">128M</env>
<env name="OMP_PROC_BIND">spread</env>
<env name="OMP_PLACES">threads</env>
</environment_variables>
<environment_variables>
<env mpilib="impi" name="I_MPI_FABRICS">shm:tmi</env>
</environment_variables>
</machine>

<machine MACH="cetus">
<DESC>ANL IBM BG/Q, os is BGQ, 16 cores/node, batch system is cobalt</DESC>
<NODENAME_REGEX>cetus</NODENAME_REGEX>
Expand Down

0 comments on commit 2644a4c

Please sign in to comment.