Skip to content

Commit

Permalink
Merge branch ACME-Climate/ndk/machinefiles/intelmpi-option-nersc (PR #…
Browse files Browse the repository at this point in the history 
…1732)

For cori-knl, add option for building with Intel MPI using --mpilib=impi
  • Loading branch information
@ndkeen
ndkeen authored Aug 23, 2017
2 parents 211d88e + e49a38b commit 6be75b8
Show file tree
Hide file tree
Showing 2 changed files with 55 additions and 29 deletions.
21 changes: 20 additions & 1 deletion config/acme/machines/config_compilers.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -632,6 +632,16 @@ for mct, etc.
<PETSC_PATH>$(PETSC_DIR)</PETSC_PATH>
</compiler>

<compiler COMPILER="intel" MACH="cori-knl" MPILIB="impi">
<!-- When using Intel MPI, can't use the Cray compiler wrappers. -->
<MPI_LIB_NAME>impi</MPI_LIB_NAME>
<MPIFC> mpiifort </MPIFC>
<MPICC> mpiicc </MPICC>
<MPICXX> mpiicpc </MPICXX>
<ADD_FFLAGS> -xMIC-AVX512 </ADD_FFLAGS>
<ADD_CFLAGS> -axMIC-AVX512 -xCORE-AVX2 </ADD_CFLAGS>
</compiler>

<!-- These 3 NERSC machines inherit from intel18 compiler settings above (as well as CNL entry) -->
<compiler COMPILER="intel18" MACH="edison">
<CONFIG_ARGS> --host=Linux </CONFIG_ARGS>
Expand Down Expand Up @@ -663,7 +673,16 @@ for mct, etc.
<SFC> ifort </SFC>
<SCC> icc </SCC>
<SCXX> icpc </SCXX>
<PETSC_PATH>$(PETSC_DIR)</PETSC_PATH>
</compiler>

<compiler COMPILER="intel18" MACH="cori-knl" MPILIB="impi">
<!-- When using Intel MPI, can't use the Cray compiler wrappers. -->
<MPI_LIB_NAME>impi</MPI_LIB_NAME>
<MPIFC> mpiifort </MPIFC>
<MPICC> mpiicc </MPICC>
<MPICXX> mpiicpc </MPICXX>
<ADD_FFLAGS> -xMIC-AVX512 </ADD_FFLAGS>
<ADD_CFLAGS> -axMIC-AVX512 -xCORE-AVX2 </ADD_CFLAGS>
</compiler>

<compiler COMPILER="intel" MACH="eos">
Expand Down
63 changes: 35 additions & 28 deletions config/acme/machines/config_machines.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -328,7 +328,7 @@
<NODENAME_REGEX>cori-knl-haswell-is-default</NODENAME_REGEX>
<TESTS>acme_developer</TESTS>
<COMPILERS>intel,intel18,gnu</COMPILERS>
<MPILIBS>mpt,mpi-serial</MPILIBS>
<MPILIBS>mpt,impi,mpi-serial</MPILIBS>
<CIME_OUTPUT_ROOT>$ENV{SCRATCH}/acme_scratch/cori-knl</CIME_OUTPUT_ROOT>
<RUNDIR>$CIME_OUTPUT_ROOT/$CASE/run</RUNDIR>
<EXEROOT>$CIME_OUTPUT_ROOT/$CASE/bld</EXEROOT>
Expand All @@ -343,7 +343,7 @@
<BATCH_SYSTEM>slurm</BATCH_SYSTEM>
<SUPPORTED_BY>acme</SUPPORTED_BY>
<GMAKE_J>8</GMAKE_J>
<MAX_TASKS_PER_NODE>256</MAX_TASKS_PER_NODE>
<MAX_TASKS_PER_NODE>128</MAX_TASKS_PER_NODE>
<PES_PER_NODE>64</PES_PER_NODE>
<PROJECT>acme</PROJECT>
<PROJECT_REQUIRED>TRUE</PROJECT_REQUIRED>
Expand All @@ -353,7 +353,7 @@
<arguments>
<arg name="label"> --label</arg>
<arg name="num_tasks" > -n $TOTALPES</arg>
<!--NOTE: hard-coding -c 4 for now, which is only optimal when using full node with no hyper-threading. (ndk) -->
<!--NOTE: hard-coding -c 4 for now. (ndk) -->
<arg name="thread_count" > -c 4 </arg>
<arg name="binding" > --cpu_bind=cores</arg>
</arguments>
Expand All @@ -368,68 +368,66 @@
<cmd_path lang="sh">module</cmd_path>
<cmd_path lang="csh">module</cmd_path>
<modules>
<command name="rm">craype</command>
<command name="rm">craype-mic-knl</command>
<command name="rm">craype-haswell</command>

<command name="rm">PrgEnv-intel</command>
<command name="rm">PrgEnv-gnu</command>
<command name="rm">intel</command>
<command name="rm">cce</command>
<command name="rm">gcc</command>
<command name="rm">cray-parallel-netcdf</command>
<command name="rm">cray-parallel-hdf5</command>
<command name="rm">pmi</command>
<command name="rm">cray-libsci</command>
<command name="rm">cray-mpich2</command>
<command name="rm">cray-mpich</command>
<command name="rm">cray-netcdf</command>
<command name="rm">cray-hdf5</command>
<command name="rm">cray-netcdf-hdf5parallel</command>
<command name="rm">craype-sandybridge</command>
<command name="rm">craype-ivybridge</command>
<command name="rm">craype</command>
<command name="rm">cray-libsci</command>
<command name="rm">papi</command>
<command name="rm">cmake</command>
<command name="rm">cray-petsc</command>
<command name="rm">esmf</command>

<!-- first load basic defaults, then remove/swap/load as necessary -->
<command name="load">craype</command>
<command name="load">PrgEnv-intel</command>
<command name="load">cray-mpich</command>
<command name="rm">craype-haswell</command>
<command name="load">craype-mic-knl</command>
</modules>

<modules mpilib="mpt">
<command name="swap">cray-mpich cray-mpich/7.6.0</command>
</modules>

<modules mpilib="impi">
<command name="swap">cray-mpich impi/2017.up2</command>
</modules>

<modules compiler="intel">
<command name="load">PrgEnv-intel</command>
<command name="rm">intel</command>
<command name="load">intel/17.0.2.174</command>
<command name="rm">craype</command>
<command name="load">craype/2.5.12</command>
<command name="rm">cray-mpich</command>
<command name="load">cray-mpich/7.6.0</command>
</modules>

<modules compiler="intel18">
<!-- A bug in the modules is not letting us unload/load mpich 7.6 with intel18, so first load intel17 here, then correct below -->
<command name="load">PrgEnv-intel</command>
<command name="rm">intel</command>
<command name="load">intel/17.0.2.174</command>

<command name="rm">craype</command>
<command name="load">craype/2.5.12</command>
<command name="rm">cray-mpich</command>
<command name="load">cray-mpich/7.6.0</command>

<command name="load">PrgEnv-intel</command>
<command name="rm">intel</command>
<command name="load">intel/2018.beta</command>
</modules>

<modules compiler="gnu">
<command name="rm">PrgEnv-intel</command>
<command name="load">PrgEnv-gnu</command>
<command name="swap">PrgEnv-intel PrgEnv-gnu</command>
<command name="rm">gcc</command>
<command name="load">gcc/6.3.0</command>
<command name="load">cray-libsci/17.06.1</command>

<command name="rm">craype</command>
<command name="load">craype/2.5.12</command>
<command name="rm">cray-mpich</command>
<command name="load">cray-mpich/7.6.0</command>
</modules>

<modules>
<command name="swap">craype craype/2.5.12</command>
<command name="rm">pmi</command>
<command name="load">pmi/5.0.12</command>
<command name="rm">craype-haswell</command>
Expand All @@ -454,6 +452,9 @@
<command name="load">cmake/3.3.2</command>
<command name="load">zlib/1.2.8</command>
</modules>

<!--command name="list">&gt;&amp; ml.txt</command-->

</module_system>

<environment_variables>
Expand All @@ -465,6 +466,12 @@
<env name="OMP_PROC_BIND">spread</env>
<env name="OMP_PLACES">threads</env>

<env name="I_MPI_FABRICS" mpilib="impi">ofi</env>
<env name="I_MPI_OFI_PROVIDER" mpilib="impi">gni</env>
<env name="I_MPI_PRINT_VERSION" mpilib="impi">yes</env>
<env name="I_MPI_OFI_LIBRARY" mpilib="impi">/global/common/cori/software/libfabric/1.4.2/gnu/lib/libfabric.so</env>
<env name="I_MPI_PMI_LIBRARY" mpilib="impi">/usr/lib64/slurmpmi/libpmi.so</env>

<env name="MPICH_MEMORY_REPORT" compiler="intel18">1</env>

<env name="FORT_BUFFERED" compiler="intel">yes</env>
Expand Down

0 comments on commit 6be75b8

Please sign in to comment.