Skip to content

Commit

Permalink
Merge branch 'sarats/machinefiles/summit-june5' (PR #2973)
Browse files Browse the repository at this point in the history 
Update Spectrum MPI module for PGI, XL and GNU.
Update XL version which has compatible parallel-netcdf install.
Update parallel-netcdf module.

More updates:

    Add basic CXX settings to enable PIO2 builds. Fixes #2988
    Backup config for home dir based builds in comments.
    ** Add options for HOME directory based builds in case performance issues are encountered with network file system. This is not enabled by default as space may not be sufficient
    Upgrade PGI to 19.4
    ** Fixes #2980 slow compilation of CAM with PGI on Summit.
  • Loading branch information
@minxu74
minxu74 authored Jun 12, 2019
2 parents c5179c3 + 22b6a44 commit fb783d1
Show file tree
Hide file tree
Showing 2 changed files with 40 additions and 13 deletions.
28 changes: 27 additions & 1 deletion config/e3sm/machines/config_compilers.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1804,7 +1804,16 @@ ntel/x86_64/2013/composer_xe_2013/composer_xe_2013_sp1.3.174/mkl/include </base>
<append>-L$ENV{NETCDF_C_PATH}/lib -lnetcdf -L$ENV{NETCDF_FORTRAN_PATH}/lib -lnetcdff -L$ENV{ESSL_PATH}/lib64 -lessl -L$ENV{NETLIB_LAPACK_PATH}/lib -llapack</append>
<append MPILIB="!mpi-serial"> -L$ENV{PNETCDF_PATH}/lib -lpnetcdf -L$ENV{HDF5_PATH}/lib -lhdf5_hl -lhdf5 </append>
</LDFLAGS>
<MPICC> mpicc </MPICC>
<MPICXX> mpiCC </MPICXX>
<MPIFC> mpif90 </MPIFC>
<SCC> gcc </SCC>
<SCXX> g++ </SCXX>
<SFC> gfortran </SFC>
<PIO_FILESYSTEM_HINTS>gpfs</PIO_FILESYSTEM_HINTS>
<NETCDF_C_PATH>$ENV{NETCDF_C_PATH}</NETCDF_C_PATH>
<NETCDF_FORTRAN_PATH>$ENV{NETCDF_FORTRAN_PATH}</NETCDF_FORTRAN_PATH>
<SUPPORTS_CXX>TRUE</SUPPORTS_CXX>
</compiler>

<compiler MACH="summit" COMPILER="ibm">
Expand All @@ -1824,11 +1833,15 @@ ntel/x86_64/2013/composer_xe_2013/composer_xe_2013_sp1.3.174/mkl/include </base>
<append> -Wl,--relax -Wl,--allow-multiple-definition </append>
</LDFLAGS>
<MPICC> mpicc </MPICC>
<MPICXX> mpicxx </MPICXX>
<MPICXX> mpiCC </MPICXX>
<MPIFC> mpif90 </MPIFC>
<PIO_FILESYSTEM_HINTS>gpfs</PIO_FILESYSTEM_HINTS>
<SCC> xlc_r </SCC>
<SFC> xlf90_r </SFC>
<SCXX> xlc++_r </SCXX>
<NETCDF_C_PATH>$ENV{NETCDF_C_PATH}</NETCDF_C_PATH>
<NETCDF_FORTRAN_PATH>$ENV{NETCDF_FORTRAN_PATH}</NETCDF_FORTRAN_PATH>
<SUPPORTS_CXX>TRUE</SUPPORTS_CXX>
</compiler>

<compiler MACH="summit" COMPILER="pgi">
Expand All @@ -1845,12 +1858,22 @@ ntel/x86_64/2013/composer_xe_2013/composer_xe_2013_sp1.3.174/mkl/include </base>
<append>-L$ENV{NETCDF_C_PATH}/lib -lnetcdf -L$ENV{NETCDF_FORTRAN_PATH}/lib -lnetcdff -L$ENV{ESSL_PATH}/lib64 -lessl -L$ENV{NETLIB_LAPACK_PATH}/lib -llapack</append>
<append MPILIB="!mpi-serial"> -L$ENV{PNETCDF_PATH}/lib -lpnetcdf -L$ENV{HDF5_PATH}/lib -lhdf5_hl -lhdf5 </append>
</LDFLAGS>
<!--
Summit (backup in case defaults are not picking up):
<CXX_LIBS>
<base>/sw/summit/gcc/6.4.0/lib/gcc/powerpc64le-none-linux-gnu/6.4.0/crtbegin.o -L/sw/summit/gcc/6.4.0/lib64 -lstdc++ -latomic</base>
</CXX_LIBS>
-->
<MPICC> mpicc </MPICC>
<MPICXX> mpiCC </MPICXX>
<MPIFC> mpif90 </MPIFC>
<PIO_FILESYSTEM_HINTS>gpfs</PIO_FILESYSTEM_HINTS>
<SCC> pgcc </SCC>
<SCXX> pgc++ </SCXX>
<SFC> pgfortran </SFC>
<NETCDF_C_PATH>$ENV{NETCDF_C_PATH}</NETCDF_C_PATH>
<NETCDF_FORTRAN_PATH>$ENV{NETCDF_FORTRAN_PATH}</NETCDF_FORTRAN_PATH>
<SUPPORTS_CXX>TRUE</SUPPORTS_CXX>
</compiler>

<compiler MACH="summit" COMPILER="pgiacc">
Expand All @@ -1874,6 +1897,9 @@ ntel/x86_64/2013/composer_xe_2013/composer_xe_2013_sp1.3.174/mkl/include </base>
<PIO_FILESYSTEM_HINTS>gpfs</PIO_FILESYSTEM_HINTS>
<SCC> pgcc </SCC>
<SFC> pgfortran </SFC>
<NETCDF_C_PATH>$ENV{NETCDF_C_PATH}</NETCDF_C_PATH>
<NETCDF_FORTRAN_PATH>$ENV{NETCDF_FORTRAN_PATH}</NETCDF_FORTRAN_PATH>
<SUPPORTS_CXX>TRUE</SUPPORTS_CXX>
</compiler>

<compiler MACH="summitdev" COMPILER="ibm">
Expand Down
25 changes: 13 additions & 12 deletions config/e3sm/machines/config_machines.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3073,8 +3073,11 @@
<PROJECT>cli115</PROJECT>
<CHARGE_ACCOUNT>cli115</CHARGE_ACCOUNT>
<SAVE_TIMING_DIR>/gpfs/alpine/proj-shared/$PROJECT</SAVE_TIMING_DIR>
<SAVE_TIMING_DIR_PROJECTS>cli115,cli127,csc190</SAVE_TIMING_DIR_PROJECTS>
<SAVE_TIMING_DIR_PROJECTS>cli115,cli127</SAVE_TIMING_DIR_PROJECTS>
<CIME_OUTPUT_ROOT>/gpfs/alpine/$PROJECT/proj-shared/$ENV{USER}/e3sm_scratch</CIME_OUTPUT_ROOT>
<!-- In case you wish to try HOME for building to check for filesystem issues, uncomment following -->
<!-- You may have to change RUNDIR below to use scratch file sustem. -->
<!-- <CIME_OUTPUT_ROOT>$ENV{HOME}/e3sm_scratch/$PROJECT</CIME_OUTPUT_ROOT> -->
<DIN_LOC_ROOT>/gpfs/alpine/cli115/world-shared/e3sm/inputdata</DIN_LOC_ROOT>
<DIN_LOC_ROOT_CLMFORC>/gpfs/alpine/cli115/world-shared/e3sm/inputdata/atm/datm7</DIN_LOC_ROOT_CLMFORC>
<DOUT_S_ROOT>/gpfs/alpine/$PROJECT/proj-shared/$ENV{USER}/archive/$CASE</DOUT_S_ROOT>
Expand All @@ -3090,7 +3093,7 @@
<PROJECT_REQUIRED>TRUE</PROJECT_REQUIRED>
<mpirun mpilib="spectrum-mpi">
<!-- Use a helper script to tweak jsrun options -->
<executable>/gpfs/alpine/world-shared/csc190/e3sm/mpirun.summit</executable>
<executable>/gpfs/alpine/world-shared/cli115/mpirun.summit</executable>
<!-- <executable>jsrun</executable> -->
<arguments>
<arg name="num_tasks"> -n {{ total_tasks }} -N $MAX_MPITASKS_PER_NODE</arg>
Expand Down Expand Up @@ -3122,10 +3125,10 @@
</modules>
<!-- List of modules elements, executing commands if compiler and mpilib condition applies -->
<modules compiler="pgi.*">
<command name="load">pgi/18.10</command>
<command name="load">pgi/19.4</command>
</modules>
<modules compiler="ibm">
<command name="load">xl/16.1.1-1</command>
<command name="load">xl/16.1.1-3</command>
</modules>
<modules compiler="gnu">
<command name="load">gcc/6.4.0</command>
Expand All @@ -3138,25 +3141,25 @@
<!-- mpi lib settings -->
<!-- Sometimes,same versions of libraries are not available for different compilers, hence the split below -->
<modules compiler="ibm" mpilib="!mpi-serial">
<command name="load">spectrum-mpi/10.2.0.11-20190201</command>
<command name="load">spectrum-mpi/10.3.0.0-20190419</command>
</modules>
<modules compiler="pgi.*" mpilib="!mpi-serial">
<command name="load">spectrum-mpi/10.2.0.11-20190201</command>
<command name="load">spectrum-mpi/10.3.0.0-20190419</command>
</modules>
<modules compiler="gnu" mpilib="!mpi-serial">
<command name="load">spectrum-mpi/10.2.0.11-20190201</command>
<command name="load">spectrum-mpi/10.3.0.0-20190419</command>
</modules>

<modules>
<command name="load">parallel-netcdf/1.8.0</command>
<command name="load">parallel-netcdf/1.8.1</command>
<command name="load">hdf5/1.10.3</command>
</modules>

</module_system>
<!-- Ref: https://www.olcf.ornl.gov/for-users/system-user-guides/summit/ -->
<!-- <CIME_OUTPUT_ROOT>$ENV{HOME}/e3sm_scratch/$PROJECT</CIME_OUTPUT_ROOT> -->
<!-- <RUNDIR>/gpfs/alpine/$PROJECT/proj-shared/$ENV{USER}/e3sm_scratch/$CASE/run</RUNDIR> -->
<RUNDIR>$CIME_OUTPUT_ROOT/$CASE/run</RUNDIR>
<EXEROOT>$CIME_OUTPUT_ROOT/$CASE/bld</EXEROOT>
<!-- Ref: https://www.olcf.ornl.gov/for-users/system-user-guides/summit/ -->
<!-- 1 core/socket not available for application, so 168 = 42cores*4 in smt4 mode -->

<!-- Useful jsrun options:
Expand Down Expand Up @@ -3184,8 +3187,6 @@
</environment_variables>
<environment_variables mpilib="!mpi-serial">
<env name="HDF5_PATH">$ENV{OLCF_HDF5_ROOT}</env>
<!-- OMPI_MCA_io env is required due to OOM errors -->
<env name="OMPI_MCA_io">romio314</env>
<env name="PNETCDF_PATH">$ENV{OLCF_PARALLEL_NETCDF_ROOT}</env>
</environment_variables>
</machine>
Expand Down

0 comments on commit fb783d1

Please sign in to comment.