From 6dd7257ede6f267e47f4ce4c0fa692f6fec08a16 Mon Sep 17 00:00:00 2001 From: Azamat Mametjanov Date: Fri, 14 Jul 2017 19:14:00 -0500 Subject: [PATCH 1/6] Bebop machine config entry --- config/acme/machines/config_machines.xml | 59 ++++++++++++++++++++++++ 1 file changed, 59 insertions(+) diff --git a/config/acme/machines/config_machines.xml b/config/acme/machines/config_machines.xml index 59acde27baa..8d49cc9727b 100644 --- a/config/acme/machines/config_machines.xml +++ b/config/acme/machines/config_machines.xml @@ -1027,6 +1027,65 @@ + + ANL/LCRC Cluster, Cray CS400, 352-nodes Xeon Phi 7230 KNLs 64C/1.3GHz + 672-nodes Xeon E5-2695v4 Broadwells 36C/2.10GHz, Intel Omni-Path network, SLURM batch system, Lmod module environment. + bebop + acme_developer + intel,gnu + mpt + $ENV{HOME}/acme_scratch/bebop + $CIME_OUTPUT_ROOT/$CASE/run + $CIME_OUTPUT_ROOT/$CASE/bld + /home/ccsm-data/inputdata + /home/ccsm-data/inputdata/atm/datm7 + $CIME_OUTPUT_ROOT/archive/$CASE + csm/$CASE + /lcrc/group/earthscience/acme_baselines + /home/ccsm-data/tools/cprnc + CNL + slurm + acme + 8 + 36 + 36 + acme + TRUE + -D PIO_BUILD_TIMING:BOOL=ON + + srun + + --label + -n $TOTALPES + + + + module + module + + intel + gcc + cmake + + + intel/17.0.4-74uvhji + intel-mkl/2017.3.196-jyjmyut + + + gcc/7.1.0-4bgguyp + + + cmake/2.8.12.2-qndad62 + mpich/3.2-5koqqym + netcdf/4.4.1.1-prsuusl + + + + 128M + spread + threads + + + ANL IBM BG/Q, os is BGQ, 16 cores/node, batch system is cobalt cetus From 5e6520935dceb89cb52ee62a2a366fbd43009c1a Mon Sep 17 00:00:00 2001 From: Azamat Mametjanov Date: Thu, 20 Jul 2017 15:36:59 -0500 Subject: [PATCH 2/6] Bebop building and batching updates --- config/acme/machines/config_batch.xml | 8 +++++ config/acme/machines/config_compilers.xml | 14 +++++++++ config/acme/machines/config_machines.xml | 37 ++++++++++++++--------- 3 files changed, 45 insertions(+), 14 deletions(-) diff --git a/config/acme/machines/config_batch.xml b/config/acme/machines/config_batch.xml index 4b74bf9cc6b..43058ba91f2 100644 --- a/config/acme/machines/config_batch.xml +++ b/config/acme/machines/config_batch.xml @@ -199,6 +199,14 @@ + + + debug + bdw + knl + + + diff --git a/config/acme/machines/config_compilers.xml b/config/acme/machines/config_compilers.xml index 66f75ef318b..c5042df6e50 100644 --- a/config/acme/machines/config_compilers.xml +++ b/config/acme/machines/config_compilers.xml @@ -1178,6 +1178,20 @@ for mct, etc. gpfs + + -O2 -qno-opt-dynamic-align + -O2 + -qopenmp + -qopenmp + -qopenmp + -qopenmp + $(shell $(NETCDF_PATH)/bin/nf-config --flibs) -mkl + -DHAVE_SLASHPROC + mpiifort + mpiicc + mpiicpc + + -O2 -O2 diff --git a/config/acme/machines/config_machines.xml b/config/acme/machines/config_machines.xml index 8d49cc9727b..75a04e36c4a 100644 --- a/config/acme/machines/config_machines.xml +++ b/config/acme/machines/config_machines.xml @@ -1032,17 +1032,17 @@ bebop acme_developer intel,gnu - mpt - $ENV{HOME}/acme_scratch/bebop + impi,mpich,mvapich,openmpi,mpi-serial + /lcrc/group/acme/$USER/acme_scratch/bebop $CIME_OUTPUT_ROOT/$CASE/run $CIME_OUTPUT_ROOT/$CASE/bld /home/ccsm-data/inputdata /home/ccsm-data/inputdata/atm/datm7 $CIME_OUTPUT_ROOT/archive/$CASE csm/$CASE - /lcrc/group/earthscience/acme_baselines + /lcrc/group/acme/acme_baselines /home/ccsm-data/tools/cprnc - CNL + LINUX slurm acme 8 @@ -1052,19 +1052,17 @@ TRUE -D PIO_BUILD_TIMING:BOOL=ON - srun + mpirun - --label -n $TOTALPES - module - module + module -q + module -q + module -q - intel - gcc - cmake + intel/17.0.4-74uvhji @@ -1073,17 +1071,28 @@ gcc/7.1.0-4bgguyp + + intel-mpi/2017.3-dfphq6k + - cmake/2.8.12.2-qndad62 - mpich/3.2-5koqqym - netcdf/4.4.1.1-prsuusl + cmake + netcdf/4.4.1.1-prsuusl + netcdf-fortran/4.4.4-ojwazvy + parallel-netcdf/1.8.1 + + $SHELL{which nf-config | xargs dirname | xargs dirname} + $SHELL{which pnetcdf_version | xargs dirname | xargs dirname} + 128M spread threads + + shm:tmi + From 22ca7edb177122d0ea561400a1aff7c7b34bf63f Mon Sep 17 00:00:00 2001 From: Azamat Mametjanov Date: Mon, 24 Jul 2017 11:56:30 -0500 Subject: [PATCH 3/6] Distinguish bebop node names in acme_dev tests --- config/acme/machines/config_machines.xml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/config/acme/machines/config_machines.xml b/config/acme/machines/config_machines.xml index 75a04e36c4a..795b58535c4 100644 --- a/config/acme/machines/config_machines.xml +++ b/config/acme/machines/config_machines.xml @@ -849,7 +849,7 @@ ANL/LCRC Linux Cluster - b.*.lcrc.anl.gov + blogin.*.lcrc.anl.gov acme_integration gnu,pgi,intel,nag mvapich,mpich,openmpi,mpi-serial @@ -924,7 +924,7 @@ ANL/LCRC Linux Cluster - b.*.lcrc.anl.gov + blogin.*.lcrc.anl.gov acme_integration intel,gnu,pgi mvapich,openmpi,mpi-serial @@ -1029,7 +1029,7 @@ ANL/LCRC Cluster, Cray CS400, 352-nodes Xeon Phi 7230 KNLs 64C/1.3GHz + 672-nodes Xeon E5-2695v4 Broadwells 36C/2.10GHz, Intel Omni-Path network, SLURM batch system, Lmod module environment. - bebop + beboplogin.* acme_developer intel,gnu impi,mpich,mvapich,openmpi,mpi-serial From acaed804c038275a52559572ce3dd1dc6f8c3aa3 Mon Sep 17 00:00:00 2001 From: Azamat Mametjanov Date: Mon, 24 Jul 2017 14:39:30 -0500 Subject: [PATCH 4/6] HOMME CMake file and static cprnc --- config/acme/machines/config_machines.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/acme/machines/config_machines.xml b/config/acme/machines/config_machines.xml index 795b58535c4..974abe0e247 100644 --- a/config/acme/machines/config_machines.xml +++ b/config/acme/machines/config_machines.xml @@ -1041,7 +1041,7 @@ $CIME_OUTPUT_ROOT/archive/$CASE csm/$CASE /lcrc/group/acme/acme_baselines - /home/ccsm-data/tools/cprnc + /lcrc/group/acme/tools/cprnc/cprnc LINUX slurm acme From cc2c423bcbea64af995122658a67b17802fe97dc Mon Sep 17 00:00:00 2001 From: Azamat Mametjanov Date: Fri, 28 Jul 2017 15:34:31 -0500 Subject: [PATCH 5/6] Fix module cmd when called from python --- config/acme/machines/config_compilers.xml | 3 +-- config/acme/machines/config_machines.xml | 11 ++++++++--- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/config/acme/machines/config_compilers.xml b/config/acme/machines/config_compilers.xml index c5042df6e50..22e56320bce 100644 --- a/config/acme/machines/config_compilers.xml +++ b/config/acme/machines/config_compilers.xml @@ -1179,8 +1179,7 @@ for mct, etc. - -O2 -qno-opt-dynamic-align - -O2 + -O2 -debug minimal -qno-opt-dynamic-align -qopenmp -qopenmp -qopenmp diff --git a/config/acme/machines/config_machines.xml b/config/acme/machines/config_machines.xml index 974abe0e247..5971217bc59 100644 --- a/config/acme/machines/config_machines.xml +++ b/config/acme/machines/config_machines.xml @@ -1058,9 +1058,14 @@ - module -q - module -q - module -q + /home/software/spack-0.10.1/opt/spack/linux-centos7-x86_64/gcc-4.8.5/lmod-7.4.9-ic63herzfgw5u3na5mdtvp3nwxy6oj2z/lmod/lmod/init/sh + /home/software/spack-0.10.1/opt/spack/linux-centos7-x86_64/gcc-4.8.5/lmod-7.4.9-ic63herzfgw5u3na5mdtvp3nwxy6oj2z/lmod/lmod/init/csh + /home/software/spack-0.10.1/opt/spack/linux-centos7-x86_64/gcc-4.8.5/lmod-7.4.9-ic63herzfgw5u3na5mdtvp3nwxy6oj2z/lmod/lmod/init/perl + /home/software/spack-0.10.1/opt/spack/linux-centos7-x86_64/gcc-4.8.5/lmod-7.4.9-ic63herzfgw5u3na5mdtvp3nwxy6oj2z/lmod/lmod/init/env_modules_python.py + /home/software/spack-0.10.1/opt/spack/linux-centos7-x86_64/gcc-4.8.5/lmod-7.4.9-ic63herzfgw5u3na5mdtvp3nwxy6oj2z/lmod/lmod/libexec/lmod perl + /home/software/spack-0.10.1/opt/spack/linux-centos7-x86_64/gcc-4.8.5/lmod-7.4.9-ic63herzfgw5u3na5mdtvp3nwxy6oj2z/lmod/lmod/libexec/lmod python + module + module From dee3a81ceef550674fa8bf30e8c209fde569da64 Mon Sep 17 00:00:00 2001 From: Azamat Mametjanov Date: Fri, 28 Jul 2017 16:16:50 -0500 Subject: [PATCH 6/6] Perl module is not installed, use a local one --- config/acme/machines/config_machines.xml | 1 + 1 file changed, 1 insertion(+) diff --git a/config/acme/machines/config_machines.xml b/config/acme/machines/config_machines.xml index 5971217bc59..b32a6ea89e1 100644 --- a/config/acme/machines/config_machines.xml +++ b/config/acme/machines/config_machines.xml @@ -1089,6 +1089,7 @@ $SHELL{which nf-config | xargs dirname | xargs dirname} $SHELL{which pnetcdf_version | xargs dirname | xargs dirname} + /lcrc/group/acme/soft/perl/5.26.0/bin:$ENV{PATH} 128M