From 0263e645e6d22db4921c4835a442deb754c660f0 Mon Sep 17 00:00:00 2001 From: Azamat Mametjanov Date: Tue, 3 Oct 2017 18:37:18 +0000 Subject: [PATCH 1/5] Add a default 64x1 PE layout --- cime/config/acme/allactive/config_pesall.xml | 35 ++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/cime/config/acme/allactive/config_pesall.xml b/cime/config/acme/allactive/config_pesall.xml index 238eecd057d7..e45f7fbe6426 100644 --- a/cime/config/acme/allactive/config_pesall.xml +++ b/cime/config/acme/allactive/config_pesall.xml @@ -36,6 +36,41 @@ + + + none + + 64 + 64 + 64 + 64 + 64 + 64 + 64 + 64 + + + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + + + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + + + From 03df39f14cc6a83c07257919c94074fb6bf92f04 Mon Sep 17 00:00:00 2001 From: Azamat Mametjanov Date: Tue, 3 Oct 2017 18:44:36 +0000 Subject: [PATCH 2/5] Update Depends.intel file --- cime/config/acme/machines/Depends.corip1 | 5 ---- cime/config/acme/machines/Depends.intel | 2 +- cime/config/acme/machines/Depends.intel14 | 28 -------------------- cime/config/acme/machines/Depends.intelmic | 6 ----- cime/config/acme/machines/Depends.intelmic14 | 6 ----- 5 files changed, 1 insertion(+), 46 deletions(-) delete mode 100644 cime/config/acme/machines/Depends.corip1 delete mode 100644 cime/config/acme/machines/Depends.intel14 delete mode 100644 cime/config/acme/machines/Depends.intelmic delete mode 100644 cime/config/acme/machines/Depends.intelmic14 diff --git a/cime/config/acme/machines/Depends.corip1 b/cime/config/acme/machines/Depends.corip1 deleted file mode 100644 index 81be8f3f16e2..000000000000 --- a/cime/config/acme/machines/Depends.corip1 +++ /dev/null @@ -1,5 +0,0 @@ -# Workaround for ICE in intel/2016.0.109 -ifeq (CPRINTEL,$(findstring CPRINTEL, $(FFLAGS))) -RtmMod.o: RtmMod.F90 - $(FC) -c $(INCLDIR) $(INCS) $(FFLAGS) $(FREEFLAGS) $(CONTIGUOUS_FLAG) -O1 $< -endif diff --git a/cime/config/acme/machines/Depends.intel b/cime/config/acme/machines/Depends.intel index b706186930d8..9851aeb00c0b 100644 --- a/cime/config/acme/machines/Depends.intel +++ b/cime/config/acme/machines/Depends.intel @@ -36,7 +36,7 @@ ifeq ($(DEBUG),FALSE) $(REDUCED_PRECISION_OBJS): %.o: %.F90 $(FC) -c $(INCLDIR) $(INCS) $(FFLAGS) $(FREEFLAGS) -fimf-precision=low -fp-model fast $< $(SHR_RANDNUM_FORT_OBJS): %.o: %.F90 - $(FC) -c $(INCLDIR) $(INCS) $(FFLAGS) $(FREEFLAGS) -O3 -fp-model fast -no-prec-div -no-prec-sqrt -override-limits $< + $(FC) -c $(INCLDIR) $(INCS) $(FFLAGS) $(FREEFLAGS) -O3 -fp-model fast -no-prec-div -no-prec-sqrt -qoverride-limits $< $(SHR_RANDNUM_C_OBJS): %.o: %.c $(CC) -c $(INCLDIR) $(INCS) $(CFLAGS) -O3 -fp-model fast $< endif diff --git a/cime/config/acme/machines/Depends.intel14 b/cime/config/acme/machines/Depends.intel14 deleted file mode 100644 index 32e4747d7a3d..000000000000 --- a/cime/config/acme/machines/Depends.intel14 +++ /dev/null @@ -1,28 +0,0 @@ -# -# 12/03/2012 the intel compiler on yellowstone 12.1.5 20120612 -# does not converge the pH computation without the -CU flag -# root cause has not been determined. JPE -# this problem is resolved in intel 13.0.1 -#ecosys_mod.o: ecosys_mod.F90 -# $(FC) -c $(INCLDIR) $(INCS) $(FFLAGS) $(FREEFLAGS) -CU $< - - - -PERFOBJS=\ -prim_advection_mod_base.o \ -vertremap_mod_base.o \ -edge_mod_base.o \ -derivative_mod_base.o \ -bndry_mod_base.o \ -prim_advance_mod.o \ -uwshcu.o \ -wetdep.o - -ifeq ($(DEBUG),FALSE) - $(PERFOBJS): %.o: %.F90 - $(FC) -c $(INCLDIR) $(INCS) $(FFLAGS) $(FREEFLAGS) -O3 -no-prec-div $< - $(REDUCED_OPT_OBJS): %.o: %.F90 - $(FC) -c $(INCLDIR) $(INCS) $(FFLAGS) $(FREEFLAGS) -O1 $< - $(REDUCED_PRECISION_OBJS): %.o: %.F90 - $(FC) -c $(INCLDIR) $(INCS) $(FFLAGS) $(FREEFLAGS) -fimf-precision=low -fp-model fast $< -endif diff --git a/cime/config/acme/machines/Depends.intelmic b/cime/config/acme/machines/Depends.intelmic deleted file mode 100644 index 0227c1bb839d..000000000000 --- a/cime/config/acme/machines/Depends.intelmic +++ /dev/null @@ -1,6 +0,0 @@ - -#derivative_mod_base.o: derivative_mod_base.F90 -# $(FC) -c $(FPPFLAGS) $(INCLDIR) $(INCS) $(FREEFLAGS) $(FFLAGS_NOOPT) -O1 $< - -shr_ncread_mod.o: shr_ncread_mod.F90 - $(FC) -c $(FPPFLAGS) $(INCLDIR) $(INCS) $(FREEFLAGS) $(FFLAGS_NOOPT) $< diff --git a/cime/config/acme/machines/Depends.intelmic14 b/cime/config/acme/machines/Depends.intelmic14 deleted file mode 100644 index 0227c1bb839d..000000000000 --- a/cime/config/acme/machines/Depends.intelmic14 +++ /dev/null @@ -1,6 +0,0 @@ - -#derivative_mod_base.o: derivative_mod_base.F90 -# $(FC) -c $(FPPFLAGS) $(INCLDIR) $(INCS) $(FREEFLAGS) $(FFLAGS_NOOPT) -O1 $< - -shr_ncread_mod.o: shr_ncread_mod.F90 - $(FC) -c $(FPPFLAGS) $(INCLDIR) $(INCS) $(FREEFLAGS) $(FFLAGS_NOOPT) $< From 57397a9768b50c0ce05d5ee8264d837622702902 Mon Sep 17 00:00:00 2001 From: Azamat Mametjanov Date: Tue, 3 Oct 2017 18:49:34 +0000 Subject: [PATCH 3/5] Update Theta modules --- cime/config/acme/machines/config_batch.xml | 2 +- .../config/acme/machines/config_compilers.xml | 5 +++ cime/config/acme/machines/config_machines.xml | 32 +++++++++---------- 3 files changed, 21 insertions(+), 18 deletions(-) diff --git a/cime/config/acme/machines/config_batch.xml b/cime/config/acme/machines/config_batch.xml index 666290d9319d..85b8b032f081 100644 --- a/cime/config/acme/machines/config_batch.xml +++ b/cime/config/acme/machines/config_batch.xml @@ -261,7 +261,7 @@ - default + default debug-cache-quad diff --git a/cime/config/acme/machines/config_compilers.xml b/cime/config/acme/machines/config_compilers.xml index 0a4d7703deb4..72b498d3f94b 100644 --- a/cime/config/acme/machines/config_compilers.xml +++ b/cime/config/acme/machines/config_compilers.xml @@ -1120,6 +1120,11 @@ for mct, etc. -DARCH_MIC_KNL + + $(shell nf-config --flibs) + --host=Linux + + $(PNETCDFROOT) $(NETCDFROOT) diff --git a/cime/config/acme/machines/config_machines.xml b/cime/config/acme/machines/config_machines.xml index 74680b8a2c5c..1c7d23079aa0 100644 --- a/cime/config/acme/machines/config_machines.xml +++ b/cime/config/acme/machines/config_machines.xml @@ -1504,7 +1504,7 @@ cobalt_theta acme 8 - 64 + 128 64 OceanClimate TRUE @@ -1515,11 +1515,16 @@ -n $TOTALPES -N $PES_PER_NODE - --cc depth -d $OMP_NUM_THREADS - -e OMP_STACKSIZE=128M - -e OMP_NUM_THREADS=$OMP_NUM_THREADS + $ENV{SMP_VARS} + + + + + + --cc depth -d $ENV{OMP_NUM_THREADS} -j $ENV{OMP_NUM_THREADS} -e OMP_NUM_THREADS=$ENV{OMP_NUM_THREADS} -e OMP_STACKSIZE=128M -e OMP_PROC_BIND=spread -e OMP_PLACES=threads + /opt/modules/default/init/perl.pm /opt/modules/default/init/python.py @@ -1546,26 +1551,26 @@ craype - craype/2.5.11 + craype/2.5.12 + intel/18.0.0.128 PrgEnv-intel/6.0.4 - intel/17.0.4.196 + cce/8.6.2 PrgEnv-cray/6.0.4 - cce/8.6.0 - PrgEnv-gnu/6.0.4 gcc/6.3.0 + PrgEnv-gnu/6.0.4 - cray-libsci/17.06.1 + cray-libsci/17.09.1 craype-mic-knl - cray-mpich/7.6.0 + cray-mpich/7.6.2 cray-netcdf-hdf5parallel/4.4.1.1.3 @@ -1577,13 +1582,6 @@ cray-netcdf/4.4.1.1.3 - - 1 - 1 - 128M - spread - threads - From 23d2c8d2e3ec2761c02df6d5afbef019da82c152 Mon Sep 17 00:00:00 2001 From: Azamat Mametjanov Date: Tue, 3 Oct 2017 18:51:55 +0000 Subject: [PATCH 4/5] Add option to use pre-built MPAS tools --- components/mpas-cice/cime_config/buildlib | 3 +++ components/mpas-o/cime_config/buildlib | 3 +++ components/mpasli/cime_config/buildlib | 3 +++ 3 files changed, 9 insertions(+) diff --git a/components/mpas-cice/cime_config/buildlib b/components/mpas-cice/cime_config/buildlib index 98715f3fd8b2..c3100a501bfc 100755 --- a/components/mpas-cice/cime_config/buildlib +++ b/components/mpas-cice/cime_config/buildlib @@ -49,6 +49,9 @@ if ( $MACH eq "edison" ) { } elsif ( $MACH eq "titan" ) { $bldcmd .= ' TOOL_TARGET_ARCH="-target-cpu=istanbul"'; } +if (defined $ENV{TOOL_DIR}) { + $bldcmd .= " TOOL_DIR=$ENV{TOOL_DIR}"; +} system($bldcmd) == 0 or die "ERROR: $component.buildlib $bldcmd failed: $?\n"; diff --git a/components/mpas-o/cime_config/buildlib b/components/mpas-o/cime_config/buildlib index 249de4b45425..9f85c35c2127 100755 --- a/components/mpas-o/cime_config/buildlib +++ b/components/mpas-o/cime_config/buildlib @@ -49,6 +49,9 @@ if ( $MACH eq "edison" ) { } elsif ( $MACH eq "titan" ) { $bldcmd .= ' TOOL_TARGET_ARCH="-target-cpu=istanbul"'; } +if (defined $ENV{TOOL_DIR}) { + $bldcmd .= " TOOL_DIR=$ENV{TOOL_DIR}"; +} system($bldcmd) == 0 or die "ERROR: $component.buildlib $bldcmd failed: $?\n"; diff --git a/components/mpasli/cime_config/buildlib b/components/mpasli/cime_config/buildlib index e8a3c3c67952..f0b30bb3d79f 100755 --- a/components/mpasli/cime_config/buildlib +++ b/components/mpasli/cime_config/buildlib @@ -50,6 +50,9 @@ if ( $MACH eq "edison" ) { } elsif ( $MACH eq "titan" ) { $bldcmd .= ' TOOL_TARGET_ARCH="-target-cpu=istanbul"'; } +if (defined $ENV{TOOL_DIR}) { + $bldcmd .= " TOOL_DIR=$ENV{TOOL_DIR}"; +} # Check for Albany build if ( $MPASLI_USE_ALBANY eq "TRUE" ) { From 4807c2655a650166dde20571b75cb1fe073f4feb Mon Sep 17 00:00:00 2001 From: Azamat Mametjanov Date: Tue, 3 Oct 2017 18:54:44 +0000 Subject: [PATCH 5/5] Reload env for tests with multiple case-dirs --- cime/scripts/lib/CIME/SystemTests/system_tests_common.py | 1 + cime/scripts/lib/CIME/case.py | 7 ++++--- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/cime/scripts/lib/CIME/SystemTests/system_tests_common.py b/cime/scripts/lib/CIME/SystemTests/system_tests_common.py index 0f7433cfec5b..577f3d38a69e 100644 --- a/cime/scripts/lib/CIME/SystemTests/system_tests_common.py +++ b/cime/scripts/lib/CIME/SystemTests/system_tests_common.py @@ -186,6 +186,7 @@ def _set_active_case(self, case): Use for tests that have multiple cases """ self._case = case + self._case.load_env(reset=True) self._caseroot = case.get_value("CASEROOT") def run_indv(self, suffix="base", st_archive=False): diff --git a/cime/scripts/lib/CIME/case.py b/cime/scripts/lib/CIME/case.py index 12ed8091f43f..c9e5063d4278 100644 --- a/cime/scripts/lib/CIME/case.py +++ b/cime/scripts/lib/CIME/case.py @@ -1138,7 +1138,7 @@ def get_mpirun_cmd(self, job="case.run"): executable, mpi_arg_list = env_mach_specific.get_mpirun(self, mpi_attribs, job=job) # special case for aprun - if executable is not None and "aprun" in executable: + if executable is not None and "aprun" in executable and "titan" in self.get_value("MACH"): aprun_args, num_nodes = get_aprun_cmd_for_case(self, run_exe) expect( (num_nodes + self.spare_nodes) == self.num_nodes, "Not using optimized num nodes") return executable + aprun_args + " " + run_misc_suffix @@ -1171,8 +1171,9 @@ def set_model_version(self, model): else: logger.warn("WARNING: No %s Model version found."%(model)) - def load_env(self): - if not self._is_env_loaded: + def load_env(self, reset=False): + if not self._is_env_loaded or reset: + os.environ["OMP_NUM_THREADS"] = str(self.thread_count) env_module = self.get_env("mach_specific") env_module.load_env(self) self._is_env_loaded = True