Merge branch 'origin/azamat/tests/add-ne4-wcycl-dev' into master (PR #…

…1951) Add a threading test to acme_developer * Add rank-to-core bindings on Melvin * Add thread bindings on Melvin * Run single-threaded on Melvin until threading is fixed there [BFB] * origin/azamat/tests/add-ne4-wcycl-dev: Add a threading test to acme_developer
ESMCI · Dec 14, 2017 · 28c14a1 · 28c14a1
2 parents e4fdd29 + 4c6cbb3
commit 28c14a1
Show file tree

Hide file tree

Showing 4 changed files with 93 additions and 12 deletions.
diff --git a/config/acme/allactive/config_pesall.xml b/config/acme/allactive/config_pesall.xml
@@ -630,14 +630,14 @@
       <pes compset="any" pesize="any">
         <comment>none</comment>
         <ntasks>
-          <ntasks_atm>64</ntasks_atm>
-          <ntasks_lnd>64</ntasks_lnd>
-          <ntasks_rof>64</ntasks_rof>
-          <ntasks_ice>64</ntasks_ice>
-          <ntasks_ocn>64</ntasks_ocn>
-          <ntasks_glc>64</ntasks_glc>
-          <ntasks_wav>64</ntasks_wav>
-          <ntasks_cpl>64</ntasks_cpl>
+          <ntasks_atm>48</ntasks_atm>
+          <ntasks_lnd>48</ntasks_lnd>
+          <ntasks_rof>48</ntasks_rof>
+          <ntasks_ice>48</ntasks_ice>
+          <ntasks_ocn>48</ntasks_ocn>
+          <ntasks_glc>48</ntasks_glc>
+          <ntasks_wav>48</ntasks_wav>
+          <ntasks_cpl>48</ntasks_cpl>
         </ntasks>
         <nthrds>
           <nthrds_atm>1</nthrds_atm>
@@ -1539,7 +1539,7 @@
     </mach>
   </grid>
   <grid name="a%T62">
-    <mach name="edison">
+    <mach name="edison|melvin">
       <pes compset="any" pesize="any">
         <comment>none</comment>
         <ntasks>
@@ -1873,6 +1873,43 @@
       </pes>
     </mach>
   </grid>
+  <grid name="a%1.9x2.5">
+    <mach name="melvin">
+      <pes compset="any" pesize="any">
+        <comment>none</comment>
+        <ntasks>
+          <ntasks_atm>48</ntasks_atm>
+          <ntasks_lnd>48</ntasks_lnd>
+          <ntasks_rof>48</ntasks_rof>
+          <ntasks_ice>48</ntasks_ice>
+          <ntasks_ocn>48</ntasks_ocn>
+          <ntasks_glc>48</ntasks_glc>
+          <ntasks_wav>48</ntasks_wav>
+          <ntasks_cpl>48</ntasks_cpl>
+        </ntasks>
+        <nthrds>
+          <nthrds_atm>1</nthrds_atm>
+          <nthrds_lnd>1</nthrds_lnd>
+          <nthrds_rof>1</nthrds_rof>
+          <nthrds_ice>1</nthrds_ice>
+          <nthrds_ocn>1</nthrds_ocn>
+          <nthrds_glc>1</nthrds_glc>
+          <nthrds_wav>1</nthrds_wav>
+          <nthrds_cpl>1</nthrds_cpl>
+        </nthrds>
+        <rootpe>
+          <rootpe_atm>0</rootpe_atm>
+          <rootpe_lnd>0</rootpe_lnd>
+          <rootpe_rof>0</rootpe_rof>
+          <rootpe_ice>0</rootpe_ice>
+          <rootpe_ocn>0</rootpe_ocn>
+          <rootpe_glc>0</rootpe_glc>
+          <rootpe_wav>0</rootpe_wav>
+          <rootpe_cpl>0</rootpe_cpl>
+        </rootpe>
+      </pes>
+    </mach>
+  </grid>
   <grid name="a%1.9x2.5">
     <mach name="edison|eos">
       <pes compset="any" pesize="any">
@@ -2097,6 +2134,44 @@
       </pes>
     </mach>
   </grid>
+
+  <grid name="a%0.9x1.25">
+    <mach name="melvin">
+      <pes compset="any" pesize="any">
+        <comment>none</comment>
+        <ntasks>
+          <ntasks_atm>48</ntasks_atm>
+          <ntasks_lnd>48</ntasks_lnd>
+          <ntasks_rof>48</ntasks_rof>
+          <ntasks_ice>48</ntasks_ice>
+          <ntasks_ocn>48</ntasks_ocn>
+          <ntasks_glc>48</ntasks_glc>
+          <ntasks_wav>48</ntasks_wav>
+          <ntasks_cpl>48</ntasks_cpl>
+        </ntasks>
+        <nthrds>
+          <nthrds_atm>1</nthrds_atm>
+          <nthrds_lnd>1</nthrds_lnd>
+          <nthrds_rof>1</nthrds_rof>
+          <nthrds_ice>1</nthrds_ice>
+          <nthrds_ocn>1</nthrds_ocn>
+          <nthrds_glc>1</nthrds_glc>
+          <nthrds_wav>1</nthrds_wav>
+          <nthrds_cpl>1</nthrds_cpl>
+        </nthrds>
+        <rootpe>
+          <rootpe_atm>0</rootpe_atm>
+          <rootpe_lnd>0</rootpe_lnd>
+          <rootpe_rof>0</rootpe_rof>
+          <rootpe_ice>0</rootpe_ice>
+          <rootpe_ocn>0</rootpe_ocn>
+          <rootpe_glc>0</rootpe_glc>
+          <rootpe_wav>0</rootpe_wav>
+          <rootpe_cpl>0</rootpe_cpl>
+        </rootpe>
+      </pes>
+    </mach>
+  </grid>
   <grid name="a%0.9x1.25">
     <mach name="edison|eos">
       <pes compset="any" pesize="any">
@@ -2359,7 +2434,7 @@
     </mach>
   </grid>
   <grid name="a%360x720cru">
-    <mach name="edison">
+    <mach name="edison|melvin">
       <pes compset="any" pesize="any">
         <comment>none</comment>
         <ntasks>

diff --git a/config/acme/machines/config_machines.xml b/config/acme/machines/config_machines.xml
@@ -572,14 +572,15 @@
     <SUPPORTED_BY>jgfouca at sandia dot gov</SUPPORTED_BY>
 <!--    <GMAKE>make</GMAKE> <- this doesn't actually work! -->
     <GMAKE_J>32</GMAKE_J>
-    <MAX_TASKS_PER_NODE>64</MAX_TASKS_PER_NODE>
-    <MAX_MPITASKS_PER_NODE>64</MAX_MPITASKS_PER_NODE>
+    <MAX_TASKS_PER_NODE>48</MAX_TASKS_PER_NODE>
+    <MAX_MPITASKS_PER_NODE>48</MAX_MPITASKS_PER_NODE>
     <BATCH_SYSTEM>none</BATCH_SYSTEM>
     <TEST_TPUT_TOLERANCE>0.1</TEST_TPUT_TOLERANCE>
     <mpirun mpilib="default">
       <executable>mpirun</executable>
       <arguments>
         <arg name="num_tasks"> -np $TOTALPES</arg>
+        <arg name="tasks_per_node"> --map-by ppr:{{ tasks_per_numa }}:socket:PE=$ENV{OMP_NUM_THREADS} --bind-to hwthread</arg>
       </arguments>
     </mpirun>
     <module_system type="module">
@@ -617,6 +618,8 @@
     <environment_variables>
       <env name="NETCDFROOT">$ENV{SEMS_NETCDF_ROOT}</env>
       <env name="OMP_STACKSIZE">64M</env>
+      <env name="OMP_PROC_BIND">spread</env>
+      <env name="OMP_PLACES">threads</env>
     </environment_variables>
     <environment_variables mpilib="!mpi-serial">
       <env name="PNETCDFROOT">$ENV{SEMS_NETCDF_ROOT}</env>

diff --git a/config/acme/testmods_dirs/allactive/mach_mods/shell_commands b/config/acme/testmods_dirs/allactive/mach_mods/shell_commands
@@ -0,0 +1,2 @@
+#!/bin/bash
+if [ `./xmlquery --value MACH` == melvin ]; then ./xmlchange --id NTHRDS --val 1; fi
diff --git a/scripts/lib/update_acme_tests.py b/scripts/lib/update_acme_tests.py
@@ -139,6 +139,7 @@
                          "ERS.f09_g16_g.MPASLISIA",
                          "SMS.T62_oQU120_ais20.MPAS_LISIO_TEST",
                          "SMS.f09_g16_a.IGCLM45_MLI"
+                        ,("SMS_P12x2.ne4_oQU240.A_WCYCL1850","mach_mods")
                         )),
 
     "acme_integration" : (("acme_developer", "acme_atm_integration"),"03:00:00",
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1,2 @@
		#!/bin/bash
		if [ `./xmlquery --value MACH` == melvin ]; then ./xmlchange --id NTHRDS --val 1; fi