Merge remote-tracking branch 'origin/master' into jgfouca/update_to_c…

…ime_5.3.0.34 Upstream merge for consistent conflict resolution. * origin/master: (32 commits) Rename TOOL_DIR to MPAS_TOOL_DIR Reload env for tests with multiple case-dirs Add option to use pre-built MPAS tools Update Theta modules Update Depends.intel file Add a default 64x1 PE layout Updates surface datasets Minor update to the flag for IMCLM tests add a flag for the reading of soil order so it only applies for ECA and RD mode Add a comment with an example of MV2_CPU_MAPPING Update Anvil threading environment - fix parameter dimension in PDynamicsMod.F90 Change BGC mode in RD compsts from CN to BGC Do not use CENTURY initial condition fille for RD and CN compsets when using nitrif_dentrif Reverts changes to CLMBuildNamelist.pm Adds a clm-rd test Fix for GSWP3 version in CIME Fix issues found during review. Add GSWP and coupler bypass compsets Update mpas-cice hash and supporting build scripts ...
E3SM-Project · Oct 11, 2017 · 8bd7114 · 8bd7114
2 parents b9a1edd + 9c31301
commit 8bd7114
Show file tree

Hide file tree

Showing 43 changed files with 962 additions and 155 deletions.
diff --git a/cime/config/acme/allactive/config_compsets.xml b/cime/config/acme/allactive/config_compsets.xml
@@ -109,6 +109,17 @@
   <lname>20TR_CAM5%AV1C-H01A_CLM45%SPBC_MPASCICE%SPUNUP_MPASO%SPUNUP_MOSART_SGLC_SWAV</lname>
 </compset>
 
+<!-- ACME V1 BGC experiment compsets -->
+<compset>
+  <alias>A_BGCEXP_BCRC_RDCN_2000</alias>
+  <lname>2000_CAM5%AV1C-L_CLM45%CNRDCTCBC_MPASCICE%BGC_MPASO%OIECOOIDMS_MOSART_SGLC_SWAV</lname>
+</compset>
+
+<compset>
+  <alias>A_BGCEXP_BCRC_RDCN_1850</alias>
+  <lname>1850_CAM5%AV1C-L_CLM45%CNRDCTCBC_MPASCICE%BGC_MPASO%OIECOOIDMS_MOSART_SGLC_SWAV</lname>
+</compset>
+
 <!-- ACME backup compsets -->
 <compset>
  <alias>A_WCYCL1850_v0atm</alias> 

diff --git a/cime/config/acme/config_grids.xml b/cime/config/acme/config_grids.xml
@@ -43,7 +43,7 @@
       <lname>a%1x1_brazil_l%1x1_brazil_oi%1x1_brazil_r%null_m%reg_g%null_w%null</lname>
     </grid>
 
-    <grid compset="DATM.+CLM">
+    <grid compset="(DATM|SATM).+CLM">
       <sname>1x1_smallvilleIA</sname>
       <lname>a%1x1_smallvilleIA_l%1x1_smallvilleIA_oi%1x1_smallvilleIA_r%null_m%reg_g%null_w%null</lname>
     </grid>
@@ -84,7 +84,7 @@
       <lname>a%360x720cru_l%360x720cru_oi%360x720cru_r%r05_m%360x720cru_g%null_w%null</lname>
     </grid>
 
-    <grid compset="DATM.+CLM">
+    <grid compset="(DATM_SATM).+CLM">
       <sname>NLDAS_NLDAS</sname>
       <lname>a%NLDAS_l%NLDAS_oi%NLDAS_r%NLDAS_m%NLDAS_g%null_w%null</lname>
     </grid>

diff --git a/cime/config/acme/machines/Depends.corip1 b/cime/config/acme/machines/Depends.corip1
diff --git a/cime/config/acme/machines/Depends.intel b/cime/config/acme/machines/Depends.intel
@@ -36,7 +36,7 @@ ifeq ($(DEBUG),FALSE)
   $(REDUCED_PRECISION_OBJS): %.o: %.F90
 	  $(FC) -c $(INCLDIR) $(INCS) $(FFLAGS) $(FREEFLAGS) -fimf-precision=low -fp-model fast $<
   $(SHR_RANDNUM_FORT_OBJS): %.o: %.F90
-	  $(FC) -c $(INCLDIR) $(INCS) $(FFLAGS) $(FREEFLAGS) -O3 -fp-model fast -no-prec-div -no-prec-sqrt -override-limits $<
+	  $(FC) -c $(INCLDIR) $(INCS) $(FFLAGS) $(FREEFLAGS) -O3 -fp-model fast -no-prec-div -no-prec-sqrt -qoverride-limits $<
   $(SHR_RANDNUM_C_OBJS): %.o: %.c
 	  $(CC) -c $(INCLDIR) $(INCS) $(CFLAGS) -O3 -fp-model fast $<
 endif
diff --git a/cime/config/acme/machines/Depends.intel14 b/cime/config/acme/machines/Depends.intel14
diff --git a/cime/config/acme/machines/Depends.intelmic b/cime/config/acme/machines/Depends.intelmic
diff --git a/cime/config/acme/machines/Depends.intelmic14 b/cime/config/acme/machines/Depends.intelmic14
diff --git a/cime/config/acme/machines/config_batch.xml b/cime/config/acme/machines/config_batch.xml
@@ -282,7 +282,7 @@
 
     <batch_system MACH="theta" type="cobalt_theta">
       <queues>
-        <queue walltimemax="01:00:00" jobmin="512" jobmax="231936" default="true">default</queue>
+        <queue walltimemin="00:30:00" walltimemax="02:00:00" jobmin="512" jobmax="231936" default="true">default</queue>
         <queue walltimemax="01:00:00" jobmin="1" jobmax="1024" strict="true">debug-cache-quad</queue>
       </queues>
     </batch_system>

diff --git a/cime/config/acme/machines/config_compilers.xml b/cime/config/acme/machines/config_compilers.xml
@@ -1121,6 +1121,11 @@ for mct, etc.
   <ADD_CPPDEFS> -DARCH_MIC_KNL </ADD_CPPDEFS>
 </compiler>
 
+<compiler COMPILER="gnu" MACH="theta">
+  <ADD_SLIBS>$(shell nf-config --flibs)</ADD_SLIBS>
+  <CONFIG_ARGS> --host=Linux </CONFIG_ARGS>
+</compiler>
+
 <compiler COMPILER="pgi" MACH="blues">
   <PNETCDF_PATH>$(PNETCDFROOT)</PNETCDF_PATH>
   <NETCDF_PATH>$(NETCDFROOT)</NETCDF_PATH>
@@ -1189,8 +1194,15 @@ for mct, etc.
 </compiler>
 
 <compiler COMPILER="intel" MACH="anvil">
-  <ADD_SLIBS>$(shell $(NETCDF_PATH)/bin/nf-config --flibs) -llapack -lblas -mkl </ADD_SLIBS>
-  <ADD_SLIBS> -Wl,-rpath -Wl,$(NETCDF_PATH)/lib </ADD_SLIBS>
+  <ADD_FFLAGS DEBUG="FALSE"> -O2 -debug minimal -qno-opt-dynamic-align</ADD_FFLAGS>
+  <ADD_CFLAGS compile_threaded="true"> -qopenmp -static-intel</ADD_CFLAGS>
+  <ADD_FFLAGS compile_threaded="true"> -qopenmp -static-intel</ADD_FFLAGS>
+  <ADD_FFLAGS_NOOPT compile_threaded="true"> -qopenmp -static-intel</ADD_FFLAGS_NOOPT>
+  <ADD_LDFLAGS compile_threaded="true"> -qopenmp -static-intel</ADD_LDFLAGS>
+  <ADD_CFLAGS compile_threaded="true" DEBUG="TRUE">-heap-arrays</ADD_CFLAGS>
+  <ADD_FFLAGS compile_threaded="true" DEBUG="TRUE">-heap-arrays</ADD_FFLAGS>
+  <ADD_SLIBS>$(shell $(NETCDF_PATH)/bin/nf-config --flibs) -Wl,-rpath -Wl,$(NETCDF_PATH)/lib</ADD_SLIBS>
+  <ADD_SLIBS>-mkl</ADD_SLIBS>
   <GPTL_CPPDEFS> -DHAVE_SLASHPROC </GPTL_CPPDEFS>
   <PIO_FILESYSTEM_HINTS>gpfs </PIO_FILESYSTEM_HINTS>
 </compiler>

diff --git a/cime/config/acme/machines/config_machines.xml b/cime/config/acme/machines/config_machines.xml
@@ -1101,11 +1101,14 @@
            <arguments>
              <arg name="num_tasks"> -n $TOTALPES </arg>
              <arg name="tasks_per_node"> --map-by ppr:{{ tasks_per_numa }}:socket:PE=$ENV{OMP_NUM_THREADS} --bind-to core</arg>
+             <!--arg name="label" DEBUG="TRUE">-tag-output</arg>
+             <arg name="bindings" DEBUG="TRUE">-report-bindings</arg-->
            </arguments>
          </mpirun>
          <mpirun mpilib="mvapich">
            <executable>mpiexec</executable>
            <arguments>
+             <arg name="label"> -l</arg>
              <arg name="num_tasks"> -n $TOTALPES </arg>
            </arguments>
          </mpirun>
@@ -1165,18 +1168,24 @@
       <env name="PNETCDF_PATH">$SHELL{which pnetcdf_version | xargs dirname | xargs dirname}</env>
     </environment_variables>
     <environment_variables SMP_PRESENT="TRUE">
-      <env name="OMP_STACKSIZE">256M</env>
-      <env name="OMP_PROC_BIND">spread</env>
-      <env name="OMP_PLACES">threads</env>
-      <env name="OMP_MAX_ACTIVE_LEVELS">1</env>
+      <env name="OMP_STACKSIZE">64M</env>
       <env name="KMP_HOT_TEAMS_MODE">1</env>
-      <env name="KMP_HOT_TEAMS_MAX_LEVEL">1</env>
+      <env name="OMP_DISPLAY_ENV" DEBUG="TRUE">verbose</env>
+      <env name="KMP_AFFINITY" DEBUG="TRUE">verbose,scatter</env>
+      <env name="KMP_AFFINITY" DEBUG="FALSE">scatter</env>
     </environment_variables>
-    <environment_variables mpilib="mvapich" SMP_PRESENT="FALSE">
+    <environment_variables mpilib="mvapich">
       <env name="MV2_ENABLE_AFFINITY">1</env>
       <env name="MV2_USE_SHARED_MEM">1</env>
       <env name="MV2_SMP_USE_CMA">1</env>
     </environment_variables>
+    <environment_variables SMP_PRESENT="TRUE" mpilib="mvapich">
+      <!--e.g. 6 threads: MV2_CPU_MAPPING=0-5:6-11:12-17:18-23:24-29:30-35: -->
+      <env name="MV2_CPU_MAPPING">$SHELL{t=$ENV{OMP_NUM_THREADS};b=0;r=$[36/$t];while [ $r -gt 0 ];do printf "$b-$[$b+$t-1]:";((r--));((b=b+t));done;}</env>
+    </environment_variables>
+    <environment_variables DEBUG="TRUE" mpilib="mvapich">
+      <env name="MV2_SHOW_CPU_BINDING">1</env>
+    </environment_variables>
 </machine>
 
 <machine MACH="bebop">
@@ -1563,7 +1572,7 @@
     <BATCH_SYSTEM>cobalt_theta</BATCH_SYSTEM>
     <SUPPORTED_BY>acme</SUPPORTED_BY>
     <GMAKE_J>8</GMAKE_J>
-    <MAX_TASKS_PER_NODE>64</MAX_TASKS_PER_NODE>
+    <MAX_TASKS_PER_NODE>128</MAX_TASKS_PER_NODE>
     <PES_PER_NODE>64</PES_PER_NODE>
     <PROJECT>OceanClimate</PROJECT>
     <PROJECT_REQUIRED>TRUE</PROJECT_REQUIRED>
@@ -1574,11 +1583,16 @@
       <arguments>
         <arg name="num_tasks" >-n $TOTALPES</arg>
         <arg name="tasks_per_node" >-N $PES_PER_NODE</arg>
-        <arg name="thread_count">--cc depth -d $OMP_NUM_THREADS</arg>
-        <arg name="env_omp_stacksize">-e OMP_STACKSIZE=128M</arg>
-        <arg name="env_thread_count">-e OMP_NUM_THREADS=$OMP_NUM_THREADS</arg>
+        <arg name="smp_vars">$ENV{SMP_VARS}</arg>
+        <!--arg name="mpi_env" DEBUG="TRUE">-e MPICH_VERSION_DISPLAY=1 -e MPICH_ENV_DISPLAY=1 -e MPICH_CPUMASK_DISPLAY=1</arg-->
       </arguments>
     </mpirun>
+    <environment_variables>
+      <env name="SMP_VARS"></env>
+    </environment_variables>
+    <environment_variables SMP_PRESENT="TRUE">
+      <env name="SMP_VARS">--cc depth -d $ENV{OMP_NUM_THREADS} -j $ENV{OMP_NUM_THREADS} -e OMP_NUM_THREADS=$ENV{OMP_NUM_THREADS} -e OMP_STACKSIZE=128M -e OMP_PROC_BIND=spread -e OMP_PLACES=threads</env>
+    </environment_variables>
     <module_system type="module">
       <init_path lang="perl">/opt/modules/default/init/perl.pm</init_path>
       <init_path lang="python">/opt/modules/default/init/python.py</init_path>
@@ -1605,26 +1619,26 @@
         <command name="rm">craype</command>
       </modules>
       <modules>
-        <command name="load">craype/2.5.11</command>
+        <command name="load">craype/2.5.12</command>
       </modules>
       <modules compiler="intel">
+        <command name="load">intel/18.0.0.128</command>
         <command name="load">PrgEnv-intel/6.0.4</command>
-        <command name="load">intel/17.0.4.196</command>
       </modules>
       <modules compiler="cray">
+        <command name="load">cce/8.6.2</command>
         <command name="load">PrgEnv-cray/6.0.4</command>
-        <command name="load">cce/8.6.0</command>
       </modules>
       <modules compiler="gnu">
-        <command name="load">PrgEnv-gnu/6.0.4</command>
         <command name="load">gcc/6.3.0</command>
+        <command name="load">PrgEnv-gnu/6.0.4</command>
       </modules>
       <modules compiler="!intel">
-        <command name="switch">cray-libsci/17.06.1</command>
+        <command name="switch">cray-libsci/17.09.1</command>
       </modules>
       <modules>
         <command name="load">craype-mic-knl</command>
-        <command name="load">cray-mpich/7.6.0</command>
+        <command name="load">cray-mpich/7.6.2</command>
       </modules>
       <modules mpilib="mpt">
         <command name="load">cray-netcdf-hdf5parallel/4.4.1.1.3</command>
@@ -1636,13 +1650,6 @@
         <command name="load">cray-netcdf/4.4.1.1.3</command>
       </modules>
     </module_system>
-    <environment_variables>
-      <env name="MPICH_ENV_DISPLAY">1</env>
-      <env name="MPICH_VERSION_DISPLAY">1</env>
-      <env name="OMP_STACKSIZE">128M</env>
-      <env name="OMP_PROC_BIND">spread</env>
-      <env name="OMP_PLACES">threads</env>
-    </environment_variables>
 </machine>
 
 <machine MACH="sooty">

diff --git a/cime/config/acme/testmods_dirs/allactive/clm/rd/user_nl_clm b/cime/config/acme/testmods_dirs/allactive/clm/rd/user_nl_clm
@@ -0,0 +1,2 @@
+finidat             = ''
+
diff --git a/cime/scripts/lib/CIME/SystemTests/system_tests_common.py b/cime/scripts/lib/CIME/SystemTests/system_tests_common.py
@@ -197,6 +197,7 @@ def _set_active_case(self, case):
         Use for tests that have multiple cases
         """
         self._case = case
+        self._case.load_env(reset=True)
         self._caseroot = case.get_value("CASEROOT")
 
     def run_indv(self, suffix="base", st_archive=False):

diff --git a/cime/scripts/lib/CIME/case.py b/cime/scripts/lib/CIME/case.py
@@ -1198,7 +1198,7 @@ def get_mpirun_cmd(self, job="case.run"):
         executable, mpi_arg_list = env_mach_specific.get_mpirun(self, mpi_attribs, job=job)
 
         # special case for aprun
-        if executable is not None and "aprun" in executable:
+        if executable is not None and "aprun" in executable and "titan" in self.get_value("MACH"):
             aprun_args, num_nodes = get_aprun_cmd_for_case(self, run_exe)
             expect( (num_nodes + self.spare_nodes) == self.num_nodes, "Not using optimized num nodes")
             return executable + aprun_args + " " + run_misc_suffix
@@ -1231,8 +1231,9 @@ def set_model_version(self, model):
         else:
             logger.warn("WARNING: No {} Model version found.".format(model))
 
-    def load_env(self):
-        if not self._is_env_loaded:
+    def load_env(self, reset=False):
+        if not self._is_env_loaded or reset:
+            os.environ["OMP_NUM_THREADS"] = str(self.thread_count)
             env_module = self.get_env("mach_specific")
             env_module.load_env(self)
             self._is_env_loaded = True

diff --git a/cime/scripts/lib/update_acme_tests.py b/cime/scripts/lib/update_acme_tests.py
@@ -78,6 +78,7 @@
                              ("ERS.f19_g16.I1850CLM45","clm-vst"),
                              ("ERS.f09_g16.I1850CLM45CN","clm-bgcinterface"),
                               "ERS.ne11_oQU240.I20TRCLM45",
+                             ("ERS.f19_g16.I1850CNRDCTCBC","clm-rd"),
                               "ERS.f09_g16.IMCLM45BC")
                              ),
 

diff --git a/cime/src/components/data_comps/datm/cime_config/namelist_definition_datm.xml b/cime/src/components/data_comps/datm/cime_config/namelist_definition_datm.xml
@@ -189,6 +189,9 @@
       <value stream="CLMCRUNCEPv7">$DIN_LOC_ROOT_CLMFORC/atm_forcing.datm7.cruncep_qianFill.0.5d.v7.c160715</value>
       <value stream="CLMCRUNCEP">$DIN_LOC_ROOT/share/domains/domain.clm</value>
       <value stream="CLMGSWP3v1">$DIN_LOC_ROOT_CLMFORC/atm_forcing.datm7.GSWP3.0.5d.v1.c170516</value>
+      <value stream="CLMCRUNCEP\.">$DIN_LOC_ROOT/share/domains/domain.clm</value>
+      <value stream="CLMCRUNCEP_V5">$DIN_LOC_ROOT_CLMFORC/atm_forcing.datm7.cruncep_qianFill.0.5d.V5.c140715</value>
+      <value stream="CLMGSWP3">$DIN_LOC_ROOT_CLMFORC/atm_forcing.datm7.GSWP3.0.5d.v1.c170516</value>
       <value stream="CORE2_NYF">$DIN_LOC_ROOT/atm/datm7/NYF</value>
       <value stream="CORE2_IAF.CORE2.ArcFactor">$DIN_LOC_ROOT/atm/datm7/CORE2</value>
       <value stream="CORE2_IAF.NCEP.DENS.SOFS">$DIN_LOC_ROOT/share/domains</value>
@@ -242,6 +245,7 @@
       <value stream="CLMCRUNCEPv7">domain.lnd.360x720_cruncep.130305.nc</value>
       <value stream="CLMCRUNCEP">domain.lnd.360x720.130305.nc</value>
       <value stream="CLMGSWP3v1">domain.lnd.360x720_gswp3.0v1.c170606.nc</value>
+      <value stream="CLMGSWP3">domain.lnd.360x720_gswp3.0v1.c170606.nc</value>
       <value stream="CORE2_NYF.GISS">nyf.giss.T62.051007.nc</value>
       <value stream="CORE2_NYF.GXGXS">nyf.gxgxs.T62.051007.nc</value>
       <value stream="CORE2_NYF.NCEP">nyf.ncep.T62.050923.nc</value>
@@ -375,6 +379,12 @@
       <value stream="CLMGSWP3v1.Solar">$DIN_LOC_ROOT_CLMFORC/atm_forcing.datm7.GSWP3.0.5d.v1.c170516/Solar</value>
       <value stream="CLMGSWP3v1.Precip">$DIN_LOC_ROOT_CLMFORC/atm_forcing.datm7.GSWP3.0.5d.v1.c170516/Precip</value>
       <value stream="CLMGSWP3v1.TPQW">$DIN_LOC_ROOT_CLMFORC/atm_forcing.datm7.GSWP3.0.5d.v1.c170516/TPHWL</value>
+      <value stream="CLMCRUNCEP_V5.Solar">$DIN_LOC_ROOT_CLMFORC/atm_forcing.datm7.cruncep_qianFill.0.5d.V5.c140715/Solar6Hrly</value>
+      <value stream="CLMCRUNCEP_V5.Precip">$DIN_LOC_ROOT_CLMFORC/atm_forcing.datm7.cruncep_qianFill.0.5d.V5.c140715/Precip6Hrly</value>
+      <value stream="CLMCRUNCEP_V5.TPQW">$DIN_LOC_ROOT_CLMFORC/atm_forcing.datm7.cruncep_qianFill.0.5d.V5.c140715/TPHWL6Hrly</value>
+      <value stream="CLMGSWP3.Solar">$DIN_LOC_ROOT_CLMFORC/atm_forcing.datm7.GSWP3.0.5d.v1.c170516/Solar3Hrly</value>
+      <value stream="CLMGSWP3.Precip">$DIN_LOC_ROOT_CLMFORC/atm_forcing.datm7.GSWP3.0.5d.v1.c170516/Precip3Hrly</value>
+      <value stream="CLMGSWP3.TPQW">$DIN_LOC_ROOT_CLMFORC/atm_forcing.datm7.GSWP3.0.5d.v1.c170516/TPHWL3Hrly</value>
       <value stream="CORE2_NYF">$DIN_LOC_ROOT/atm/datm7/NYF</value>
       <value stream="CORE2_IAF.CORE2.ArcFactor">$DIN_LOC_ROOT/atm/datm7/CORE2</value>
       <value stream="CORE2_IAF">$DIN_LOC_ROOT/ocn/iaf</value>

diff --git a/cime/src/drivers/mct/cime_config/config_component_acme.xml b/cime/src/drivers/mct/cime_config/config_component_acme.xml
@@ -178,9 +178,13 @@
       <value compset="_DATM"    >none</value>
       <value compset="_BGC%BPRP">CO2C</value>
       <value compset="_BGC%BDRD">CO2C</value>
-      <value compset="HIST.*_DATM%(QIA|CRU)">CO2A</value>
-      <value compset="20TR.*_DATM%(QIA|CRU)">CO2A</value>
-      <value compset="RCP.*_DATM%(QIA|CRU)" >CO2A</value>
+      <value compset="HIST.*_DATM%(QIA|CRU|GSW)">CO2A</value>
+      <value compset="20TR.*_DATM%(QIA|CRU|GSW)">CO2A</value>
+      <value compset="RCP.*_DATM%(QIA|CRU|GSW)" >CO2A</value>
+      <value compset="_DATM%IAF*_MPASO%OECO" >CO2A</value>
+      <value compset="_DATM%IAF*_MPASO%OIECO" >CO2A</value>
+      <value compset="_DATM%IAF*_MPASO%TOIECO_" >CO2A</value>
+      <value compset="_DATM%IAF_SLND_MPASCICE%BGC_MPASO%TOIECO_" >CO2A</value>
     </values>
     <group>run_coupling</group>
     <file>env_run.xml</file>