Merge branch 'master' of https://github.com/ESMCI/cime

cime_config/cesm/allactive/testmods_dirs/allactive/cism/test_coupling/include_user_mods

@@ -1 +1 @@
-../apply_to_multiinstance
+../../defaultio

utils/python/CIME/SystemTests/nck.py

@@ -23,44 +23,58 @@ def __init__(self, case):
         SystemTestsCommon.__init__(self, case)
 
     def build(self, sharedlib_only=False, model_only=False):
+        '''
+        build can be called once (sharedlib_only and model_only both False)
+        or twice (once with each true)
+        This test requires a sharedlib build for both phases
+        we must handle both cases correctly
+        '''
         exeroot = self._case.get_value("EXEROOT")
         cime_model = CIME.utils.get_model()
-
-        machpes1 = os.path.join("LockedFiles","env_mach_pes.NCK1.xml")
-        if ( os.path.isfile(machpes1) ):
-            shutil.copy(machpes1,"env_mach_pes.xml")
+        if not model_only:
+            machpes1 = os.path.join("LockedFiles","env_mach_pes.orig.xml")
+            if os.path.isfile(machpes1):
+                shutil.copy(machpes1,"env_mach_pes.xml")
+            else:
+                shutil.copy("env_mach_pes.xml", machpes1)
 
         # Build two exectuables for this test, the first is a default build, the
         # second halves the number of tasks and runs two instances for each component
         # Lay all of the components out sequentially
         for bld in range(1,3):
             logging.warn("Starting bld %s"%bld)
             machpes = os.path.join("LockedFiles","env_mach_pes.NCK%s.xml"%bld)
-            for comp in ['ATM','OCN','WAV','GLC','ICE','ROF','LND']:
-                self._case.set_value("NINST_%s"%comp, bld)
-                if(bld == 2):
+            if model_only:
+                # This file should have been created in the sharedlib_only phase
+                shutil.copy(machpes,"env_mach_pes.xml")
+                self._case.read_xml(self._caseroot)
+            else:
+                for comp in ['ATM','OCN','WAV','GLC','ICE','ROF','LND']:
+                    self._case.set_value("NINST_%s"%comp, bld)
                     ntasks      = self._case.get_value("NTASKS_%s"%comp)
-                    rootpe      = self._case.get_value("ROOTPE_%s"%comp)
-                    if ( ntasks > 1 ):
-                        self._case.set_value("NTASKS_%s"%comp, ntasks/2)
-                        self._case.set_value("ROOTPE_%s"%comp, rootpe/2)
-            self._case.flush()
+                    if(bld == 1):
+                        if ( ntasks > 1 ):
+                            self._case.set_value("NTASKS_%s"%comp, int(ntasks/2))
+                    else:
+                        self._case.set_value("NTASKS_%s"%comp, ntasks*2)
+                    self._case.flush()
 
             case_setup(self._case, test_mode=True, reset=True)
-            self.clean_build()
+            if not sharedlib_only:
+                self.clean_build()
 
             SystemTestsCommon.build(self, sharedlib_only=sharedlib_only, model_only=model_only)
-            if (not sharedlib_only):
-                shutil.move("%s/%s.exe"%(exeroot,cime_model),
-                            "%s/%s.exe.NCK%s"%(exeroot,cime_model,bld))
-            shutil.copy("env_build.xml",os.path.join("LockedFiles","env_build.NCK%s.xml"%bld))
-            shutil.copy("env_mach_pes.xml", machpes)
+            if not model_only:
+                shutil.copy("env_mach_pes.xml", machpes)
+            if not sharedlib_only:
+                shutil.move("%s/%s.exe"%(exeroot,cime_model),"%s/%s.exe.NCK%s"%(exeroot,cime_model,bld))
+                shutil.copy("env_build.xml",os.path.join("LockedFiles","env_build.NCK%s.xml"%bld))
 
         # Because mira/cetus interprets its run script differently than
         # other systems we need to copy the original env_mach_pes.xml back
-        shutil.copy(machpes1,"env_mach_pes.xml")
-        shutil.copy("env_mach_pes.xml",
-                    os.path.join("LockedFiles","env_mach_pes.xml"))
+#        shutil.copy(machpes1,"env_mach_pes.xml")
+#        shutil.copy("env_mach_pes.xml",
+#                    os.path.join("LockedFiles","env_mach_pes.xml"))
 
     def run(self):
         os.chdir(self._caseroot)
@@ -106,6 +120,8 @@ def run(self):
                         "%s/%s.exe" % (exeroot, cime_model))
             shutil.copy("LockedFiles/env_build.NCK2.xml", "env_build.xml")
             shutil.copy("env_build.xml", "LockedFiles/env_build.xml")
+            shutil.copy("LockedFiles/env_mach_pes.NCK2.xml", "env_mach_pes.xml")
+            shutil.copy("env_mach_pes.xml", "LockedFiles/env_mach_pes.xml")
 
             logger.info("default: doing a %s %s with NINST2" % (stop_n, stop_option))
             success = SystemTestsCommon._run(self, "multiinst")

utils/python/CIME/buildnml.py

@@ -81,7 +81,7 @@ def build_xcpl_nml(argv, compclass):
         if ninst == 1:
             filename = os.path.join(rundir, "%s_in" % compname)
         else:
-            filename = os.path.join(rundir, "%s_in%4.4d" % (compname, i))
+            filename = os.path.join(rundir, "%s_in_%4.4d" % (compname, i))
 
         with open(filename, 'w') as infile:
             infile.write("%-20d ! i-direction global dimension\n" % nx)

utils/python/CIME/case_setup.py

@@ -9,7 +9,8 @@
 from CIME.XML.env_mach_pes  import EnvMachPes
 from CIME.XML.component     import Component
 from CIME.XML.compilers     import Compilers
-from CIME.utils             import expect, run_cmd, append_status
+from CIME.utils             import expect, run_cmd, append_status, parse_test_name
+from CIME.user_mod_support  import apply_user_mods
 
 import shutil, time, glob
 
@@ -159,14 +160,17 @@ def case_setup(case, clean=False, test_mode=False, reset=False):
 
         # Check ninst.
         # In CIME there can be multiple instances of each component model (an ensemble) NINST is the instance of that component.
+        # Save ninst in a dict to use later in apply_user_mods
+        ninst = dict()
         for comp in models:
-            ninst  = case.get_value("NINST_%s" % comp)
+            comp_model = case.get_value("COMP_%s" % comp)
+            ninst[comp_model]  = case.get_value("NINST_%s" % comp)
             ntasks = case.get_value("NTASKS_%s" % comp)
-            if ninst > ntasks:
+            if ninst[comp_model] > ntasks:
                 if ntasks == 1:
-                    case.set_value("NTASKS_%s" % comp, ninst)
+                    case.set_value("NTASKS_%s" % comp, ninst[comp_model])
                 else:
-                    expect(False, "NINST_%s value %d greater than NTASKS_%s %d" % (comp, ninst, comp, ntasks))
+                    expect(False, "NINST_%s value %d greater than NTASKS_%s %d" % (comp, ninst[comp_model], comp, ntasks))
 
         expect(not (case.get_value("BUILD_THREADED") and case.get_value("COMPILER") == "nag"),
                "it is not possible to run with OpenMP if using the NAG Fortran compiler")
@@ -251,6 +255,12 @@ def case_setup(case, clean=False, test_mode=False, reset=False):
 
         _build_usernl_files(case, "drv", "cpl")
 
+        if case.get_value("TEST"):
+            test_mods = parse_test_name(case.get_value("CASEBASEID"))[6]
+            if test_mods is not None:
+                user_mods_path = os.path.join(case.get_value("TESTS_MODS_DIR"), test_mods)
+                apply_user_mods(caseroot, user_mods_path=user_mods_path, ninst=ninst)
+
         # Run preview namelists for scripts
         logger.info("preview_namelists")
         preview_namelists(case)

utils/python/CIME/user_mod_support.py

@@ -10,7 +10,7 @@
 
 def apply_user_mods(caseroot, user_mods_path, ninst=None):
     '''
-    Recursivlely apply user_mods to caseroot - this includes updating user_nl_xxx, 
+    Recursivlely apply user_mods to caseroot - this includes updating user_nl_xxx,
     updating SourceMods and creating case_shel_commands and xmlchange_cmds files
     '''
     include_dirs = build_include_dirs_list(user_mods_path)
@@ -23,10 +23,10 @@ def apply_user_mods(caseroot, user_mods_path, ninst=None):
             case_user_nl = user_nl.replace(include_dir, caseroot)
             comp = case_user_nl.split('_')[-1]
             if ninst is not None and comp in ninst.keys():
-                for comp_inst in xrange(1, ninst[comp]):
+                for comp_inst in xrange(1, ninst[comp]+1):
                     case_user_nl_inst = case_user_nl + "_%4.4d"%comp_inst
                     logger.info("Pre-pending file %s"%case_user_nl_inst)
-                    if os.path.isfile(case_user_nl):
+                    if os.path.isfile(case_user_nl_inst):
                         with open(case_user_nl_inst, "r") as fd:
                             old_contents = fd.read()
                             contents = contents + old_contents
@@ -70,7 +70,7 @@ def apply_user_mods(caseroot, user_mods_path, ninst=None):
             with open(case_shell_commands, "a") as fd:
                 fd.write(new_shell_commands)
 
-    shell_command_files = [os.path.join(caseroot,"shell_commands"), 
+    shell_command_files = [os.path.join(caseroot,"shell_commands"),
                            os.path.join(caseroot,"xmlchange_cmnds")]
     for shell_command_file in shell_command_files:
         if os.path.isfile(shell_command_file):

utils/python/CIME/utils.py

@@ -2,12 +2,7 @@
 Common functions used by cime python scripts
 Warning: you cannot use CIME Classes in this module as it causes circular dependencies
 """
-import logging
-import logging.config
-import sys
-import os
-import time
-import re
+import logging, gzip, sys, os, time, re, shutil
 
 # Return this error code if the scripts worked but tests failed
 TESTS_FAILED_ERR_CODE = 100
@@ -454,7 +449,6 @@ def safe_copy(src_dir, tgt_dir, file_map):
     read-only file. Files can be relative paths and the relative path will be
     matched on the tgt side.
     """
-    import shutil
     for src_file, tgt_file in file_map:
         full_tgt = os.path.join(tgt_dir, tgt_file)
         full_src = src_file if os.path.isabs(src_file) else os.path.join(src_dir, src_file)