Add a new tool for very simple python profiling

More advanced tools coming.

Other changes:
1) Rename system_test.py to test_scheduler.py to reduce name overloading of system_test
2) Fix pylint errors that have built-up recently

scripts/Tools/bless_test_results

@@ -13,7 +13,7 @@ with versions of the files that you should not bless.
 from standard_script_setup import *
 
 import wait_for_tests, compare_namelists, simple_compare
-from CIME.system_test import NAMELIST_PHASE
+from CIME.test_scheduler import NAMELIST_PHASE
 from CIME.utils import run_cmd, run_cmd_no_fail, expect
 from CIME.XML.machines import Machines
 

scripts/Tools/simple-py-prof

@@ -0,0 +1,5 @@
+#! /bin/bash
+
+# Usage: simple-py-prof <python exec and arguments>
+
+python -m cProfile -s time "$@"

scripts/create_test

@@ -11,7 +11,7 @@ If this tool is missing any feature that you need, please notify jgfouca@sandia.
 from Tools.standard_script_setup import *
 
 import update_acme_tests
-from CIME.system_test import SystemTest, RUN_PHASE
+from CIME.test_scheduler import TestScheduler, RUN_PHASE
 from CIME.utils import expect, convert_to_seconds, compute_total_time, convert_to_babylonian_time, run_cmd_no_fail
 from CIME.XML.machines import Machines
 from CIME.case import Case
@@ -73,6 +73,9 @@ OR
     parser.add_argument("--no-build", action="store_true",
                         help="Do not build generated tests, implies --no-run")
 
+    parser.add_argument("--no-setup", action="store_true",
+                        help="Do not setup generated tests, implies --no-build and --no-run")
+
     parser.add_argument("-u", "--use-existing", action="store_true",
                         help="Use pre-existing case directories. Requires test-id")
 
@@ -190,11 +193,15 @@ OR
     if args.use_existing:
         expect(args.test_id is not None, "Must provide test-id of pre-existing cases")
 
+    if args.no_setup:
+        args.no_build = True
+
     if args.no_build:
         args.no_run = True
 
     # Namelist-only forces some other options:
     if args.namelists_only:
+        expect(not args.no_setup, "Cannot compare namelists without setup")
         args.no_build = True
         args.no_run   = True
         args.no_batch = True
@@ -208,14 +215,13 @@ OR
     if args.test_id is None:
         args.test_id = CIME.utils.get_utc_timestamp()
 
-
     if args.testfile is not None:
         with open(args.testfile[0], "r") as fd:
-                args.testargs.extend( fd.read().splitlines() )
+            args.testargs.extend( fd.read().splitlines() )
     # Remove empty items if any
     args.testargs = filter(None, args.testargs)
 
-    return args.testargs, args.compiler, args.machine, args.no_run, args.no_build, args.no_batch,\
+    return args.testargs, args.compiler, args.machine, args.no_run, args.no_build, args.no_setup, args.no_batch,\
         args.test_root, args.baseline_root, args.clean, args.compare, args.generate, \
         args.baseline_name, args.namelists_only, args.project, args.test_id, args.parallel_jobs, \
         args.xml_machine, args.xml_compiler, args.xml_category, args.xml_testlist, args.walltime, \
@@ -321,7 +327,7 @@ def single_submit_impl(machine_name, test_id, proc_pool, project, args, job_cost
     run_cmd_no_fail(submit_cmd, input_str=script, arg_stdout=None, arg_stderr=None, verbose=True)
 
 ###############################################################################
-def create_test(testargs, compiler, machine_name, no_run, no_build, no_batch, test_root,
+def create_test(testargs, compiler, machine_name, no_run, no_build, no_setup, no_batch, test_root,
                 baseline_root, clean, compare, generate,
                 baseline_name, namelists_only, project, test_id, parallel_jobs,
                 xml_machine, xml_compiler, xml_category, xml_testlist, walltime,
@@ -337,8 +343,8 @@ def create_test(testargs, compiler, machine_name, no_run, no_build, no_batch, te
                     expect(machine_name == testsplit[4],
                            "ambiguity in machine, please use the --machine option")
 
-    impl = SystemTest(testargs,
-                      no_run=no_run, no_build=no_build, no_batch=no_batch,
+    impl = TestScheduler(testargs,
+                      no_run=no_run, no_build=no_build, no_setup=no_setup, no_batch=no_batch,
                       test_root=test_root, test_id=test_id,
                       baseline_root=baseline_root, baseline_name=baseline_name,
                       clean=clean, machine_name=machine_name, compiler=compiler,
@@ -348,7 +354,7 @@ def create_test(testargs, compiler, machine_name, no_run, no_build, no_batch, te
                       xml_category=xml_category, xml_testlist=xml_testlist, walltime=walltime,
                       proc_pool=proc_pool, use_existing=use_existing, save_timing=save_timing)
 
-    success = impl.system_test()
+    success = impl.run_tests()
 
     if single_submit:
         # Get real test root
@@ -378,17 +384,17 @@ def create_test(testargs, compiler, machine_name, no_run, no_build, no_batch, te
 def _main_func(description):
 ###############################################################################
     if "--test" in sys.argv:
-        CIME.utils.run_cmd_no_fail("python -m doctest %s/CIME/system_test.py -v" %
+        CIME.utils.run_cmd_no_fail("python -m doctest %s/CIME/test_scheduler.py -v" %
                                    CIME.utils.get_python_libs_root(), arg_stdout=None, arg_stderr=None)
         return
 
-    testargs, compiler, machine_name, no_run, no_build, no_batch, test_root, baseline_root, clean, \
+    testargs, compiler, machine_name, no_run, no_build, no_setup, no_batch, test_root, baseline_root, clean, \
         compare, generate, baseline_name, namelists_only, project, test_id, parallel_jobs, \
         xml_machine, xml_compiler, xml_category, xml_testlist, walltime, single_submit, proc_pool, \
         use_existing, save_timing \
         = parse_command_line(sys.argv, description)
 
-    sys.exit(create_test(testargs, compiler, machine_name, no_run, no_build, no_batch, test_root,
+    sys.exit(create_test(testargs, compiler, machine_name, no_run, no_build, no_setup, no_batch, test_root,
                          baseline_root, clean, compare, generate, baseline_name, namelists_only,
                          project, test_id, parallel_jobs, xml_machine, xml_compiler, xml_category,
                          xml_testlist, walltime, single_submit, proc_pool, use_existing, save_timing))

utils/python/CIME/case.py

@@ -538,20 +538,18 @@ def configure(self, compset_name, grid_name, machine_name=None,
         #--------------------------------------------
         # pe payout
         #--------------------------------------------
-        match1 = None
-        match2 = None
+        match1 = re.match('([0-9]+)x([0-9]+)', "" if pecount is None else pecount)
+        match2 = re.match('([0-9]+)', "" if pecount is None else pecount)
         pes_ntasks = {}
         pes_nthrds = {}
         pes_rootpe = {}
-        if pecount is not None:
-            match1 =  re.match('([0-9]+)x([0-9]+)', pecount)
-            match2 =  re.match('([0-9]+)', pecount)
-            if match1:
-                opti_tasks = match1.group(1)
-                opti_thrds = match1.group(2)
-            elif match2:
-                opti_tasks = match2.group(1)
-                opti_thrds = 1
+        if match1:
+            opti_tasks = match1.group(1)
+            opti_thrds = match1.group(2)
+        elif match2:
+            opti_tasks = match2.group(1)
+            opti_thrds = 1
+
         if match1 or match2:
             for component_class in self._component_classes:
                 if component_class == "DRV":

utils/python/CIME/system_test.py → utils/python/CIME/test_scheduler.py

@@ -1,6 +1,9 @@
 """
-Implementation of System Test functionality from CIME
+A library for scheduling/running through the phases of a set
+of system tests. Supports phase-level parallelism (can make progres
+on multiple system tests at once).
 """
+
 import shutil, traceback, stat, glob, threading, time, thread
 from CIME.XML.standard_module_setup import *
 import compare_namelists
@@ -30,12 +33,12 @@
 logger = logging.getLogger(__name__)
 
 ###############################################################################
-class SystemTest(object):
+class TestScheduler(object):
 ###############################################################################
 
     ###########################################################################
     def __init__(self, test_names,
-                 no_run=False, no_build=False, no_batch=None,
+                 no_run=False, no_build=False, no_setup=False, no_batch=None,
                  test_root=None, test_id=None,
                  machine_name=None, compiler=None,
                  baseline_root=None, baseline_name=None,
@@ -61,8 +64,9 @@ def __init__(self, test_names,
         self._machobj = Machines(machine=machine_name)
         machine_name = self._machobj.get_machine_name()
 
-        self._no_build = no_build if not namelists_only else True
-        self._no_run = no_run if not self._no_build else True
+        self._no_setup = no_setup
+        self._no_build = no_build or no_setup or namelists_only
+        self._no_run   = no_run or self._no_build
 
         # Figure out what project to use
         if project is None:
@@ -203,10 +207,12 @@ def __init__(self, test_names,
 
         # Setup phases
         self._phases = list(PHASES)
-        if no_build:
+        if self._no_setup:
+            self._phases.remove(SETUP_PHASE)
+        if self._no_build:
             self._phases.remove(SHAREDLIB_BUILD_PHASE)
             self._phases.remove(MODEL_BUILD_PHASE)
-        if no_run:
+        if self._no_run:
             self._phases.remove(RUN_PHASE)
         if not self._compare and not self._generate:
             self._phases.remove(NAMELIST_PHASE)
@@ -793,28 +799,32 @@ def _setup_cs_files(self):
 
             template_file = os.path.join(python_libs_root, "cs.submit.template")
             template = open(template_file, "r").read()
-            build_cmd = "./*.build" if self._no_build else ":"
-            cmd = "./*.test" if self._no_batch else "./*.submit"
-            template = template.replace("<BUILD_CMD>", build_cmd).\
-                       replace("<RUN_CMD>", cmd).\
+            setup_cmd = "./case.setup" if self._no_setup else ":"
+            build_cmd = "./case.build" if self._no_build else ":"
+            test_cmd  = "./case.submit"
+            template = template.replace("<SETUP_CMD>", setup_cmd).\
+                       replace("<BUILD_CMD>", build_cmd).\
+                       replace("<RUN_CMD>", test_cmd).\
                        replace("<TESTID>", self._test_id)
 
-            if self._no_build or self._no_run:
+            if self._no_run:
                 cs_submit_file = os.path.join(self._test_root, "cs.submit.%s" % self._test_id)
                 with open(cs_submit_file, "w") as fd:
                     fd.write(template)
                 os.chmod(cs_submit_file,
                          os.stat(cs_submit_file).st_mode | stat.S_IXUSR | stat.S_IXGRP)
+
             if CIME.utils.get_model == "cesm":
                 testreporter =  os.path.join(self._test_root,"testreporter.pl")
                 shutil.copy(os.path.join(self._cime_root,"scripts","Testing","testreporter.pl"),
                             testreporter)
                 os.chmod(testreporter, os.stat(testreporter).st_mode | stat.S_IXUSR | stat.S_IXGRP)
+
         except Exception as e:
             logger.warning("FAILED to set up cs files: %s" % str(e))
 
     ###########################################################################
-    def system_test(self):
+    def run_tests(self):
     ###########################################################################
         """
         Main API for this class.
@@ -838,7 +848,7 @@ def system_test(self):
         self._setup_cs_files()
 
         # Return True if all tests passed
-        logger.info( "At system_test close, state is:")
+        logger.info( "At test-scheduler close, state is:")
         rv = True
         for test in self._tests:
             phase, status, nl_fail = self._get_test_data(test)
@@ -864,6 +874,6 @@ def system_test(self):
 
             logger.info( "    Case dir: %s" % self._get_test_dir(test))
 
-        logger.info( "system_test took %s seconds"% (time.time() - start_time))
+        logger.info( "test-scheduler took %s seconds"% (time.time() - start_time))
 
         return rv

utils/python/CIME/test_utils.py

@@ -1,5 +1,5 @@
 """
-Utility functions used in system_test.py
+Utility functions used in test_scheduler.py
 """
 
 from CIME.XML.standard_module_setup import *

utils/python/cs.submit.template

@@ -2,6 +2,6 @@
 
 for item in $(\ls -1 *.<TESTID>/TestStatus); do
     cd $(dirname $item)
-    <BUILD_CMD> && <RUN_CMD>
+    <SETUP_CMD> && <BUILD_CMD> && <RUN_CMD>
     cd -
 done

utils/python/tests/scripts_regression_tests.py

@@ -13,8 +13,8 @@
 
 from CIME.utils import run_cmd, run_cmd_no_fail
 import CIME.utils, update_acme_tests, wait_for_tests
-import CIME.system_test
-from  CIME.system_test import SystemTest
+import CIME.test_scheduler
+from  CIME.test_scheduler import TestScheduler
 from  CIME.XML.machines import Machines
 from  CIME.XML.files import Files
 from  CIME.case import Case
@@ -476,7 +476,7 @@ def test_create_test_rebless_namelist(self):
         self.simple_test(True, "-c -n -b %s -t %s-%s" % (self._baseline_name, self._baseline_name, CIME.utils.get_utc_timestamp()))
 
 ###############################################################################
-class E_TestSystemTest(TestCreateTestCommon):
+class E_TestTestScheduler(TestCreateTestCommon):
 ###############################################################################
 
     ###########################################################################
@@ -487,7 +487,7 @@ def test_a_phases(self):
                                                        "^TESTMEMLEAKFAIL_Mmpi-serial.f19_g16.X", "^TESTMEMLEAKPASS_Mmpi-serial.f19_g16.X"],
                                                       self._machine, self._compiler)
         self.assertEqual(len(tests), 3)
-        ct = SystemTest(tests)
+        ct = TestScheduler(tests)
 
         build_fail_test = [item for item in tests if "TESTBUILDFAIL" in item][0]
         run_fail_test   = [item for item in tests if "TESTRUNFAIL" in item][0]
@@ -499,9 +499,9 @@ def test_a_phases(self):
 
         for idx, phase in enumerate(ct._phases):
             for test in ct._tests:
-                if (phase == CIME.system_test.INITIAL_PHASE):
+                if (phase == CIME.test_scheduler.INITIAL_PHASE):
                     continue
-                elif (phase == CIME.system_test.MODEL_BUILD_PHASE):
+                elif (phase == CIME.test_scheduler.MODEL_BUILD_PHASE):
                     ct._update_test_status(test, phase, wait_for_tests.TEST_PENDING_STATUS)
 
                     if (test == build_fail_test):
@@ -513,7 +513,7 @@ def test_a_phases(self):
                         self.assertFalse(ct._is_broken(test))
                         self.assertTrue(ct._work_remains(test))
 
-                elif (phase == CIME.system_test.RUN_PHASE):
+                elif (phase == CIME.test_scheduler.RUN_PHASE):
                     if (test == build_fail_test):
                         with self.assertRaises(SystemExit):
                             ct._update_test_status(test, phase, wait_for_tests.TEST_PENDING_STATUS)
@@ -557,7 +557,7 @@ def test_b_full(self):
     ###########################################################################
         tests = update_acme_tests.get_full_test_names(["acme_test_only"], self._machine, self._compiler)
         test_id="%s-%s" % (self._baseline_name, CIME.utils.get_utc_timestamp())
-        ct = SystemTest(tests, test_id=test_id, no_batch=NO_BATCH)
+        ct = TestScheduler(tests, test_id=test_id, no_batch=NO_BATCH)
 
         build_fail_test = [item for item in tests if "TESTBUILDFAIL" in item][0]
         run_fail_test   = [item for item in tests if "TESTRUNFAIL" in item][0]
@@ -568,7 +568,7 @@ def test_b_full(self):
         log_lvl = logging.getLogger().getEffectiveLevel()
         logging.disable(logging.CRITICAL)
         try:
-            ct.system_test()
+            ct.run_tests()
         finally:
             logging.getLogger().setLevel(log_lvl)
 
@@ -581,16 +581,16 @@ def test_b_full(self):
         for test_status in test_statuses:
             status, test_name = wait_for_tests.parse_test_status_file(test_status)
             if (test_name == build_fail_test):
-                self.assertEqual(status[CIME.system_test.MODEL_BUILD_PHASE], wait_for_tests.TEST_FAIL_STATUS)
+                self.assertEqual(status[CIME.test_scheduler.MODEL_BUILD_PHASE], wait_for_tests.TEST_FAIL_STATUS)
             elif (test_name == run_fail_test):
-                self.assertEqual(status[CIME.system_test.RUN_PHASE], wait_for_tests.TEST_FAIL_STATUS)
+                self.assertEqual(status[CIME.test_scheduler.RUN_PHASE], wait_for_tests.TEST_FAIL_STATUS)
             elif (test_name == mem_fail_test):
                 self.assertTrue("memleak" in status, "memleak missing in %s for test %s" % (status, test_name))
                 self.assertEqual(status["memleak"], wait_for_tests.TEST_FAIL_STATUS)
-                self.assertEqual(status[CIME.system_test.RUN_PHASE], wait_for_tests.TEST_PASS_STATUS)
+                self.assertEqual(status[CIME.test_scheduler.RUN_PHASE], wait_for_tests.TEST_PASS_STATUS)
             else:
                 self.assertTrue(test_name in [pass_test, mem_pass_test])
-                self.assertEqual(status[CIME.system_test.RUN_PHASE], wait_for_tests.TEST_PASS_STATUS)
+                self.assertEqual(status[CIME.test_scheduler.RUN_PHASE], wait_for_tests.TEST_PASS_STATUS)
                 if (test_name == mem_pass_test):
                     self.assertEqual(status["memleak"], wait_for_tests.TEST_PASS_STATUS)