From 0e1553fe113bf1d19664c49e1ee3a63da879956b Mon Sep 17 00:00:00 2001 From: James Foucar Date: Wed, 27 Jul 2016 17:42:19 -0600 Subject: [PATCH] Add a new tool for very simple python profiling More advanced tools coming. Other changes: 1) Rename system_test.py to test_scheduler.py to reduce name overloading of system_test 2) Fix pylint errors that have built-up recently --- scripts/Tools/bless_test_results | 2 +- scripts/Tools/simple-py-prof | 5 +++ scripts/create_test | 28 ++++++++----- utils/python/CIME/case.py | 20 +++++----- .../{system_test.py => test_scheduler.py} | 40 ++++++++++++------- utils/python/CIME/test_utils.py | 2 +- utils/python/cs.submit.template | 2 +- .../python/tests/scripts_regression_tests.py | 26 ++++++------ 8 files changed, 72 insertions(+), 53 deletions(-) create mode 100755 scripts/Tools/simple-py-prof rename utils/python/CIME/{system_test.py => test_scheduler.py} (97%) diff --git a/scripts/Tools/bless_test_results b/scripts/Tools/bless_test_results index 9ffcd4dd6e60..1f58a729eaf5 100755 --- a/scripts/Tools/bless_test_results +++ b/scripts/Tools/bless_test_results @@ -13,7 +13,7 @@ with versions of the files that you should not bless. from standard_script_setup import * import wait_for_tests, compare_namelists, simple_compare -from CIME.system_test import NAMELIST_PHASE +from CIME.test_scheduler import NAMELIST_PHASE from CIME.utils import run_cmd, run_cmd_no_fail, expect from CIME.XML.machines import Machines diff --git a/scripts/Tools/simple-py-prof b/scripts/Tools/simple-py-prof new file mode 100755 index 000000000000..f904521e397d --- /dev/null +++ b/scripts/Tools/simple-py-prof @@ -0,0 +1,5 @@ +#! /bin/bash + +# Usage: simple-py-prof + +python -m cProfile -s time "$@" diff --git a/scripts/create_test b/scripts/create_test index 19d0a4bfb87a..20bd712ed782 100755 --- a/scripts/create_test +++ b/scripts/create_test @@ -11,7 +11,7 @@ If this tool is missing any feature that you need, please notify jgfouca@sandia. from Tools.standard_script_setup import * import update_acme_tests -from CIME.system_test import SystemTest, RUN_PHASE +from CIME.test_scheduler import TestScheduler, RUN_PHASE from CIME.utils import expect, convert_to_seconds, compute_total_time, convert_to_babylonian_time, run_cmd_no_fail from CIME.XML.machines import Machines from CIME.case import Case @@ -73,6 +73,9 @@ OR parser.add_argument("--no-build", action="store_true", help="Do not build generated tests, implies --no-run") + parser.add_argument("--no-setup", action="store_true", + help="Do not setup generated tests, implies --no-build and --no-run") + parser.add_argument("-u", "--use-existing", action="store_true", help="Use pre-existing case directories. Requires test-id") @@ -190,11 +193,15 @@ OR if args.use_existing: expect(args.test_id is not None, "Must provide test-id of pre-existing cases") + if args.no_setup: + args.no_build = True + if args.no_build: args.no_run = True # Namelist-only forces some other options: if args.namelists_only: + expect(not args.no_setup, "Cannot compare namelists without setup") args.no_build = True args.no_run = True args.no_batch = True @@ -208,14 +215,13 @@ OR if args.test_id is None: args.test_id = CIME.utils.get_utc_timestamp() - if args.testfile is not None: with open(args.testfile[0], "r") as fd: - args.testargs.extend( fd.read().splitlines() ) + args.testargs.extend( fd.read().splitlines() ) # Remove empty items if any args.testargs = filter(None, args.testargs) - return args.testargs, args.compiler, args.machine, args.no_run, args.no_build, args.no_batch,\ + return args.testargs, args.compiler, args.machine, args.no_run, args.no_build, args.no_setup, args.no_batch,\ args.test_root, args.baseline_root, args.clean, args.compare, args.generate, \ args.baseline_name, args.namelists_only, args.project, args.test_id, args.parallel_jobs, \ args.xml_machine, args.xml_compiler, args.xml_category, args.xml_testlist, args.walltime, \ @@ -321,7 +327,7 @@ def single_submit_impl(machine_name, test_id, proc_pool, project, args, job_cost run_cmd_no_fail(submit_cmd, input_str=script, arg_stdout=None, arg_stderr=None, verbose=True) ############################################################################### -def create_test(testargs, compiler, machine_name, no_run, no_build, no_batch, test_root, +def create_test(testargs, compiler, machine_name, no_run, no_build, no_setup, no_batch, test_root, baseline_root, clean, compare, generate, baseline_name, namelists_only, project, test_id, parallel_jobs, xml_machine, xml_compiler, xml_category, xml_testlist, walltime, @@ -337,8 +343,8 @@ def create_test(testargs, compiler, machine_name, no_run, no_build, no_batch, te expect(machine_name == testsplit[4], "ambiguity in machine, please use the --machine option") - impl = SystemTest(testargs, - no_run=no_run, no_build=no_build, no_batch=no_batch, + impl = TestScheduler(testargs, + no_run=no_run, no_build=no_build, no_setup=no_setup, no_batch=no_batch, test_root=test_root, test_id=test_id, baseline_root=baseline_root, baseline_name=baseline_name, clean=clean, machine_name=machine_name, compiler=compiler, @@ -348,7 +354,7 @@ def create_test(testargs, compiler, machine_name, no_run, no_build, no_batch, te xml_category=xml_category, xml_testlist=xml_testlist, walltime=walltime, proc_pool=proc_pool, use_existing=use_existing, save_timing=save_timing) - success = impl.system_test() + success = impl.run_tests() if single_submit: # Get real test root @@ -378,17 +384,17 @@ def create_test(testargs, compiler, machine_name, no_run, no_build, no_batch, te def _main_func(description): ############################################################################### if "--test" in sys.argv: - CIME.utils.run_cmd_no_fail("python -m doctest %s/CIME/system_test.py -v" % + CIME.utils.run_cmd_no_fail("python -m doctest %s/CIME/test_scheduler.py -v" % CIME.utils.get_python_libs_root(), arg_stdout=None, arg_stderr=None) return - testargs, compiler, machine_name, no_run, no_build, no_batch, test_root, baseline_root, clean, \ + testargs, compiler, machine_name, no_run, no_build, no_setup, no_batch, test_root, baseline_root, clean, \ compare, generate, baseline_name, namelists_only, project, test_id, parallel_jobs, \ xml_machine, xml_compiler, xml_category, xml_testlist, walltime, single_submit, proc_pool, \ use_existing, save_timing \ = parse_command_line(sys.argv, description) - sys.exit(create_test(testargs, compiler, machine_name, no_run, no_build, no_batch, test_root, + sys.exit(create_test(testargs, compiler, machine_name, no_run, no_build, no_setup, no_batch, test_root, baseline_root, clean, compare, generate, baseline_name, namelists_only, project, test_id, parallel_jobs, xml_machine, xml_compiler, xml_category, xml_testlist, walltime, single_submit, proc_pool, use_existing, save_timing)) diff --git a/utils/python/CIME/case.py b/utils/python/CIME/case.py index 30209d39c4f0..63dd6a3a5de3 100644 --- a/utils/python/CIME/case.py +++ b/utils/python/CIME/case.py @@ -538,20 +538,18 @@ def configure(self, compset_name, grid_name, machine_name=None, #-------------------------------------------- # pe payout #-------------------------------------------- - match1 = None - match2 = None + match1 = re.match('([0-9]+)x([0-9]+)', "" if pecount is None else pecount) + match2 = re.match('([0-9]+)', "" if pecount is None else pecount) pes_ntasks = {} pes_nthrds = {} pes_rootpe = {} - if pecount is not None: - match1 = re.match('([0-9]+)x([0-9]+)', pecount) - match2 = re.match('([0-9]+)', pecount) - if match1: - opti_tasks = match1.group(1) - opti_thrds = match1.group(2) - elif match2: - opti_tasks = match2.group(1) - opti_thrds = 1 + if match1: + opti_tasks = match1.group(1) + opti_thrds = match1.group(2) + elif match2: + opti_tasks = match2.group(1) + opti_thrds = 1 + if match1 or match2: for component_class in self._component_classes: if component_class == "DRV": diff --git a/utils/python/CIME/system_test.py b/utils/python/CIME/test_scheduler.py similarity index 97% rename from utils/python/CIME/system_test.py rename to utils/python/CIME/test_scheduler.py index 20f2f3e4bca8..75c103125b4c 100644 --- a/utils/python/CIME/system_test.py +++ b/utils/python/CIME/test_scheduler.py @@ -1,6 +1,9 @@ """ -Implementation of System Test functionality from CIME +A library for scheduling/running through the phases of a set +of system tests. Supports phase-level parallelism (can make progres +on multiple system tests at once). """ + import shutil, traceback, stat, glob, threading, time, thread from CIME.XML.standard_module_setup import * import compare_namelists @@ -30,12 +33,12 @@ logger = logging.getLogger(__name__) ############################################################################### -class SystemTest(object): +class TestScheduler(object): ############################################################################### ########################################################################### def __init__(self, test_names, - no_run=False, no_build=False, no_batch=None, + no_run=False, no_build=False, no_setup=False, no_batch=None, test_root=None, test_id=None, machine_name=None, compiler=None, baseline_root=None, baseline_name=None, @@ -61,8 +64,9 @@ def __init__(self, test_names, self._machobj = Machines(machine=machine_name) machine_name = self._machobj.get_machine_name() - self._no_build = no_build if not namelists_only else True - self._no_run = no_run if not self._no_build else True + self._no_setup = no_setup + self._no_build = no_build or no_setup or namelists_only + self._no_run = no_run or self._no_build # Figure out what project to use if project is None: @@ -203,10 +207,12 @@ def __init__(self, test_names, # Setup phases self._phases = list(PHASES) - if no_build: + if self._no_setup: + self._phases.remove(SETUP_PHASE) + if self._no_build: self._phases.remove(SHAREDLIB_BUILD_PHASE) self._phases.remove(MODEL_BUILD_PHASE) - if no_run: + if self._no_run: self._phases.remove(RUN_PHASE) if not self._compare and not self._generate: self._phases.remove(NAMELIST_PHASE) @@ -793,28 +799,32 @@ def _setup_cs_files(self): template_file = os.path.join(python_libs_root, "cs.submit.template") template = open(template_file, "r").read() - build_cmd = "./*.build" if self._no_build else ":" - cmd = "./*.test" if self._no_batch else "./*.submit" - template = template.replace("", build_cmd).\ - replace("", cmd).\ + setup_cmd = "./case.setup" if self._no_setup else ":" + build_cmd = "./case.build" if self._no_build else ":" + test_cmd = "./case.submit" + template = template.replace("", setup_cmd).\ + replace("", build_cmd).\ + replace("", test_cmd).\ replace("", self._test_id) - if self._no_build or self._no_run: + if self._no_run: cs_submit_file = os.path.join(self._test_root, "cs.submit.%s" % self._test_id) with open(cs_submit_file, "w") as fd: fd.write(template) os.chmod(cs_submit_file, os.stat(cs_submit_file).st_mode | stat.S_IXUSR | stat.S_IXGRP) + if CIME.utils.get_model == "cesm": testreporter = os.path.join(self._test_root,"testreporter.pl") shutil.copy(os.path.join(self._cime_root,"scripts","Testing","testreporter.pl"), testreporter) os.chmod(testreporter, os.stat(testreporter).st_mode | stat.S_IXUSR | stat.S_IXGRP) + except Exception as e: logger.warning("FAILED to set up cs files: %s" % str(e)) ########################################################################### - def system_test(self): + def run_tests(self): ########################################################################### """ Main API for this class. @@ -838,7 +848,7 @@ def system_test(self): self._setup_cs_files() # Return True if all tests passed - logger.info( "At system_test close, state is:") + logger.info( "At test-scheduler close, state is:") rv = True for test in self._tests: phase, status, nl_fail = self._get_test_data(test) @@ -864,6 +874,6 @@ def system_test(self): logger.info( " Case dir: %s" % self._get_test_dir(test)) - logger.info( "system_test took %s seconds"% (time.time() - start_time)) + logger.info( "test-scheduler took %s seconds"% (time.time() - start_time)) return rv diff --git a/utils/python/CIME/test_utils.py b/utils/python/CIME/test_utils.py index 5ef436b2ffd8..74e59941cc06 100644 --- a/utils/python/CIME/test_utils.py +++ b/utils/python/CIME/test_utils.py @@ -1,5 +1,5 @@ """ -Utility functions used in system_test.py +Utility functions used in test_scheduler.py """ from CIME.XML.standard_module_setup import * diff --git a/utils/python/cs.submit.template b/utils/python/cs.submit.template index a04f875266ae..34c4eb297142 100644 --- a/utils/python/cs.submit.template +++ b/utils/python/cs.submit.template @@ -2,6 +2,6 @@ for item in $(\ls -1 *./TestStatus); do cd $(dirname $item) - && + && && cd - done diff --git a/utils/python/tests/scripts_regression_tests.py b/utils/python/tests/scripts_regression_tests.py index 58ea89c3000f..2eb9a03e437b 100755 --- a/utils/python/tests/scripts_regression_tests.py +++ b/utils/python/tests/scripts_regression_tests.py @@ -13,8 +13,8 @@ from CIME.utils import run_cmd, run_cmd_no_fail import CIME.utils, update_acme_tests, wait_for_tests -import CIME.system_test -from CIME.system_test import SystemTest +import CIME.test_scheduler +from CIME.test_scheduler import TestScheduler from CIME.XML.machines import Machines from CIME.XML.files import Files from CIME.case import Case @@ -476,7 +476,7 @@ def test_create_test_rebless_namelist(self): self.simple_test(True, "-c -n -b %s -t %s-%s" % (self._baseline_name, self._baseline_name, CIME.utils.get_utc_timestamp())) ############################################################################### -class E_TestSystemTest(TestCreateTestCommon): +class E_TestTestScheduler(TestCreateTestCommon): ############################################################################### ########################################################################### @@ -487,7 +487,7 @@ def test_a_phases(self): "^TESTMEMLEAKFAIL_Mmpi-serial.f19_g16.X", "^TESTMEMLEAKPASS_Mmpi-serial.f19_g16.X"], self._machine, self._compiler) self.assertEqual(len(tests), 3) - ct = SystemTest(tests) + ct = TestScheduler(tests) build_fail_test = [item for item in tests if "TESTBUILDFAIL" in item][0] run_fail_test = [item for item in tests if "TESTRUNFAIL" in item][0] @@ -499,9 +499,9 @@ def test_a_phases(self): for idx, phase in enumerate(ct._phases): for test in ct._tests: - if (phase == CIME.system_test.INITIAL_PHASE): + if (phase == CIME.test_scheduler.INITIAL_PHASE): continue - elif (phase == CIME.system_test.MODEL_BUILD_PHASE): + elif (phase == CIME.test_scheduler.MODEL_BUILD_PHASE): ct._update_test_status(test, phase, wait_for_tests.TEST_PENDING_STATUS) if (test == build_fail_test): @@ -513,7 +513,7 @@ def test_a_phases(self): self.assertFalse(ct._is_broken(test)) self.assertTrue(ct._work_remains(test)) - elif (phase == CIME.system_test.RUN_PHASE): + elif (phase == CIME.test_scheduler.RUN_PHASE): if (test == build_fail_test): with self.assertRaises(SystemExit): ct._update_test_status(test, phase, wait_for_tests.TEST_PENDING_STATUS) @@ -557,7 +557,7 @@ def test_b_full(self): ########################################################################### tests = update_acme_tests.get_full_test_names(["acme_test_only"], self._machine, self._compiler) test_id="%s-%s" % (self._baseline_name, CIME.utils.get_utc_timestamp()) - ct = SystemTest(tests, test_id=test_id, no_batch=NO_BATCH) + ct = TestScheduler(tests, test_id=test_id, no_batch=NO_BATCH) build_fail_test = [item for item in tests if "TESTBUILDFAIL" in item][0] run_fail_test = [item for item in tests if "TESTRUNFAIL" in item][0] @@ -568,7 +568,7 @@ def test_b_full(self): log_lvl = logging.getLogger().getEffectiveLevel() logging.disable(logging.CRITICAL) try: - ct.system_test() + ct.run_tests() finally: logging.getLogger().setLevel(log_lvl) @@ -581,16 +581,16 @@ def test_b_full(self): for test_status in test_statuses: status, test_name = wait_for_tests.parse_test_status_file(test_status) if (test_name == build_fail_test): - self.assertEqual(status[CIME.system_test.MODEL_BUILD_PHASE], wait_for_tests.TEST_FAIL_STATUS) + self.assertEqual(status[CIME.test_scheduler.MODEL_BUILD_PHASE], wait_for_tests.TEST_FAIL_STATUS) elif (test_name == run_fail_test): - self.assertEqual(status[CIME.system_test.RUN_PHASE], wait_for_tests.TEST_FAIL_STATUS) + self.assertEqual(status[CIME.test_scheduler.RUN_PHASE], wait_for_tests.TEST_FAIL_STATUS) elif (test_name == mem_fail_test): self.assertTrue("memleak" in status, "memleak missing in %s for test %s" % (status, test_name)) self.assertEqual(status["memleak"], wait_for_tests.TEST_FAIL_STATUS) - self.assertEqual(status[CIME.system_test.RUN_PHASE], wait_for_tests.TEST_PASS_STATUS) + self.assertEqual(status[CIME.test_scheduler.RUN_PHASE], wait_for_tests.TEST_PASS_STATUS) else: self.assertTrue(test_name in [pass_test, mem_pass_test]) - self.assertEqual(status[CIME.system_test.RUN_PHASE], wait_for_tests.TEST_PASS_STATUS) + self.assertEqual(status[CIME.test_scheduler.RUN_PHASE], wait_for_tests.TEST_PASS_STATUS) if (test_name == mem_pass_test): self.assertEqual(status["memleak"], wait_for_tests.TEST_PASS_STATUS)