diff --git a/config/acme/machines/config_batch.xml b/config/acme/machines/config_batch.xml index e611ca68a28..9b3c01773e2 100644 --- a/config/acme/machines/config_batch.xml +++ b/config/acme/machines/config_batch.xml @@ -35,27 +35,28 @@ - - qstat - qsub - qdel - -v - - (\d+) - --dependencies - %H:%M:%s - -M - - - - - - - - - - - + + qstat + qsub + qdel + -v + + (\d+) + --dependencies jobid + : + %H:%M:%s + -M + + + + + + + + + + + qstat @@ -64,7 +65,8 @@ -v #COBALT (\d+) - --dependencies + --dependencies jobid + : -M @@ -84,7 +86,8 @@ < #BSUB <(\d+)> - -w 'done(jobid)' + -w 'done(jobid)' + && %H:%M -u @@ -114,7 +117,8 @@ -v #PBS ^(\S+)$ - -W depend=afterok:jobid + -W depend=afterok:jobid + : %H:%M:%S -M -m @@ -140,7 +144,8 @@ canceljob #MSUB (\d+)$ - -W depend=afterok:jobid + -W depend=afterok:jobid + : %H:%M:%S -M -m @@ -165,6 +170,7 @@ #SBATCH (\d+)$ -l depend=jobid + : %H:%M:%S --mail-user --mail-type @@ -187,29 +193,30 @@ - - squeue - sbatch - scancel - #SBATCH - (\d+)$ - --dependency=afterok:jobid - %H:%M:%S - --mail-user - --mail-type - none, all, begin, end, fail - - - - - - - --job-name={{ job_id }} - --nodes={{ num_nodes }} - --output={{ output_error_path }}.%j - --exclusive - - + + squeue + sbatch + scancel + #SBATCH + (\d+)$ + --dependency=afterok:jobid + : + %H:%M:%S + --mail-user + --mail-type + none, all, begin, end, fail + + + + + + + --job-name={{ job_id }} + --nodes={{ num_nodes }} + --output={{ output_error_path }}.%j + --exclusive + + diff --git a/config/xml_schemas/config_batch.xsd b/config/xml_schemas/config_batch.xsd index 227cc6fa4e1..4a29fc5dd40 100644 --- a/config/xml_schemas/config_batch.xsd +++ b/config/xml_schemas/config_batch.xsd @@ -16,6 +16,7 @@ + @@ -68,6 +69,9 @@ a previous job has completed successfully --> + + + diff --git a/scripts/Tools/case.submit b/scripts/Tools/case.submit index fc1787680bf..1a8a42bffc4 100755 --- a/scripts/Tools/case.submit +++ b/scripts/Tools/case.submit @@ -61,24 +61,23 @@ OR args = CIME.utils.parse_args_and_handle_standard_logging_options(args, parser) - CIME.utils.expect(args.prereq is None, "--prereq not currently supported") - - return args.test, args.caseroot, args.job, args.no_batch, args.resubmit, \ - args.skip_preview_namelist, args.mail_user, args.mail_type, \ + return args.test, args.caseroot, args.job, args.no_batch, args.prereq, \ + args.resubmit, args.skip_preview_namelist, args.mail_user, args.mail_type, \ args.batch_args ############################################################################### def _main_func(description): ############################################################################### - test, caseroot, job, no_batch, resubmit, skip_pnl, \ + test, caseroot, job, no_batch, prereq, resubmit, skip_pnl, \ mail_user, mail_type, batch_args = parse_command_line(sys.argv, description) if test: test_results = doctest.testmod(verbose=True) sys.exit(1 if test_results.failed > 0 else 0) with Case(caseroot, read_only=False) as case: - submit(case, job=job, no_batch=no_batch, resubmit=resubmit, skip_pnl=skip_pnl, - mail_user=mail_user, mail_type=mail_type, batch_args=batch_args) + submit(case, job=job, no_batch=no_batch, prereq=prereq, resubmit=resubmit, + skip_pnl=skip_pnl, mail_user=mail_user, mail_type=mail_type, + batch_args=batch_args) if __name__ == "__main__": _main_func(__doc__) diff --git a/scripts/lib/CIME/XML/env_batch.py b/scripts/lib/CIME/XML/env_batch.py index 86e496ddc1b..b3ba5178c6b 100644 --- a/scripts/lib/CIME/XML/env_batch.py +++ b/scripts/lib/CIME/XML/env_batch.py @@ -21,7 +21,6 @@ def __init__(self, case_root=None, infile="env_batch.xml"): """ initialize an object interface to file env_batch.xml in the case directory """ - self._prereq_jobid = None self._batchtype = None # This arbitrary setting should always be overwritten self._default_walltime = "00:20:00" @@ -316,9 +315,9 @@ def get_submit_args(self, case, job): return submitargs - def submit_jobs(self, case, no_batch=False, job=None, skip_pnl=False, - mail_user=None, mail_type='never', batch_args=None, - dry_run=False): + def submit_jobs(self, case, no_batch=False, job=None, user_prereq=None, + skip_pnl=False, mail_user=None, mail_type='never', + batch_args=None, dry_run=False): alljobs = self.get_jobs() startindex = 0 jobs = [] @@ -355,25 +354,16 @@ def submit_jobs(self, case, no_batch=False, job=None, skip_pnl=False, deps = dependency.split() else: deps = [] - jobid = "" - if self._prereq_jobid is not None: - jobid = self._prereq_jobid + dep_jobs = [] + if user_prereq is not None: + dep_jobs.append(user_prereq) for dep in deps: - if dep in depid and depid[dep] is not None: - jobid += " " + str(depid[dep]) -#TODO: doubt these will be used -# elif dep == "and": -# jobid += " && " -# elif dep == "or": -# jobid += " || " + if dep in depid.keys() and depid[dep] is not None: + dep_jobs.append(str(depid[dep])) - - slen = len(jobid) - if slen == 0: - jobid = None - - logger.warning("job is {}".format(job)) - result = self._submit_single_job(case, job, jobid, + logger.warning("job {} depends on {}".format(job, dep_jobs)) + result = self._submit_single_job(case, job, + dep_jobs=dep_jobs, no_batch=no_batch, skip_pnl=skip_pnl, mail_user=mail_user, @@ -391,7 +381,7 @@ def submit_jobs(self, case, no_batch=False, job=None, skip_pnl=False, else: return depid - def _submit_single_job(self, case, job, depid=None, no_batch=False, + def _submit_single_job(self, case, job, dep_jobs=None, no_batch=False, skip_pnl=False, mail_user=None, mail_type='never', batch_args=None, dry_run=False): logger.warning("Submit job {}".format(job)) @@ -415,9 +405,15 @@ def _submit_single_job(self, case, job, depid=None, no_batch=False, if args_override: submitargs = args_override - if depid is not None: + if dep_jobs is not None and len(dep_jobs) > 0: + logger.info("dependencies: {}".format(dep_jobs)) dep_string = self.get_value("depend_string", subgroup=None) - dep_string = dep_string.replace("jobid",depid.strip()) # pylint: disable=maybe-no-member + separator_string = self.get_value("depend_separator", subgroup=None) + expect("jobid" in dep_string, "depend_string is missing jobid for prerequisite jobs") + dep_ids_str = str(dep_jobs[0]) + for dep_id in dep_jobs[1:]: + dep_ids_str += separator_string + str(dep_id) + dep_string = dep_string.replace("jobid",dep_ids_str.strip()) # pylint: disable=maybe-no-member submitargs += " " + dep_string if batch_args is not None: diff --git a/scripts/lib/CIME/case.py b/scripts/lib/CIME/case.py index fcfc37f27b4..f6ea456ea76 100644 --- a/scripts/lib/CIME/case.py +++ b/scripts/lib/CIME/case.py @@ -1165,11 +1165,11 @@ def _get_comp_user_mods(self, component): else: return comp_user_mods - def submit_jobs(self, no_batch=False, job=None, skip_pnl=False, + def submit_jobs(self, no_batch=False, job=None, prereq=None, skip_pnl=False, mail_user=None, mail_type='never', batch_args=None, dry_run=False): env_batch = self.get_env('batch') - return env_batch.submit_jobs(self, no_batch=no_batch, job=job, + return env_batch.submit_jobs(self, no_batch=no_batch, job=job, user_prereq=prereq, skip_pnl=skip_pnl, mail_user=mail_user, mail_type=mail_type, batch_args=batch_args, dry_run=dry_run) diff --git a/scripts/lib/CIME/case_submit.py b/scripts/lib/CIME/case_submit.py index 20243b6368e..8a4008e6f74 100644 --- a/scripts/lib/CIME/case_submit.py +++ b/scripts/lib/CIME/case_submit.py @@ -15,8 +15,8 @@ logger = logging.getLogger(__name__) -def _submit(case, job=None, resubmit=False, no_batch=False, skip_pnl=False, - mail_user=None, mail_type='never', batch_args=None): +def _submit(case, job=None, no_batch=False, prereq=None, resubmit=False, + skip_pnl=False, mail_user=None, mail_type='never', batch_args=None): if job is None: if case.get_value("TEST"): job = "case.test" @@ -64,8 +64,8 @@ def _submit(case, job=None, resubmit=False, no_batch=False, skip_pnl=False, logger.warning("submit_jobs {}".format(job)) job_ids = case.submit_jobs(no_batch=no_batch, job=job, skip_pnl=skip_pnl, - mail_user=mail_user, mail_type=mail_type, - batch_args=batch_args) + prereq=prereq, mail_user=mail_user, + mail_type=mail_type, batch_args=batch_args) xml_jobids = [] for jobname, jobid in job_ids.items(): @@ -77,8 +77,8 @@ def _submit(case, job=None, resubmit=False, no_batch=False, skip_pnl=False, if xml_jobid_text: case.set_value("JOB_IDS", xml_jobid_text) -def submit(case, job=None, resubmit=False, no_batch=False, skip_pnl=False, - mail_user=None, mail_type='never', batch_args=None): +def submit(case, job=None, no_batch=False, prereq=None, resubmit=False, + skip_pnl=False, mail_user=None, mail_type='never', batch_args=None): if case.get_value("TEST"): caseroot = case.get_value("CASEROOT") casebaseid = case.get_value("CASEBASEID") @@ -93,8 +93,10 @@ def submit(case, job=None, resubmit=False, no_batch=False, skip_pnl=False, ts.set_status(SUBMIT_PHASE, TEST_PASS_STATUS) try: - functor = lambda: _submit(case, job, resubmit, no_batch, skip_pnl, - mail_user, mail_type, batch_args) + functor = lambda: _submit(case, job=job, no_batch=no_batch, prereq=prereq, + resubmit=resubmit, skip_pnl=skip_pnl, + mail_user=mail_user, mail_type=mail_type, + batch_args=batch_args) run_and_log_case_status(functor, "case.submit", caseroot=case.get_value("CASEROOT")) except: # If something failed in the batch system, make sure to mark diff --git a/scripts/tests/scripts_regression_tests.py b/scripts/tests/scripts_regression_tests.py index f9f0b83bafc..20ce9bad2dc 100755 --- a/scripts/tests/scripts_regression_tests.py +++ b/scripts/tests/scripts_regression_tests.py @@ -13,6 +13,8 @@ from six import assertRaisesRegex import six +import collections + from CIME.utils import run_cmd, run_cmd_no_fail, get_lids, get_current_commit import update_acme_tests import CIME.test_scheduler, CIME.wait_for_tests @@ -1394,6 +1396,47 @@ def test_cime_case(self): # Test some test properties self.assertEqual(case.get_value("TESTCASE"), "TESTRUNPASS") + ########################################################################### + def test_cime_case_prereq(self): + ########################################################################### + testcase_name = 'prereq_test' + testdir = os.path.join(TEST_ROOT, testcase_name) + if os.path.exists(testdir): + shutil.rmtree(testdir) + run_cmd_assert_result(self, ("%s/create_newcase --case %s --script-root %s --compset X --res f19_g16 --output-root %s" + % (SCRIPT_DIR, testcase_name, testdir, testdir)), + from_dir=SCRIPT_DIR) + + with Case(testdir, read_only=False) as case: + job_name = "case.run" + prereq_name = 'prereq_test' + batch_commands = case.submit_jobs(prereq=prereq_name, job=job_name, skip_pnl=True, dry_run=True) + self.assertTrue(isinstance(batch_commands, collections.Sequence), "case.submit_jobs did not return a sequence for a dry run") + self.assertTrue(len(batch_commands) > 0, "case.submit_jobs did not return any job submission string") + # The first element in the internal sequence should just be the job name + # The second one (batch_cmd_index) should be the actual batch submission command + batch_cmd_index = 1 + # The prerequisite should be applied to all jobs, though we're only expecting one + for batch_cmd in batch_commands: + self.assertTrue(isinstance(batch_cmd, collections.Sequence), "case.submit_jobs did not return a sequence of sequences") + self.assertTrue(len(batch_cmd) > batch_cmd_index, "case.submit_jobs returned internal sequences with length <= {}".format(batch_cmd_index)) + self.assertTrue(isinstance(batch_cmd[1], str), "case.submit_jobs returned internal sequences without the batch command string as the second parameter: {}".format(batch_cmd[1])) + batch_cmd_args = batch_cmd[1] + + jobid_ident = 'jobid' + dep_str_fmt = case.get_env('batch').get_value('depend_string', subgroup=None) + self.assertTrue(jobid_ident in dep_str_fmt, "dependency string doesn't include the jobid identifier {}".format(jobid_ident)) + dep_str = dep_str_fmt[:-len(jobid_ident)] + + while dep_str in batch_cmd_args: + dep_id_pos = batch_cmd_args.find(dep_str) + len(dep_str) + batch_cmd_args = batch_cmd_args[dep_id_pos:] + prereq_substr = batch_cmd_args[:len(prereq_name)] + if prereq_substr == prereq_name: + break + + self.assertTrue(prereq_name in prereq_substr, "Dependencies added, but not the user specified one") + ########################################################################### def test_cime_case_build_threaded_1(self): ###########################################################################