From af10dbe8884e5977af10f5d48db6a57c0cab4b8c Mon Sep 17 00:00:00 2001 From: James Foucar Date: Thu, 13 Apr 2017 16:12:39 -0600 Subject: [PATCH 1/7] New tool: get_key_commands --- scripts/Tools/get_key_commands | 63 +++++++++++++++++++++++++++++++ scripts/lib/CIME/XML/env_batch.py | 50 ++++++++++++++---------- scripts/lib/CIME/case.py | 5 ++- scripts/lib/CIME/case_setup.py | 13 +------ 4 files changed, 97 insertions(+), 34 deletions(-) create mode 100755 scripts/Tools/get_key_commands diff --git a/scripts/Tools/get_key_commands b/scripts/Tools/get_key_commands new file mode 100755 index 00000000000..32e67539846 --- /dev/null +++ b/scripts/Tools/get_key_commands @@ -0,0 +1,63 @@ +#!/usr/bin/env python + +""" +Script to query key CIME shell commands (mpirun and batch submission). + +To force a certain mpirun command, use: +./xmlchange MPIRUN_OVERRIDE $your_cmd + +To force a certain qsub command, use: +./xmlchange SUBMIT_OVERRIDE $your_cmd +""" + +from standard_script_setup import * + +from CIME.case import Case + +############################################################################### +def parse_command_line(args, description): +############################################################################### + parser = argparse.ArgumentParser( + usage="""\n%s [--verbose] +OR +%s --help +OR +%s --test + +\033[1mEXAMPLES:\033[0m + \033[1;32m# Run the tool \033[0m + > %s +""" % ((os.path.basename(args[0]), ) * 4), + description=description, + formatter_class=argparse.ArgumentDefaultsHelpFormatter + ) + + parser.add_argument("caseroot", nargs="?", default=os.getcwd(), + help="Case directory to build") + + args = parser.parse_args(args[1:]) + + return args.caseroot + +############################################################################### +def _main_func(description): +############################################################################### + if "--test" in sys.argv: + test_results = doctest.testmod(verbose=True) + sys.exit(1 if test_results.failed > 0 else 0) + + caseroot = parse_command_line(sys.argv, description) + + logging.disable(logging.CRITICAL) + + with Case(caseroot, read_only=False) as case: + print "BATCH SUBMIT:" + job = "case.test" if case.get_value("TEST") else "case.run" + job_id_to_cmd = case.submit_jobs(dry_run=True, job=job) + for job_id, cmd in job_id_to_cmd: + print " ", job_id, "->", case.get_resolved_value(cmd) + print + print "MPIRUN:", case.get_resolved_value(case.get_mpirun_cmd()) + +if __name__ == "__main__": + _main_func(__doc__) diff --git a/scripts/lib/CIME/XML/env_batch.py b/scripts/lib/CIME/XML/env_batch.py index 88fcdfc8f21..96e78a5df64 100644 --- a/scripts/lib/CIME/XML/env_batch.py +++ b/scripts/lib/CIME/XML/env_batch.py @@ -281,7 +281,7 @@ def get_submit_args(self, case, job): return submitargs - def submit_jobs(self, case, no_batch=False, job=None, batch_args=None): + def submit_jobs(self, case, no_batch=False, job=None, batch_args=None, dry_run=False): alljobs = self.get_jobs() startindex = 0 jobs = [] @@ -299,16 +299,23 @@ def submit_jobs(self, case, no_batch=False, job=None, batch_args=None): if prereq is None or job == firstjob: prereq = True else: + if dry_run: + # Assume build is complete + prereq = prereq.replace("$BUILD_COMPLETE", "True") prereq = case.get_resolved_value(prereq) prereq = eval(prereq) except: expect(False,"Unable to evaluate prereq expression '%s' for job '%s'"%(self.get_value('prereq',subgroup=job), job)) + if prereq: - jobs.append((job,self.get_value('dependency', subgroup=job))) + jobs.append((job, self.get_value('dependency', subgroup=job))) + if self.batchtype == "cobalt": break depid = {} + jobcmds = [] + for job, dependency in jobs: if dependency is not None: deps = dependency.split() @@ -331,16 +338,21 @@ def submit_jobs(self, case, no_batch=False, job=None, batch_args=None): if slen == 0: jobid = None - logger.warn("job is %s"%job) - depid[job] = self.submit_single_job(case, job, jobid, no_batch=no_batch, batch_args=batch_args) + logger.warn("job is %s" % job) + result = self._submit_single_job(case, job, jobid, no_batch=no_batch, batch_args=batch_args, dry_run=dry_run) + batch_job_id = str(alljobs.index(job)) if dry_run else result + depid[job] = batch_job_id + jobcmds.append( (job, result) ) if self.batchtype == "cobalt": break - return sorted(list(depid.values())) + if dry_run: + return jobcmds + else: + return sorted(list(depid.values())) - def submit_single_job(self, case, job, depid=None, no_batch=False, batch_args=None): + def _submit_single_job(self, case, job, depid=None, no_batch=False, batch_args=None, dry_run=False): logger.warn("Submit job %s"%job) - caseroot = case.get_value("CASEROOT") batch_system = self.get_value("BATCH_SYSTEM", subgroup=None) if batch_system is None or batch_system == "none" or no_batch: # Import here to avoid circular include @@ -351,14 +363,9 @@ def submit_single_job(self, case, job, depid=None, no_batch=False, batch_args=No logger.info("Starting job script %s" % job) - # Hack until all testcases are ported to python - testcase = case.get_value("TESTCASE") - cimeroot = get_cime_root() - testscript = os.path.join(cimeroot, "scripts", "Testing", "Testcases", "%s_script" % testcase) - if job == "case.test" and testcase is not None and os.path.exists(testscript): - run_cmd_no_fail("%s --caseroot %s" % (os.path.join(".", job), caseroot)) - else: - # This is what we want longterm + # This is what we want longterm + function_name = job.replace(".", "_") + if not dry_run: function_name = job.replace(".", "_") locals()[function_name](case) @@ -383,11 +390,14 @@ def submit_single_job(self, case, job, depid=None, no_batch=False, batch_args=No if string is not None: submitcmd += string + " " - logger.info("Submitting job script %s"%submitcmd) - output = run_cmd_no_fail(submitcmd, combine_output=True) - jobid = self.get_job_id(output) - logger.info("Submitted job id is %s"%jobid) - return jobid + if dry_run: + return submitcmd + else: + logger.info("Submitting job script %s"%submitcmd) + output = run_cmd_no_fail(submitcmd, combine_output=True) + jobid = self.get_job_id(output) + logger.info("Submitted job id is %s"%jobid) + return jobid def get_batch_system_type(self): nodes = self.get_nodes("batch_system") diff --git a/scripts/lib/CIME/case.py b/scripts/lib/CIME/case.py index ca727570fa7..1426f5a2a1b 100644 --- a/scripts/lib/CIME/case.py +++ b/scripts/lib/CIME/case.py @@ -862,6 +862,7 @@ def _create_caseroot_tools(self): os.path.join(toolsdir, "lt_archive.sh"), os.path.join(toolsdir, "getTiming"), os.path.join(toolsdir, "save_provenance"), + os.path.join(toolsdir, "get_key_commands"), os.path.join(machines_dir,"Makefile"), os.path.join(machines_dir,"mkSrcfiles"), os.path.join(machines_dir,"mkDepends")] @@ -1062,9 +1063,9 @@ def create_clone(self, newcase, keepexe=False, mach_dir=None, project=None, cime return newcase - def submit_jobs(self, no_batch=False, job=None, batch_args=None): + def submit_jobs(self, no_batch=False, job=None, batch_args=None, dry_run=False): env_batch = self.get_env('batch') - return env_batch.submit_jobs(self, no_batch=no_batch, job=job, batch_args=batch_args) + return env_batch.submit_jobs(self, no_batch=no_batch, job=job, batch_args=batch_args, dry_run=dry_run) def get_mpirun_cmd(self, job="case.run"): env_mach_specific = self.get_env('mach_specific') diff --git a/scripts/lib/CIME/case_setup.py b/scripts/lib/CIME/case_setup.py index 94f3fccec87..f527665e17a 100644 --- a/scripts/lib/CIME/case_setup.py +++ b/scripts/lib/CIME/case_setup.py @@ -165,10 +165,7 @@ def _case_setup_impl(case, caseroot, clean=False, test_mode=False, reset=False, input_batch_script = os.path.join(case.get_value("MACHDIR"), env_batch.get_value('template', subgroup=job)) if job == "case.test" and testcase is not None and not test_mode: logger.info("Writing %s script" % job) - testscript = os.path.join(cimeroot, "scripts", "Testing", "Testcases", "%s_script" % testcase) - # Short term fix to be removed when csh tests are removed - if not os.path.exists(testscript): - env_batch.make_batch_script(input_batch_script, job, case, pestot, tasks_per_node, num_nodes, thread_count) + env_batch.make_batch_script(input_batch_script, job, case, pestot, tasks_per_node, num_nodes, thread_count) elif job != "case.test": logger.info("Writing %s script from input template %s" % (job, input_batch_script)) env_batch.make_batch_script(input_batch_script, job, case, pestot, tasks_per_node, num_nodes, thread_count) @@ -203,14 +200,6 @@ def _case_setup_impl(case, caseroot, clean=False, test_mode=False, reset=False, logger.info("If an old case build already exists, might want to run \'case.build --clean\' before building") - # Create test script if appropriate - # Short term fix to be removed when csh tests are removed - if os.path.exists("env_test.xml"): - if not os.path.exists("case.test"): - logger.info("Starting testcase.setup") - run_cmd_no_fail("./testcase.setup -caseroot %s" % caseroot) - logger.info("Finished testcase.setup") - # Some tests need namelists created here (ERP) - so do this if are in test mode if test_mode or get_model() == "acme": logger.info("Generating component namelists as part of setup") From 50658485842e1e27c098029890fcfc926203a2ba Mon Sep 17 00:00:00 2001 From: James Foucar Date: Fri, 14 Apr 2017 15:24:18 -0600 Subject: [PATCH 2/7] Rename to preview_run --- scripts/Tools/{get_key_commands => preview_run} | 0 scripts/lib/CIME/case.py | 2 +- 2 files changed, 1 insertion(+), 1 deletion(-) rename scripts/Tools/{get_key_commands => preview_run} (100%) diff --git a/scripts/Tools/get_key_commands b/scripts/Tools/preview_run similarity index 100% rename from scripts/Tools/get_key_commands rename to scripts/Tools/preview_run diff --git a/scripts/lib/CIME/case.py b/scripts/lib/CIME/case.py index 1426f5a2a1b..0d9b39a3ddb 100644 --- a/scripts/lib/CIME/case.py +++ b/scripts/lib/CIME/case.py @@ -862,7 +862,7 @@ def _create_caseroot_tools(self): os.path.join(toolsdir, "lt_archive.sh"), os.path.join(toolsdir, "getTiming"), os.path.join(toolsdir, "save_provenance"), - os.path.join(toolsdir, "get_key_commands"), + os.path.join(toolsdir, "preview_run"), os.path.join(machines_dir,"Makefile"), os.path.join(machines_dir,"mkSrcfiles"), os.path.join(machines_dir,"mkDepends")] From e790212047c52a3ef1db1010d86345d0a3c9adbf Mon Sep 17 00:00:00 2001 From: James Foucar Date: Fri, 14 Apr 2017 16:10:46 -0600 Subject: [PATCH 3/7] Add mpirun and batch overrides --- scripts/lib/CIME/XML/env_batch.py | 6 ++- scripts/lib/CIME/case.py | 41 +++++++++++-------- .../mct/cime_config/config_component.xml | 11 ++++- 3 files changed, 37 insertions(+), 21 deletions(-) diff --git a/scripts/lib/CIME/XML/env_batch.py b/scripts/lib/CIME/XML/env_batch.py index 96e78a5df64..330e2382a22 100644 --- a/scripts/lib/CIME/XML/env_batch.py +++ b/scripts/lib/CIME/XML/env_batch.py @@ -363,7 +363,6 @@ def _submit_single_job(self, case, job, depid=None, no_batch=False, batch_args=N logger.info("Starting job script %s" % job) - # This is what we want longterm function_name = job.replace(".", "_") if not dry_run: function_name = job.replace(".", "_") @@ -372,6 +371,9 @@ def _submit_single_job(self, case, job, depid=None, no_batch=False, batch_args=N return submitargs = self.get_submit_args(case, job) + args_override = self.get_value("BATCH_COMMAND", subgroup=job) + if args_override: + submitargs = args_override if depid is not None: dep_string = self.get_value("depend_string", subgroup=None) @@ -455,7 +457,7 @@ def get_all_queues(self): def get_nodes(self, nodename, attributes=None, root=None, xpath=None): if nodename in ("JOB_WALLCLOCK_TIME", "PROJECT", "PROJECT_REQUIRED", - "JOB_QUEUE"): + "JOB_QUEUE", "BATCH_COMMAND"): nodes = EnvBase.get_nodes(self, "entry", attributes={"id":nodename}, root=root, xpath=xpath) else: diff --git a/scripts/lib/CIME/case.py b/scripts/lib/CIME/case.py index 0d9b39a3ddb..9b8000a6a1d 100644 --- a/scripts/lib/CIME/case.py +++ b/scripts/lib/CIME/case.py @@ -1074,28 +1074,33 @@ def get_mpirun_cmd(self, job="case.run"): run_misc_suffix = "" if run_misc_suffix is None else run_misc_suffix run_suffix = run_exe + run_misc_suffix - # Things that will have to be matched against mpirun element attributes - mpi_attribs = { - "compiler" : self.get_value("COMPILER"), - "mpilib" : self.get_value("MPILIB"), - "threaded" : self.get_build_threaded(), - "unit_testing" : False - } - - executable, args = env_mach_specific.get_mpirun(self, mpi_attribs, job=job) + mpirun_cmd_override = self.get_value("MPI_RUN_COMMAND") - # special case for aprun - if executable == "aprun": - aprun_cmd, num_nodes = get_aprun_cmd_for_case(self, run_exe) - expect(num_nodes == self.num_nodes, "Not using optimized num nodes") - return aprun_cmd + " " + run_misc_suffix + if mpirun_cmd_override not in ["", None, "UNSET"]: + return mpirun_cmd_override + " " + run_exe + " " + run_misc_suffix else: - mpi_arg_string = " ".join(args.values()) + # Things that will have to be matched against mpirun element attributes + mpi_attribs = { + "compiler" : self.get_value("COMPILER"), + "mpilib" : self.get_value("MPILIB"), + "threaded" : self.get_build_threaded(), + "unit_testing" : False + } + + executable, args = env_mach_specific.get_mpirun(self, mpi_attribs, job=job) + + # special case for aprun + if executable == "aprun": + aprun_cmd, num_nodes = get_aprun_cmd_for_case(self, run_exe) + expect(num_nodes == self.num_nodes, "Not using optimized num nodes") + return aprun_cmd + " " + run_misc_suffix + else: + mpi_arg_string = " ".join(args.values()) - if self.get_value("BATCH_SYSTEM") == "cobalt": - mpi_arg_string += " : " + if self.get_value("BATCH_SYSTEM") == "cobalt": + mpi_arg_string += " : " - return "%s %s %s" % (executable if executable is not None else "", mpi_arg_string, run_suffix) + return "%s %s %s" % (executable if executable is not None else "", mpi_arg_string, run_suffix) def set_model_version(self, model): version = "unknown" diff --git a/src/drivers/mct/cime_config/config_component.xml b/src/drivers/mct/cime_config/config_component.xml index 958edf5759c..186fe6e85a1 100644 --- a/src/drivers/mct/cime_config/config_component.xml +++ b/src/drivers/mct/cime_config/config_component.xml @@ -1899,7 +1899,7 @@ UNSET run_mpi env_run.xml - mpi run command + override the mpi run command, do not include model executable @@ -2602,6 +2602,15 @@ The machine wallclock setting. Default determined in config_machines.xml can be overwritten by testing + + char + + + job_submission + env_batch.xml + Override the batch submit command this job. Do not include executable or dependencies + + char From 7948d497a0b480ed6b9830517106da394f8b7a8f Mon Sep 17 00:00:00 2001 From: James Foucar Date: Fri, 14 Apr 2017 16:37:12 -0600 Subject: [PATCH 4/7] Improve error reporting --- scripts/lib/CIME/utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/lib/CIME/utils.py b/scripts/lib/CIME/utils.py index 9216c85ff14..e33db5f80d0 100644 --- a/scripts/lib/CIME/utils.py +++ b/scripts/lib/CIME/utils.py @@ -259,7 +259,7 @@ def run_cmd_no_fail(cmd, input_str=None, from_dir=None, verbose=None, if stat != 0: # If command produced no errput, put output in the exception since we # have nothing else to go on. - errput = output if errput == "" else errput + errput = output if not errput else errput expect(False, "Command: '%s' failed with error '%s'%s" % (cmd, errput, "" if from_dir is None else " from dir '%s'" % from_dir)) @@ -1172,7 +1172,7 @@ def run_and_log_case_status(func, phase, caseroot='.'): try: rv = func() except: - e = sys.exc_info()[0] + e = sys.exc_info()[1] append_case_status(phase, "error", msg=("\n%s" % e), caseroot=caseroot) raise else: From e7319b0376f7bdef629429bae1db7b49408223e3 Mon Sep 17 00:00:00 2001 From: James Foucar Date: Fri, 14 Apr 2017 16:44:15 -0600 Subject: [PATCH 5/7] Update documentation --- scripts/Tools/preview_run | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/Tools/preview_run b/scripts/Tools/preview_run index 32e67539846..1cc94320e20 100755 --- a/scripts/Tools/preview_run +++ b/scripts/Tools/preview_run @@ -4,10 +4,10 @@ Script to query key CIME shell commands (mpirun and batch submission). To force a certain mpirun command, use: -./xmlchange MPIRUN_OVERRIDE $your_cmd +./xmlchange MPI_RUN_COMMAND $your_cmd To force a certain qsub command, use: -./xmlchange SUBMIT_OVERRIDE $your_cmd +./xmlchange --subgroup=case.run BATCH_COMMAND $your_cmd """ from standard_script_setup import * From 845bf15b18f8c948cceb3f6270e2f9df59fa1157 Mon Sep 17 00:00:00 2001 From: James Foucar Date: Thu, 27 Apr 2017 13:44:12 -0600 Subject: [PATCH 6/7] Changes based on review --- scripts/Tools/preview_run | 12 +++++++++--- scripts/lib/CIME/XML/env_batch.py | 4 ++-- scripts/lib/CIME/case.py | 2 +- src/drivers/mct/cime_config/config_component.xml | 2 +- 4 files changed, 13 insertions(+), 7 deletions(-) diff --git a/scripts/Tools/preview_run b/scripts/Tools/preview_run index 1cc94320e20..eca424d052a 100755 --- a/scripts/Tools/preview_run +++ b/scripts/Tools/preview_run @@ -4,10 +4,16 @@ Script to query key CIME shell commands (mpirun and batch submission). To force a certain mpirun command, use: -./xmlchange MPI_RUN_COMMAND $your_cmd +./xmlchange MPI_RUN_COMMAND=$your_cmd + +Example: +./xmlchange MPI_RUN_COMMAND='mpiexec -np 16 --some-flag' To force a certain qsub command, use: -./xmlchange --subgroup=case.run BATCH_COMMAND $your_cmd +./xmlchange --subgroup=case.run BATCH_COMMAND_FLAGS=$your_flags + +Example: +./xmlchange --subgroup=case.run BATCH_COMMAND_FLAGS='--some-flag --other-flag' """ from standard_script_setup import * @@ -29,7 +35,7 @@ OR > %s """ % ((os.path.basename(args[0]), ) * 4), description=description, - formatter_class=argparse.ArgumentDefaultsHelpFormatter + formatter_class=argparse.RawTextHelpFormatter ) parser.add_argument("caseroot", nargs="?", default=os.getcwd(), diff --git a/scripts/lib/CIME/XML/env_batch.py b/scripts/lib/CIME/XML/env_batch.py index 330e2382a22..724c48143f9 100644 --- a/scripts/lib/CIME/XML/env_batch.py +++ b/scripts/lib/CIME/XML/env_batch.py @@ -371,7 +371,7 @@ def _submit_single_job(self, case, job, depid=None, no_batch=False, batch_args=N return submitargs = self.get_submit_args(case, job) - args_override = self.get_value("BATCH_COMMAND", subgroup=job) + args_override = self.get_value("BATCH_COMMAND_FLAGS", subgroup=job) if args_override: submitargs = args_override @@ -457,7 +457,7 @@ def get_all_queues(self): def get_nodes(self, nodename, attributes=None, root=None, xpath=None): if nodename in ("JOB_WALLCLOCK_TIME", "PROJECT", "PROJECT_REQUIRED", - "JOB_QUEUE", "BATCH_COMMAND"): + "JOB_QUEUE", "BATCH_COMMAND_FLAGS"): nodes = EnvBase.get_nodes(self, "entry", attributes={"id":nodename}, root=root, xpath=xpath) else: diff --git a/scripts/lib/CIME/case.py b/scripts/lib/CIME/case.py index 9b8000a6a1d..a6afa77e214 100644 --- a/scripts/lib/CIME/case.py +++ b/scripts/lib/CIME/case.py @@ -844,6 +844,7 @@ def _create_caseroot_tools(self): os.path.join(toolsdir, "case.submit"), os.path.join(toolsdir, "case.cmpgen_namelists"), os.path.join(toolsdir, "preview_namelists"), + os.path.join(toolsdir, "preview_run"), os.path.join(toolsdir, "check_input_data"), os.path.join(toolsdir, "check_case"), os.path.join(toolsdir, "archive_metadata.sh"), @@ -862,7 +863,6 @@ def _create_caseroot_tools(self): os.path.join(toolsdir, "lt_archive.sh"), os.path.join(toolsdir, "getTiming"), os.path.join(toolsdir, "save_provenance"), - os.path.join(toolsdir, "preview_run"), os.path.join(machines_dir,"Makefile"), os.path.join(machines_dir,"mkSrcfiles"), os.path.join(machines_dir,"mkDepends")] diff --git a/src/drivers/mct/cime_config/config_component.xml b/src/drivers/mct/cime_config/config_component.xml index 186fe6e85a1..48cf2c38d3d 100644 --- a/src/drivers/mct/cime_config/config_component.xml +++ b/src/drivers/mct/cime_config/config_component.xml @@ -2602,7 +2602,7 @@ The machine wallclock setting. Default determined in config_machines.xml can be overwritten by testing - + char From 78cf98cdcd864afcb066abca15246ad6682ab90f Mon Sep 17 00:00:00 2001 From: James Foucar Date: Thu, 27 Apr 2017 13:55:04 -0600 Subject: [PATCH 7/7] Fix merge problem --- scripts/lib/CIME/case.py | 34 ++++++++++++++-------------------- 1 file changed, 14 insertions(+), 20 deletions(-) diff --git a/scripts/lib/CIME/case.py b/scripts/lib/CIME/case.py index c94e2eb1d51..667801cbb30 100644 --- a/scripts/lib/CIME/case.py +++ b/scripts/lib/CIME/case.py @@ -1117,6 +1117,16 @@ def get_mpirun_cmd(self, job="case.run"): if mpirun_cmd_override not in ["", None, "UNSET"]: return mpirun_cmd_override + " " + run_exe + " " + run_misc_suffix + # Things that will have to be matched against mpirun element attributes + mpi_attribs = { + "compiler" : self.get_value("COMPILER"), + "mpilib" : self.get_value("MPILIB"), + "threaded" : self.get_build_threaded(), + "unit_testing" : False + } + + executable, args = env_mach_specific.get_mpirun(self, mpi_attribs, job=job) + # special case for aprun if executable is not None and "aprun" in executable: aprun_args, num_nodes = get_aprun_cmd_for_case(self, run_exe) @@ -1124,28 +1134,12 @@ def get_mpirun_cmd(self, job="case.run"): return executable + aprun_args + " " + run_misc_suffix else: - # Things that will have to be matched against mpirun element attributes - mpi_attribs = { - "compiler" : self.get_value("COMPILER"), - "mpilib" : self.get_value("MPILIB"), - "threaded" : self.get_build_threaded(), - "unit_testing" : False - } - - executable, args = env_mach_specific.get_mpirun(self, mpi_attribs, job=job) - - # special case for aprun - if executable == "aprun": - aprun_cmd, num_nodes = get_aprun_cmd_for_case(self, run_exe) - expect(num_nodes == self.num_nodes, "Not using optimized num nodes") - return aprun_cmd + " " + run_misc_suffix - else: - mpi_arg_string = " ".join(args.values()) + mpi_arg_string = " ".join(args.values()) - if self.get_value("BATCH_SYSTEM") == "cobalt": - mpi_arg_string += " : " + if self.get_value("BATCH_SYSTEM") == "cobalt": + mpi_arg_string += " : " - return "%s %s %s" % (executable if executable is not None else "", mpi_arg_string, run_suffix) + return "%s %s %s" % (executable if executable is not None else "", mpi_arg_string, run_suffix) def set_model_version(self, model): version = "unknown"