Skip to content

Commit

Permalink
Allow setting of default batch system directives in the platform config
Browse files Browse the repository at this point in the history
  • Loading branch information
wxtim committed May 24, 2022
1 parent 28ed330 commit d7d7e51
Show file tree
Hide file tree
Showing 3 changed files with 105 additions and 1 deletion.
11 changes: 11 additions & 0 deletions cylc/flow/cfgspec/globalcfg.py
Original file line number Diff line number Diff line change
Expand Up @@ -1308,6 +1308,17 @@
be appropriate if following the pattern
``hosts = main, backup, failsafe``.
''')
with Conf('directives', desc='''
Job runner (batch scheduler) directives.
Default directives for :cylc:conf:
`[runtime][<namespace>][directives]`
'''):
Conf('<directive>', VDR.V_STRING, desc='''
Example directives for the built-in job runner handlers
are shown in :ref:`AvailableMethods`.
''')

with Conf('localhost', meta=Platform, desc='''
A default platform defining settings for jobs to be run on the
same host as the workflow scheduler.
Expand Down
25 changes: 24 additions & 1 deletion cylc/flow/task_job_mgr.py
Original file line number Diff line number Diff line change
Expand Up @@ -1278,6 +1278,27 @@ def _prep_submit_task_job_impl(self, workflow, itask, rtconfig):
job_d=job_d
)

@staticmethod
def default_dict_merger(defaults, tdef):
"""Use directive from platform if not set in rtconfig.
For item in defaults add item to output if not in tdef.
Examples:
>>> defaults = {'use': 1, 'override': 1}
>>> tdef = {'override': 2, 'nodefault': 1}
>>> result = TaskJobManager.default_dict_merger(defaults, tdef)
>>> {i: result[i] for i in sorted(result)}
{'nodefault': 1, 'override': 2, 'use': 1}
"""
output = {}
for key in set(list(defaults.keys()) + list(tdef.keys())):
if key in defaults and key not in tdef:
output[key] = defaults[key]
else:
output[key] = tdef[key]
return output

def get_job_conf(
self,
workflow,
Expand All @@ -1299,7 +1320,9 @@ def get_job_conf(
itask.platform['job runner command template']
),
'dependencies': itask.state.get_resolved_dependencies(),
'directives': rtconfig['directives'],
'directives': self.default_dict_merger(
itask.platform['directives'], rtconfig['directives']
),
'environment': rtconfig['environment'],
'execution_time_limit': itask.summary[self.KEY_EXECUTE_TIME_LIMIT],
'env-script': rtconfig['env-script'],
Expand Down
70 changes: 70 additions & 0 deletions tests/functional/platforms/09-default-directives.t
Original file line number Diff line number Diff line change
@@ -0,0 +1,70 @@
#!/usr/bin/env bash
# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE.
# Copyright (C) NIWA & British Crown (Met Office) & Contributors.
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#-------------------------------------------------------------------------------
# Test validating platforms in workflow.
REQUIRE_PLATFORM="runner:slurm"
. "$(dirname "$0")/test_header"

set_test_number 5

create_test_global_config "" "
[platforms]
[[no_default, default_only, overridden, neither]]
hosts = localhost
job runner = slurm
[[default_only, overridden]]
[[[directives]]]
--wurble=foo
"

init_workflow "${TEST_NAME_BASE}" <<'__FLOW_CONF__'
[scheduler]
[[events]]
stall timeout = PT0S
[scheduling]
[[graph]]
R1 = lewis & morse & barnaby & cadfael
[runtime]
[[lewis]]
platform = no_default
[[[directives]]]
--wurble=bar
[[morse]]
platform = default_only
[[barnaby]]
platform = overridden
[[[directives]]]
--wurble=qux
[[cadfael]]
platform = neither
__FLOW_CONF__

run_fail "${TEST_NAME_BASE}-play" \
cylc play ${WORKFLOW_NAME} --no-detach

LOG_DIR="${RUN_DIR}/${WORKFLOW_NAME}/log/job/1/"

named_grep_ok "${TEST_NAME_BASE}-no-default" "--wurble=bar" "${LOG_DIR}/lewis/NN/job"
named_grep_ok "${TEST_NAME_BASE}-default-only" "--wurble=foo" "${LOG_DIR}/morse/NN/job"
named_grep_ok "${TEST_NAME_BASE}-overridden" "--wurble=qux" "${LOG_DIR}/barnaby/NN/job"
grep_fail "--wurble" "${LOG_DIR}/cadfael/NN/job"

purge
exit 0

0 comments on commit d7d7e51

Please sign in to comment.