diff --git a/cylc/flow/cfgspec/globalcfg.py b/cylc/flow/cfgspec/globalcfg.py index 650b8285a13..d7cdbd1bc17 100644 --- a/cylc/flow/cfgspec/globalcfg.py +++ b/cylc/flow/cfgspec/globalcfg.py @@ -33,7 +33,7 @@ ) from cylc.flow.parsec.exceptions import ParsecError, ItemNotFoundError from cylc.flow.parsec.upgrade import upgrader -from cylc.flow.parsec.util import printcfg +from cylc.flow.parsec.util import printcfg, expand_many_section from cylc.flow.parsec.validate import ( CylcConfigValidator as VDR, DurationFloat, @@ -1493,6 +1493,7 @@ def load(self): self._set_default_editors() self._no_platform_group_name_overlap() + self._expand_platforms() def _set_default_editors(self): # default to $[G]EDITOR unless an editor is defined in the config @@ -1521,6 +1522,17 @@ def _no_platform_group_name_overlap(self): msg += f'\n * {name}' raise GlobalConfigError(msg) + def _expand_platforms(self): + """Expand comma separated platform names. + + E.G. turn [platforms][foo, bar] into [platforms][foo] and + platforms[bar]. + """ + if self.sparse.get('platforms'): + self.sparse['platforms'] = expand_many_section( + self.sparse['platforms'] + ) + def platform_dump( self, print_platform_names: bool = True, diff --git a/cylc/flow/parsec/util.py b/cylc/flow/parsec/util.py index 05976819ed7..49778f79f06 100644 --- a/cylc/flow/parsec/util.py +++ b/cylc/flow/parsec/util.py @@ -20,6 +20,7 @@ """ from copy import copy +import re import sys from cylc.flow.parsec.OrderedDict import OrderedDictWithDefaults @@ -355,3 +356,65 @@ def itemstr(parents=None, item=None, value=None): text = str(value) return text + + +# pattern for picking out comma separated items which does not split commas +# inside of quotes +SECTION_EXPAND_PATTERN = re.compile( + r''' + (?: + [^,"']+ + | + "[^"]*" + | + '[^']*' + )+ + ''', + re.X +) + + +def dequote(string: str, chars='"\'') -> str: + """Simple approach to strip quotes from strings. + + Examples: + >>> dequote('"foo"') + 'foo' + >>> dequote("'foo'") + 'foo' + >>> dequote('a"b"c') + 'a"b"c' + + """ + if len(string) < 2: + return string + for char in chars: + if string[0] == char and string[-1] == char: + return string[1:-1] + return string + + +def expand_many_section(config): + """Expand comma separated entries. + + Intended for use in __MANY__ sections i.e. ones in which headings are + user defined. + + Returns the expanded config i.e. does not modify it in place (this is + necessary to preserve definition order). + """ + ret = {} + for section_name, section in config.items(): + expanded_names = [ + dequote(name.strip()).strip() + for name in SECTION_EXPAND_PATTERN.findall(section_name) + ] + for name in expanded_names: + if name in ret: + # already defined -> merge + replicate(ret[name], section) + + else: + ret[name] = {} + replicate(ret[name], section) + return ret diff --git a/tests/unit/cfgspec/test_globalcfg.py b/tests/unit/cfgspec/test_globalcfg.py index c0bed7d7af6..35832bd7e39 100644 --- a/tests/unit/cfgspec/test_globalcfg.py +++ b/tests/unit/cfgspec/test_globalcfg.py @@ -58,3 +58,40 @@ def test_dump_platform_details(capsys, fake_global_conf): '[platform groups]\n [[BAR]]\n platforms = mario, sonic\n' ) assert expected == out + + +def test_expand_platforms(tmp_path): + """It should expand comma separated platform definitions.""" + glblcfg = GlobalConfig(SPEC) + (tmp_path / 'global.cylc').write_text(''' + [platforms] + [[foo]] + [[[meta]]] + x = 1 + [["bar"]] # double quoted name + [[[meta]]] + x = 2 + [[baz, bar, pub]] # baz before bar to test order is handled correctly + [[[meta]]] + x = 3 + [['pub']] # single quoted name + [[[meta]]] + x = 4 + ''') + glblcfg.loadcfg(tmp_path / 'global.cylc') + glblcfg._expand_platforms() + + # ensure the definition order is preserved + assert glblcfg.get(['platforms']).keys() == [ + 'localhost', + 'foo', + 'bar', + 'baz', + 'pub', + ] + + # ensure sections are correctly deep-merged + assert glblcfg.get(['platforms', 'foo', 'meta', 'x']) == '1' + assert glblcfg.get(['platforms', 'bar', 'meta', 'x']) == '3' + assert glblcfg.get(['platforms', 'baz', 'meta', 'x']) == '3' + assert glblcfg.get(['platforms', 'pub', 'meta', 'x']) == '4' diff --git a/tests/unit/parsec/test_util.py b/tests/unit/parsec/test_util.py index b9a34f11d9f..b02fddab23b 100644 --- a/tests/unit/parsec/test_util.py +++ b/tests/unit/parsec/test_util.py @@ -20,8 +20,16 @@ from cylc.flow.parsec.OrderedDict import OrderedDictWithDefaults from cylc.flow.parsec.util import ( - itemstr, listjoin, m_override, pdeepcopy, poverride, printcfg, - replicate, un_many + SECTION_EXPAND_PATTERN, + expand_many_section, + itemstr, + listjoin, + m_override, + pdeepcopy, + poverride, + printcfg, + replicate, + un_many, ) @@ -417,3 +425,95 @@ def test_itemstr_no_parents(): def test_itemstr_no_parents_no_value(): text = itemstr(parents=None, item="Value", value=None) assert text == 'Value' + + +# --- expand_many_section + +@pytest.mark.parametrize( + 'in_,out', + [ + # basically a fancy version of string.split(',') + ('foo', ['foo']), + ('foo,bar', ['foo', 'bar']), + ('foo, bar', ['foo', ' bar']), # doesn't remove whitespace + # except that it doesn't split quoted things + ('"foo", "bar"', ['"foo"', ' "bar"']), + ('"foo,", "b,ar"', ['"foo,"', ' "b,ar"']), # doesn't split in " quotes + ("'foo', 'bar'", ["'foo'", " 'bar'"]), + ("'foo,', 'b,ar'", ["'foo,'", " 'b,ar'"]), # doesn"t split in ' quotes + ] +) +def test_SECTION_EXPAND_PATTERN(in_, out): + """It should split sections which contain commas. + + This is used in order to expand [foo, bar] into [foo] and [bar]. + """ + assert SECTION_EXPAND_PATTERN.findall(in_) == out + + +@pytest.mark.parametrize( + 'in_,out', + [ + ('foo,bar', ['foo', 'bar']), + ('foo , bar', ['foo', 'bar']), + ('"foo", "bar"', ['foo', 'bar']), + ('"foo,", "b,ar"', ['foo,', 'b,ar']), + ] +) +def test_expand_many_section_expand(in_, out): + """It should expand sections which contain commas. + + E.G. it should expand [foo, bar] into [foo] and [bar]. + """ + config = {in_: {'whatever': True}} + assert list(expand_many_section(config)) == out + + +def test_expand_many_section_order(): + """It should maintain order when expanding sections.""" + assert list(expand_many_section({ + 'a': {}, + 'b, a': {}, + 'c, b, a, d': {}, + 'e': {}, + 'a, e': {}, + 'f, e': {}, + 'g, h': {}, + })) == ['a', 'b', 'c', 'd', 'e', 'f', 'g', 'h'] + + +def test_expand_many_section_merge(): + """It should merge sections together in definition order.""" + config = expand_many_section({ + 'b': {'x': 1}, + 'b, a, c, d': {'x': 2}, + 'c': {'x': 3}, + }) + assert config == { + 'b': {'x': 2}, + 'a': {'x': 2}, + 'c': {'x': 3}, + 'd': {'x': 2}, + } + # bonus marks: ensure all values coppied rather than referenced + config['a']['x'] = 4 + assert config['b']['x'] == 2 + + +def test_expand_many_section_merge_deep(): + """It should deep-merge nested sections - see replicate().""" + config = expand_many_section({ + 'b': {'x': {'y': 1}}, + 'b, a, c, d': {'x': {'y': 2}}, + 'c': {'x': {'y': 3}}, + }) + assert config == { + 'b': {'x': {'y': 2}}, + 'a': {'x': {'y': 2}}, + 'c': {'x': {'y': 3}}, + 'd': {'x': {'y': 2}}, + } + # bonus marks: ensure all values are unique objects + # (i.e. they have been coppied rather than referenced) + config['a']['x']['y'] = 4 + assert config['b']['x']['y'] == 2