Merge branch 'expand_platforms_section' into warn.global-config-conta…

…ins-localhost

* expand_platforms_section:
  glblcfg: split comma separated platform definitions
  tests/u: parsec/test_util unittest->pytest
  bump deprecation warnings from Cylc 9 to 8.x (cylc#4853)
  Update tests/functional/execution-time-limit/04-polling-intervals.t
  Stop a few deprecation warnings in Cylc7 back compat mode
  Add tests
  Add polling interval test and change log entry.
  Set excecution polling delays correctly

CHANGES.md

@@ -49,6 +49,10 @@ workflow source argument), and rename the `--flow-name` option to
 
 ### Fixes
 
+[#4844](https://github.com/cylc/cylc-flow/pull/4844) - Fixes bug where
+execution polling intervals used in combination with an execution time limit
+resulted in incorrect polling intervals.
+
 [#4829](https://github.com/cylc/cylc-flow/pull/4829) -
 Suppress deprecated configuration warnings in Cylc 7 compatibility mode.
 

cylc/flow/cfgspec/globalcfg.py

@@ -33,7 +33,7 @@
 )
 from cylc.flow.parsec.exceptions import ParsecError, ItemNotFoundError
 from cylc.flow.parsec.upgrade import upgrader
-from cylc.flow.parsec.util import printcfg
+from cylc.flow.parsec.util import printcfg, expand_many_section
 from cylc.flow.parsec.validate import (
     CylcConfigValidator as VDR,
     DurationFloat,
@@ -1493,6 +1493,7 @@ def load(self):
 
         self._set_default_editors()
         self._no_platform_group_name_overlap()
+        self._expand_platforms()
 
     def _set_default_editors(self):
         # default to $[G]EDITOR unless an editor is defined in the config
@@ -1521,6 +1522,17 @@ def _no_platform_group_name_overlap(self):
                     msg += f'\n * {name}'
                 raise GlobalConfigError(msg)
 
+    def _expand_platforms(self):
+        """Expand comma separated platform names.
+
+        E.G. turn [platforms][foo, bar] into [platforms][foo] and
+        platforms[bar].
+        """
+        if self.sparse.get('platforms'):
+            self.sparse['platforms'] = expand_many_section(
+                self.sparse['platforms']
+            )
+
     def platform_dump(
         self,
         print_platform_names: bool = True,

cylc/flow/cfgspec/workflow.py

@@ -51,7 +51,8 @@
 
    Deprecated section kept for compatibility with Cylc 7 workflow definitions.
 
-   **It will not be available at Cylc 9**.
+
+   This will be removed in a future version of Cylc 8.
 
    Use :cylc:conf:`flow.cylc[runtime][<namespace>]platform` instead.
 '''
@@ -63,8 +64,8 @@
 
    This config item has been moved to a platform setting in the
    :cylc:conf:`global.cylc[platforms]` section. It will be used by the
-   automated platform upgrade mechanism at Cylc 8, and deprecated
-   at Cylc 9.
+   automated platform upgrade mechanism and remove in a future version
+   of Cylc 8.
 
    Ideally, as a user this should be set by your site admins
    and you will only need to pick a suitable
@@ -1628,7 +1629,7 @@ def get_script_common_text(this: str, example: Optional[str] = None):
 
                 This was done to allow users to control the order of
                 definition of the variables. This section will be removed
-                in Cylc 9.
+                in a future version of Cylc 8.
 
                 For the time being, the contents of this section will be
                 prepended to the ``[environment]`` section when running
@@ -1846,13 +1847,11 @@ def upg(cfg, descr):
         ):
             if job_setting in cfg['runtime'][task]:
                 LOG.warning(
-                    f"* (8.0.0) '[runtime][{task}]{job_setting}' - this "
-                    "setting is deprecated; use "
-                    "'global.cylc[platforms][<platform name>]"
-                    f"{job_setting}' "
-                    "instead. Currently, this item will override"
-                    " the corresponding item in global.cylc, "
-                    "but support for this will be removed in Cylc 9."
+                    f"* (8.0.0) '[runtime][{task}]{job_setting}' - this"
+                    " setting is deprecated; use"
+                    f" 'global.cylc[platforms][<platform name>]{job_setting}'"
+                    " instead. Currently, this item will override"
+                    " the corresponding item in global.cylc."
                 )
 
 
@@ -1903,15 +1902,17 @@ def upgrade_param_env_templates(cfg, descr):
         for task_name, task_items in cfg['runtime'].items():
             if 'parameter environment templates' not in task_items:
                 continue
-            if first_warn is True:
+            if not cylc.flow.flags.cylc7_back_compat:
+                if first_warn:
+                    LOG.warning(
+                        'deprecated items automatically upgraded in '
+                        f'"{descr}":'
+                    )
+                    first_warn = False
                 LOG.warning(
-                    f'deprecated items automatically upgraded in "{descr}":'
+                    f' * (8.0.0) {dep % task_name} contents prepended to '
+                    f'{new % task_name}'
                 )
-                first_warn = False
-            LOG.warning(
-                f' * (8.0.0) {dep % task_name} contents prepended to '
-                f'{new % task_name}'
-            )
             for key, val in reversed(
                     task_items['parameter environment templates'].items()):
                 if 'environment' in task_items:
@@ -1941,19 +1942,19 @@ def warn_about_depr_platform(cfg):
             fail_if_platform_and_host_conflict(task_cfg, task_name)
             # Fail if backticks subshell e.g. platform = `foo`:
             is_platform_definition_subshell(task_cfg['platform'])
-        else:
+        elif not cylc.flow.flags.cylc7_back_compat:
             depr = get_platform_deprecated_settings(task_cfg, task_name)
             if depr:
                 msg = "\n".join(depr)
                 LOG.warning(
-                    f'Task {task_name}: deprecated "host" and "batch system" '
-                    f'will be removed at Cylc 9 - upgrade to platform:\n{msg}'
+                    f'Task {task_name}: deprecated "host" and "batch system"'
+                    f' use "platform".\n{msg}'
                 )
 
 
 def warn_about_depr_event_handler_tmpl(cfg):
     """Warn if deprecated template strings appear in event handlers."""
-    if 'runtime' not in cfg:
+    if 'runtime' not in cfg or cylc.flow.flags.cylc7_back_compat:
         return
     deprecation_msg = (
         'The event handler template variable "%({0})s" is deprecated - '

cylc/flow/config.py

@@ -2286,7 +2286,7 @@ def process_metadata_urls(self):
                 # to:
                 #     Cylc8
                 # remove at:
-                #     Cylc9
+                #     Cylc8.x
                 self.cfg['meta']['URL'] = url % {
                     # cylc 7
                     'suite_name': self.workflow,
@@ -2319,7 +2319,7 @@ def process_metadata_urls(self):
                 # to:
                 #     Cylc8
                 # remove at:
-                #     Cylc9
+                #     Cylc8.x
                 try:
                     cfg['meta']['URL'] = cfg['meta']['URL'] % {
                         # cylc 7

cylc/flow/etc/job.sh

@@ -103,14 +103,14 @@ cylc__job__main() {
         export CYLC_WORKFLOW_HOST CYLC_WORKFLOW_OWNER
         # BACK COMPAT: DEPRECATED environment variables
         # remove at:
-        #     Cylc9
+        #     Cylc8.x
         export CYLC_SUITE_HOST="${CYLC_WORKFLOW_HOST}"
         export CYLC_SUITE_OWNER="${CYLC_WORKFLOW_OWNER}"
     fi
 
     # BACK COMPAT: DEPRECATED environment variables
     # remove at:
-    #     Cylc9
+    #     Cylc8.x
     export CYLC_SUITE_SHARE_DIR="${CYLC_WORKFLOW_SHARE_DIR}"
     export CYLC_SUITE_SHARE_PATH="${CYLC_WORKFLOW_SHARE_DIR}"
     export CYLC_SUITE_NAME="${CYLC_WORKFLOW_ID}"

cylc/flow/parsec/fileparse.py

@@ -165,7 +165,7 @@ def addict(cfig, key, val, parents, index):
             # to:
             #     Cylc8
             # remove at:
-            #     Cylc9
+            #     Cylc8.x
             parents[0:2] == ['scheduling', 'dependencies']
         ):
             # append the new graph string to the existing one

cylc/flow/parsec/util.py

@@ -20,6 +20,7 @@
 """
 
 from copy import copy
+import re
 import sys
 
 from cylc.flow.parsec.OrderedDict import OrderedDictWithDefaults
@@ -355,3 +356,65 @@ def itemstr(parents=None, item=None, value=None):
         text = str(value)
 
     return text
+
+
+# pattern for picking out comma separated items which does not split commas
+# inside of quotes
+SECTION_EXPAND_PATTERN = re.compile(
+    r'''
+        (?:
+          [^,"']+
+          |
+          "[^"]*"
+          |
+          '[^']*'
+        )+
+    ''',
+    re.X
+)
+
+
+def dequote(string: str, chars='"\'') -> str:
+    """Simple approach to strip quotes from strings.
+
+    Examples:
+        >>> dequote('"foo"')
+        'foo'
+        >>> dequote("'foo'")
+        'foo'
+        >>> dequote('a"b"c')
+        'a"b"c'
+
+    """
+    if len(string) < 2:
+        return string
+    for char in chars:
+        if string[0] == char and string[-1] == char:
+            return string[1:-1]
+    return string
+
+
+def expand_many_section(config):
+    """Expand comma separated entries.
+
+    Intended for use in __MANY__ sections i.e. ones in which headings are
+    user defined.
+
+    Returns the expanded config i.e. does not modify it in place (this is
+    necessary to preserve definition order).
+    """
+    ret = {}
+    for section_name, section in config.items():
+        expanded_names = [
+            dequote(name.strip()).strip()
+            for name in SECTION_EXPAND_PATTERN.findall(section_name)
+        ]
+        for name in expanded_names:
+            if name in ret:
+                # already defined -> merge
+                replicate(ret[name], section)
+
+            else:
+                ret[name] = {}
+                replicate(ret[name], section)
+    return ret

cylc/flow/platforms.py

@@ -92,13 +92,13 @@ def get_platform(
 
 
 # BACK COMPAT: get_platform
-#     At Cylc 9 remove all Cylc7 upgrade logic.
+#     At Cylc 8.x remove all Cylc7 upgrade logic.
 # from:
 #     Cylc8
 # to:
-#     Cylc9
+#     Cylc8.x
 # remove at:
-#     Cylc9
+#     Cylc8.x
 def get_platform(
     task_conf: Union[str, Dict[str, Any], None] = None,
     task_id: str = UNKNOWN_TASK,

cylc/flow/task_events_mgr.py

@@ -1458,7 +1458,7 @@ def _get_handler_template_variables(
             # from:
             #     Cylc 8
             # remove at:
-            #     Cylc 9
+            #     Cylc 8.x
             EventData.Suite.value:  # deprecated
                 quote(self.workflow),
             EventData.SuiteUUID.value:  # deprecated
@@ -1471,7 +1471,7 @@ def _get_handler_template_variables(
             # from:
             #     Cylc < 8
             # remove at:
-            #     Cylc9 - pending announcement of deprecation
+            #     Cylc8.x - pending announcement of deprecation
             # next 2 (JobID_old, JobRunnerName_old) are deprecated
             EventData.JobID_old.value:
                 quote(str(itask.summary['submit_method_id'])),
@@ -1511,13 +1511,15 @@ def _reset_job_timers(self, itask):
                 if not time_limit_delays:
                     time_limit_delays = [60, 120, 420]
                 timeout = time_limit + sum(time_limit_delays)
-                # Remove excessive polling before time limit
-                while sum(delays) > time_limit:
-                    del delays[-1]
-                # But fill up the gap before time limit
-                if delays:
-                    size = int((time_limit - sum(delays)) / delays[-1])
-                    delays.extend([delays[-1]] * size)
+                if sum(delays) > time_limit:
+                    # Remove excessive polling before time limit
+                    while sum(delays) > time_limit:
+                        del delays[-1]
+                else:
+                    # Fill up the gap before time limit
+                    if delays:
+                        size = int((time_limit - sum(delays)) / delays[-1])
+                        delays.extend([delays[-1]] * size)
                 time_limit_delays[0] += time_limit - sum(delays)
                 delays += time_limit_delays
         else:  # if itask.state.status == TASK_STATUS_SUBMITTED:

cylc/flow/task_job_mgr.py

@@ -1110,7 +1110,7 @@ def _prep_submit_task_job(self, workflow, itask, check_syntax=True):
         # - Platform exists, host doesn't = eval platform_name
         # - host exists - eval host_n
         # remove at:
-        #     Cylc9
+        #     Cylc8.x
         if (
             rtconfig['platform'] is not None and
             rtconfig['remote']['host'] is not None

cylc/flow/task_remote_mgr.py

@@ -126,7 +126,7 @@ def subshell_eval(self, command, command_pattern, host_check=True):
         """
         # BACK COMPAT: references to "host"
         # remove at:
-        #     Cylc9
+        #     Cylc8.x
         if not command:
             return 'localhost'
 
@@ -159,7 +159,7 @@ def subshell_eval(self, command, command_pattern, host_check=True):
         # Environment variable substitution
         command = os.path.expandvars(command)
         # Remote?
-        # TODO - Remove at Cylc 9 as this only makes sense with host logic
+        # TODO - Remove at Cylc 8.x as this only makes sense with host logic
         if host_check is True:
             if is_remote_host(command):
                 return command

cylc/flow/workflow_events.py

@@ -86,7 +86,7 @@ class EventData(Enum):
     # from:
     #     Cylc 8
     # remove at:
-    #     Cylc 9
+    #     Cylc 8.x
     Suite = 'suite'
     """The workflow ID
 
@@ -101,7 +101,7 @@ class EventData(Enum):
     # from:
     #     Cylc 8
     # remove at:
-    #     Cylc 9
+    #     Cylc 8.x
     Suite_UUID = 'suite_uuid'
     """The unique identification string for this workflow run.
 
@@ -116,7 +116,7 @@ class EventData(Enum):
     # from:
     #     Cylc 8
     # remove at:
-    #     Cylc 9
+    #     Cylc 8.x
     SuiteURL = 'suite_url'
     """The URL defined in :cylc:conf:`flow.cylc[meta]URL`.
 
@@ -158,7 +158,7 @@ def get_template_variables(
         # from:
         #     Cylc 8
         # remove at:
-        #     Cylc 9
+        #     Cylc 8.x
         EventData.Suite.value:
             schd.workflow,
         EventData.Suite_UUID.value: