From 6fc3ca3517cce37da8940d92ed7c76775bdeb18b Mon Sep 17 00:00:00 2001
From: Tim Pillinger <26465611+wxtim@users.noreply.github.com>
Date: Wed, 14 Dec 2022 09:39:37 +0000
Subject: [PATCH] add Cylc VRO

---
 cylc/flow/option_parsers.py                   |  25 ++-
 cylc/flow/pathutil.py                         |  29 ++++
 cylc/flow/scripts/config.py                   |   2 +-
 cylc/flow/scripts/cylc.py                     |   3 +-
 cylc/flow/scripts/graph.py                    |   2 +-
 cylc/flow/scripts/list.py                     |   2 +-
 cylc/flow/scripts/psutil.py                   |   2 +-
 cylc/flow/scripts/reinstall.py                |  23 ++-
 cylc/flow/scripts/reload.py                   |   4 +
 cylc/flow/scripts/validate.py                 |   2 +-
 cylc/flow/scripts/validate_install_play.py    |   9 +-
 .../flow/scripts/validate_reinstall_reload.py | 150 ++++++++++++++++++
 cylc/flow/scripts/view.py                     |   1 -
 cylc/flow/templatevars.py                     |  13 +-
 cylc/flow/workflow_files.py                   |  10 +-
 setup.cfg                                     |   1 +
 .../cylc-combination-scripts/01-vro-reload.t  |  53 +++++++
 .../cylc-combination-scripts/02-vro-restart.t |  54 +++++++
 .../cylc-combination-scripts/03-vro-resume.t  |  56 +++++++
 .../04-vro-fail-validate.t                    |  53 +++++++
 .../05-vro-fail-is-running.t                  |  58 +++++++
 .../vro_workflow/flow.cylc                    |   8 +
 tests/integration/conftest.py                 |   6 +
 tests/unit/scripts/test_validate.py           |   0
 tests/unit/test_get_old_tvars_units.py        |  65 ++++++++
 tests/unit/test_pathutil.py                   |  41 +++++
 26 files changed, 643 insertions(+), 29 deletions(-)
 create mode 100644 cylc/flow/scripts/validate_reinstall_reload.py
 create mode 100644 tests/functional/cylc-combination-scripts/01-vro-reload.t
 create mode 100644 tests/functional/cylc-combination-scripts/02-vro-restart.t
 create mode 100644 tests/functional/cylc-combination-scripts/03-vro-resume.t
 create mode 100644 tests/functional/cylc-combination-scripts/04-vro-fail-validate.t
 create mode 100644 tests/functional/cylc-combination-scripts/05-vro-fail-is-running.t
 create mode 100644 tests/functional/cylc-combination-scripts/vro_workflow/flow.cylc
 create mode 100644 tests/unit/scripts/test_validate.py
 create mode 100644 tests/unit/test_get_old_tvars_units.py

diff --git a/cylc/flow/option_parsers.py b/cylc/flow/option_parsers.py
index 3bb6b2e568a..542d9fbc9a9 100644
--- a/cylc/flow/option_parsers.py
+++ b/cylc/flow/option_parsers.py
@@ -80,9 +80,16 @@ def __init__(self, argslist, sources=None, useif=None, **kwargs):
             self.kwargs.update({kwarg: value})
 
     def __eq__(self, other):
-        """Args and Kwargs, but not other props equal."""
+        """Args and Kwargs, but not other props equal.
+
+        (Also make an exception for kwargs['help'] to allow lists of sources
+        prepended to 'help' to be passed through.)
+        """
         return (
-            self.kwargs == other.kwargs
+            (
+                {k: v for k, v in self.kwargs.items() if k != 'help'}
+                == {k: v for k, v in other.kwargs.items() if k != 'help'}
+            )
             and self.args == other.args
         )
 
@@ -729,9 +736,17 @@ def combine_options_pair(first_list, second_list):
                 and first & second
             ):
                 # if any of the args are different:
-                if first.args == second.args:
-                    # if none of the arg names are different.
-                    raise Exception(f'Clashing Options \n{first}\n{second}')
+
+                if (
+                    first.args == second.args
+                    # and (
+                    #     first.kwargs['help'].split('\x1b[0m ')[-1] !=
+                    #     second.kwargs['help'].split('\x1b[0m')[-1]
+                    # )
+                ):
+                    # if none of the arg names are different
+                    raise Exception(
+                        f'Clashing Options \n{first.args}\n{second.args}')
                 else:
                     first_args = first - second
                     second.args = second - first
diff --git a/cylc/flow/pathutil.py b/cylc/flow/pathutil.py
index 0d62ef37ad4..9d25cc4e757 100644
--- a/cylc/flow/pathutil.py
+++ b/cylc/flow/pathutil.py
@@ -456,3 +456,32 @@ def get_workflow_name_from_id(workflow_id: str) -> str:
         name_path = id_path
 
     return str(name_path.relative_to(cylc_run_dir))
+
+
+def get_source_conf_from_id(workflow_id):
+    """Give a workflow id, get the flow.cylc file of the source.
+
+    Additionally Check
+        1. Flow.cylc or suite.rc exists at source.
+    """
+    # Avoid circular import:
+    from cylc.flow.workflow_files import WorkflowFiles
+    # Get path of source:
+    run_dir = Path(get_workflow_run_dir(workflow_id))
+    relative_path = run_dir.relative_to(get_cylc_run_dir())
+    src = (
+        Path(get_cylc_run_dir())
+        / relative_path.parts[0]
+        / WorkflowFiles.Install.DIRNAME
+        / WorkflowFiles.Install.SOURCE
+    )
+
+    # Test whether there is a config file at source:
+    if (src / WorkflowFiles.FLOW_FILE).exists():
+        return src.resolve() / WorkflowFiles.FLOW_FILE
+    elif (src / WorkflowFiles.SUITE_RC).exists():
+        return src.resolve() / WorkflowFiles.SUITE_RC
+    else:
+        raise WorkflowFilesError(
+            f'Source file not present at: {src.resolve()}'
+        )
diff --git a/cylc/flow/scripts/config.py b/cylc/flow/scripts/config.py
index aa7144a934d..2b3b3d134e6 100755
--- a/cylc/flow/scripts/config.py
+++ b/cylc/flow/scripts/config.py
@@ -206,7 +206,7 @@ async def _main(
         workflow_id,
         flow_file,
         options,
-        get_template_vars(options)
+        get_template_vars(options, flow_file)
     )
 
     config.pcfg.idump(
diff --git a/cylc/flow/scripts/cylc.py b/cylc/flow/scripts/cylc.py
index 74a37119043..4c474e4dd6d 100644
--- a/cylc/flow/scripts/cylc.py
+++ b/cylc/flow/scripts/cylc.py
@@ -202,7 +202,8 @@ def get_version(long=False):
     'shutdown': 'stop',
     'task-message': 'message',
     'unhold': 'release',
-    'validate-install-play': 'vip'
+    'validate-install-play': 'vip',
+    'validate-reinstall-reload': 'vro',
 }
 
 
diff --git a/cylc/flow/scripts/graph.py b/cylc/flow/scripts/graph.py
index ce22cfdc2ed..fa24fb3e827 100644
--- a/cylc/flow/scripts/graph.py
+++ b/cylc/flow/scripts/graph.py
@@ -199,7 +199,7 @@ def _get_inheritance_nodes_and_edges(
 
 def get_config(workflow_id: str, opts: 'Values', flow_file) -> WorkflowConfig:
     """Return a WorkflowConfig object for the provided reg / path."""
-    template_vars = get_template_vars(opts)
+    template_vars = get_template_vars(opts, flow_file)
     return WorkflowConfig(
         workflow_id, flow_file, opts, template_vars=template_vars
     )
diff --git a/cylc/flow/scripts/list.py b/cylc/flow/scripts/list.py
index c0f1ed18ab7..5b715831dfe 100755
--- a/cylc/flow/scripts/list.py
+++ b/cylc/flow/scripts/list.py
@@ -116,7 +116,7 @@ async def _main(parser: COP, options: 'Values', workflow_id: str) -> None:
         src=True,
         constraint='workflows',
     )
-    template_vars = get_template_vars(options)
+    template_vars = get_template_vars(options, flow_file)
 
     if options.all_tasks and options.all_namespaces:
         parser.error("Choose either -a or -n")
diff --git a/cylc/flow/scripts/psutil.py b/cylc/flow/scripts/psutil.py
index 8747acc9a5a..47ff7ec1dbd 100644
--- a/cylc/flow/scripts/psutil.py
+++ b/cylc/flow/scripts/psutil.py
@@ -22,7 +22,7 @@
 the `psutil` on remote platforms.
 
 Exits:
-    0 - If successfull.
+    0 - If successful.
     2 - For errors in extracting results from psutil
     1 - For all other errors.
 """
diff --git a/cylc/flow/scripts/reinstall.py b/cylc/flow/scripts/reinstall.py
index c199c912508..65716b685ab 100644
--- a/cylc/flow/scripts/reinstall.py
+++ b/cylc/flow/scripts/reinstall.py
@@ -83,6 +83,7 @@
 from cylc.flow.id_cli import parse_id
 from cylc.flow.option_parsers import (
     CylcOptionParser as COP,
+    OptionSettings,
     Options,
     WORKFLOW_ID_ARG_DOC,
 )
@@ -100,6 +101,18 @@
 _input = input  # to enable testing
 
 
+REINSTALL_CYLC_ROSE_OPTIONS = [
+    OptionSettings(
+        ['--clear-rose-install-options'],
+        help="Clear options previously set by cylc-rose.",
+        action='store_true',
+        default=False,
+        dest="clear_rose_install_opts",
+        sources={'reinstall'}
+    )
+]
+
+
 def get_option_parser() -> COP:
     parser = COP(
         __doc__, comms=True, argdoc=[WORKFLOW_ID_ARG_DOC]
@@ -112,14 +125,8 @@ def get_option_parser() -> COP:
     except ImportError:
         pass
     else:
-        parser.add_option(
-            "--clear-rose-install-options",
-            help="Clear options previously set by cylc-rose.",
-            action='store_true',
-            default=False,
-            dest="clear_rose_install_opts"
-        )
-
+        for option in REINSTALL_CYLC_ROSE_OPTIONS:
+            parser.add_option(*option.args, **option.kwargs)
     return parser
 
 
diff --git a/cylc/flow/scripts/reload.py b/cylc/flow/scripts/reload.py
index 125e7d40fac..f399cc948cb 100755
--- a/cylc/flow/scripts/reload.py
+++ b/cylc/flow/scripts/reload.py
@@ -104,6 +104,10 @@ async def run(options: 'Values', workflow_id: str) -> None:
 
 @cli_function(get_option_parser)
 def main(parser: COP, options: 'Values', *ids) -> None:
+    reload_cli(options, *ids)
+
+
+def reload_cli(options: 'Values', *ids) -> None:
     call_multi(
         partial(run, options),
         *ids,
diff --git a/cylc/flow/scripts/validate.py b/cylc/flow/scripts/validate.py
index 0db83d1bdfa..2b47d073b39 100755
--- a/cylc/flow/scripts/validate.py
+++ b/cylc/flow/scripts/validate.py
@@ -155,7 +155,7 @@ async def wrapped_main(
         workflow_id,
         flow_file,
         options,
-        get_template_vars(options),
+        get_template_vars(options, flow_file),
         output_fname=options.output,
         mem_log_func=profiler.log_memory
     )
diff --git a/cylc/flow/scripts/validate_install_play.py b/cylc/flow/scripts/validate_install_play.py
index 25f6e805b5e..02be6039cd1 100644
--- a/cylc/flow/scripts/validate_install_play.py
+++ b/cylc/flow/scripts/validate_install_play.py
@@ -30,7 +30,7 @@
 
 from cylc.flow.scripts.validate import (
     VALIDATE_OPTIONS,
-    _main as validate_main
+    _main as cylc_validate
 )
 from cylc.flow.scripts.install import (
     INSTALL_OPTIONS, install_cli as cylc_install, get_source_location
@@ -75,10 +75,7 @@ def get_option_parser() -> COP:
         ]
     )
     for option in VIP_OPTIONS:
-        # Make a special exception for option against_source which makes
-        # no sense in a VIP context.
-        if option.kwargs.get('dest') != 'against_source':
-            parser.add_option(*option.args, **option.kwargs)
+        parser.add_option(*option.args, **option.kwargs)
     return parser
 
 
@@ -92,7 +89,7 @@ def main(parser: COP, options: 'Values', workflow_id: Optional[str] = None):
     source = get_source_location(workflow_id)
 
     log_subcommand('validate', source)
-    validate_main(parser, options, str(source))
+    cylc_validate(parser, options, str(source))
 
     log_subcommand('install', source)
     workflow_id = cylc_install(options, workflow_id)
diff --git a/cylc/flow/scripts/validate_reinstall_reload.py b/cylc/flow/scripts/validate_reinstall_reload.py
new file mode 100644
index 00000000000..b65a5e4da77
--- /dev/null
+++ b/cylc/flow/scripts/validate_reinstall_reload.py
@@ -0,0 +1,150 @@
+#!/usr/bin/env python3
+
+# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE.
+# Copyright (C) NIWA & British Crown (Met Office) & Contributors.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+"""cylc validate-reinstall-reload/play [OPTIONS] ARGS
+
+Validate and install a single workflow. Then if:
+- Workflow running => reload.
+- Workflow paused => resume.
+- Workflow stopped => play.
+
+This script is equivalent to:
+
+    $ cylc validate myworkflow --against-source     # See note 1
+    $ cylc reinstall myworkflow
+
+    $ if [[ my workflow is running ]];
+          cylc reload myworkflow
+      else
+          cylc play myworkflow
+
+Note 1:
+
+Cylc validate myworkflow --against-source is eqivelent of (It doesn't write
+any temporary files though):
+
+    # Install from run directory
+    $ cylc install ~/cylc-run/myworkflow -n temporary
+    # Install from source directory over the top
+    $ cylc install /path/to/myworkflow -n temporary
+    # Validate combined config
+    $ cylc validate ~/cylc-run/temporary
+"""
+
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from optparse import Values
+
+from cylc.flow.exceptions import ServiceFileError, CylcError
+from cylc.flow.scheduler_cli import PLAY_OPTIONS, scheduler_cli
+from cylc.flow.scripts.validate import (
+    VALIDATE_OPTIONS,
+    _main as cylc_validate
+)
+from cylc.flow.scripts.reinstall import (
+    REINSTALL_CYLC_ROSE_OPTIONS, reinstall_cli as cylc_reinstall
+)
+from cylc.flow.scripts.reload import (
+    reload_cli as cylc_reload
+)
+from cylc.flow.option_parsers import (
+    WORKFLOW_ID_ARG_DOC,
+    CylcOptionParser as COP,
+    combine_options,
+    log_subcommand,
+    cleanup_sysargv
+)
+from cylc.flow.terminal import cli_function
+from cylc.flow.workflow_files import detect_old_contact_file
+
+
+CYLC_ROSE_OPTIONS = COP.get_cylc_rose_options()
+VRO_OPTIONS = combine_options(
+    VALIDATE_OPTIONS,
+    REINSTALL_CYLC_ROSE_OPTIONS,
+    PLAY_OPTIONS,
+    CYLC_ROSE_OPTIONS,
+    modify={'cylc-rose': 'validate, install'}
+)
+
+
+def get_option_parser() -> COP:
+    parser = COP(
+        __doc__,
+        comms=True,
+        jset=True,
+        argdoc=[WORKFLOW_ID_ARG_DOC],
+    )
+    for option in VRO_OPTIONS:
+        parser.add_option(*option.args, **option.kwargs)
+    return parser
+
+
+@cli_function(get_option_parser)
+def main(parser: COP, options: 'Values', workflow_id: str):
+    vro_cli(parser, options, workflow_id)
+
+
+def vro_cli(parser: COP, options: 'Values', workflow_id: str):
+    """Run Cylc (re)validate - reinstall - reload in sequence."""
+    # Attempt to work out whether the workflow is running.
+    # We are trying to avoid reinstalling a subsequently being
+    # unable to play or reload because we cannot identify workflow state.
+    try:
+        detect_old_contact_file(workflow_id, quiet=True)
+    except ServiceFileError:
+        # Workflow is definately still running:
+        workflow_running = True
+    except CylcError as exc:
+        # We can't tell whether the workflow is running.
+        # TODO - consider a more helpful error
+        raise exc
+        exit(1)
+    else:
+        # Workflow is definately stopped:
+        workflow_running = False
+
+    # Force on the against_source option:
+    options.against_source = True   # Make validate check against source.
+    log_subcommand('validate --against-source', workflow_id)
+    cylc_validate(parser, options, workflow_id)
+
+    log_subcommand('reinstall', workflow_id)
+    cylc_reinstall(options, workflow_id)
+
+    # Run reload if workflow is running, else play:
+    if workflow_running:
+        log_subcommand('reload', workflow_id)
+        cylc_reload(options, workflow_id)
+
+    # run play anyway, to resume a paused workflow:
+    else:
+        cleanup_sysargv(
+            'play',
+            workflow_id,
+            options,
+            compound_script_opts=VRO_OPTIONS,
+            script_opts=(
+                PLAY_OPTIONS + CYLC_ROSE_OPTIONS
+                + parser.get_std_options()
+            ),
+            source='',  # Intentionally blank
+        )
+        log_subcommand('play', workflow_id)
+        scheduler_cli(options, workflow_id)
diff --git a/cylc/flow/scripts/view.py b/cylc/flow/scripts/view.py
index dc5abf6dafe..e52b0a578ac 100644
--- a/cylc/flow/scripts/view.py
+++ b/cylc/flow/scripts/view.py
@@ -109,7 +109,6 @@ def main(parser: COP, options: 'Values', workflow_id: str) -> None:
 
 async def _main(parser: COP, options: 'Values', workflow_id: str) -> None:
     workflow_id, _, flow_file = await parse_id_async(
-
         workflow_id,
         src=True,
         constraint='workflows',
diff --git a/cylc/flow/templatevars.py b/cylc/flow/templatevars.py
index f09e367d25a..2773264bbc3 100644
--- a/cylc/flow/templatevars.py
+++ b/cylc/flow/templatevars.py
@@ -82,9 +82,18 @@ def eval_var(var):
         ) from None
 
 
-def load_template_vars(template_vars=None, template_vars_file=None):
+def load_template_vars(
+    template_vars=None, template_vars_file=None, flow_file=None
+):
     """Load template variables from key=value strings."""
     res = {}
+    if flow_file is not None:
+        srcdir = str(Path(flow_file).parent)
+        db_tvars = OldTemplateVars(srcdir).template_vars
+        if db_tvars:
+            for key, val in db_tvars.items():
+                res[key] = val
+
     if template_vars_file:
         with open(template_vars_file, 'r') as handle:
             for line in handle:
@@ -101,7 +110,7 @@ def load_template_vars(template_vars=None, template_vars_file=None):
     return res
 
 
-def get_template_vars(options: Values) -> Dict[str, Any]:
+def get_template_vars(options: Values, flow_file) -> Dict[str, Any]:
     """Convienence wrapper for ``load_template_vars``.
 
     Args:
diff --git a/cylc/flow/workflow_files.py b/cylc/flow/workflow_files.py
index e96cf662521..cd40fc7a650 100644
--- a/cylc/flow/workflow_files.py
+++ b/cylc/flow/workflow_files.py
@@ -463,7 +463,9 @@ def _is_process_running(
     return cli_format(process['cmdline']) == command
 
 
-def detect_old_contact_file(reg: str, contact_data=None) -> None:
+def detect_old_contact_file(
+    reg: str, contact_data=None, quiet=False
+) -> None:
     """Check if the workflow process is still running.
 
     As a side-effect this should detect and rectify the situation
@@ -479,6 +481,10 @@ def detect_old_contact_file(reg: str, contact_data=None) -> None:
 
     Args:
         reg: workflow name
+        contact_date:
+        quiet: Controls whether to return already running message -
+            this is not required if Cylc VRO is using this function to
+            decide whether to resume or reload.
 
     Raises:
         CylcError:
@@ -514,6 +520,8 @@ def detect_old_contact_file(reg: str, contact_data=None) -> None:
     fname = get_contact_file_path(reg)
     if process_is_running:
         # ... the process is running, raise an exception
+        if quiet:
+            raise ServiceFileError()
         raise ServiceFileError(
             CONTACT_FILE_EXISTS_MSG % {
                 "host": old_host,
diff --git a/setup.cfg b/setup.cfg
index f6ee306837f..4acd9a3b0ba 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -200,6 +200,7 @@ cylc.command =
     validate = cylc.flow.scripts.validate:main
     view = cylc.flow.scripts.view:main
     vip = cylc.flow.scripts.validate_install_play:main
+    vro = cylc.flow.scripts.validate_reinstall_reload:main
 # async functions to run within the scheduler main loop
 cylc.main_loop =
     health_check = cylc.flow.main_loop.health_check
diff --git a/tests/functional/cylc-combination-scripts/01-vro-reload.t b/tests/functional/cylc-combination-scripts/01-vro-reload.t
new file mode 100644
index 00000000000..38fcba2c27c
--- /dev/null
+++ b/tests/functional/cylc-combination-scripts/01-vro-reload.t
@@ -0,0 +1,53 @@
+#!/usr/bin/env bash
+# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE.
+# Copyright (C) NIWA & British Crown (Met Office) & Contributors.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+#------------------------------------------------------------------------------
+# Test `cylc vro` (Validate Reinstall relOad)
+# In this case the target workflow is running so cylc reload is run.
+
+. "$(dirname "$0")/test_header"
+set_test_number 6
+
+
+# Setup (Must be a running workflow, note the unusual absence of --no-detach)
+WORKFLOW_NAME="cylctb-x$(< /dev/urandom tr -dc _A-Z-a-z-0-9 | head -c6)"
+cp "${TEST_SOURCE_DIR}/vro_workflow/flow.cylc" .
+run_ok "setup (vip)" \
+    cylc vip --debug \
+    --workflow-name "${WORKFLOW_NAME}" \
+    --no-run-name
+export WORKFLOW_RUN_DIR="${RUN_DIR}/${WORKFLOW_NAME}"
+poll_workflow_running
+
+
+# It validates and reloads:
+
+# Add allow implicit tasks our flow.cylc so that reload will validate:
+sed -i 's@P1Y@P5Y@' flow.cylc
+run_ok "${TEST_NAME_BASE}-runs" cylc vro "${WORKFLOW_NAME}"
+
+# Grep for VRO reporting revalidation, reinstallation and reloading
+grep "\$" "${TEST_NAME_BASE}-runs.stdout" > VIPOUT.txt
+named_grep_ok "${TEST_NAME_BASE}-it-revalidated" "$ cylc validate --against-source" "VIPOUT.txt"
+named_grep_ok "${TEST_NAME_BASE}-it-installed" "$ cylc reinstall" "VIPOUT.txt"
+named_grep_ok "${TEST_NAME_BASE}-it-reloaded" "$ cylc reload" "VIPOUT.txt"
+
+
+# Clean Up.
+run_ok "teardown (stop workflow)" cylc stop "${WORKFLOW_NAME}" --now --now
+purge
+exit 0
diff --git a/tests/functional/cylc-combination-scripts/02-vro-restart.t b/tests/functional/cylc-combination-scripts/02-vro-restart.t
new file mode 100644
index 00000000000..66b537cbe73
--- /dev/null
+++ b/tests/functional/cylc-combination-scripts/02-vro-restart.t
@@ -0,0 +1,54 @@
+#!/usr/bin/env bash
+# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE.
+# Copyright (C) NIWA & British Crown (Met Office) & Contributors.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+#------------------------------------------------------------------------------
+# Test `cylc vro` (Validate Reinstall restart)
+# In this case the target workflow is stopped so cylc play is run.
+
+
+. "$(dirname "$0")/test_header"
+set_test_number 6
+
+# Setup
+WORKFLOW_NAME="cylctb-x$(< /dev/urandom tr -dc _A-Z-a-z-0-9 | head -c6)"
+cp "${TEST_SOURCE_DIR}/vro_workflow/flow.cylc" .
+run_ok "setup (vip)" \
+    cylc vip --debug \
+    --workflow-name "${WORKFLOW_NAME}" \
+    --no-run-name \
+# Get the workflow into a stopped state
+cylc stop --now --now "${WORKFLOW_NAME}"
+export WORKFLOW_RUN_DIR="${RUN_DIR}/${WORKFLOW_NAME}"
+poll_workflow_stopped
+
+# It validates and restarts:
+
+# Change source workflow and run vro:
+sed -i 's@P1Y@P5Y@' flow.cylc
+run_ok "${TEST_NAME_BASE}-runs" cylc vro "${WORKFLOW_NAME}"
+
+# Grep for VRO reporting revalidation, reinstallation and playing the workflow.
+grep "\$" "${TEST_NAME_BASE}-runs.stdout" > VIPOUT.txt
+named_grep_ok "${TEST_NAME_BASE}-it-revalidated" "$ cylc validate --against-source" "VIPOUT.txt"
+named_grep_ok "${TEST_NAME_BASE}-it-installed" "$ cylc reinstall" "VIPOUT.txt"
+named_grep_ok "${TEST_NAME_BASE}-it-played" "$ cylc play" "VIPOUT.txt"
+
+
+# Clean Up.
+run_ok "teardown (stop workflow)" cylc stop "${WORKFLOW_NAME}" --now --now
+purge
+exit 0
diff --git a/tests/functional/cylc-combination-scripts/03-vro-resume.t b/tests/functional/cylc-combination-scripts/03-vro-resume.t
new file mode 100644
index 00000000000..4a1eb0916d2
--- /dev/null
+++ b/tests/functional/cylc-combination-scripts/03-vro-resume.t
@@ -0,0 +1,56 @@
+#!/usr/bin/env bash
+# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE.
+# Copyright (C) NIWA & British Crown (Met Office) & Contributors.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+#------------------------------------------------------------------------------
+# Test `cylc vro` (Validate Reinstall restart)
+# In this case the target workflow is paused so cylc reload & cylc play are run.
+
+. "$(dirname "$0")/test_header"
+set_test_number 6
+
+# Setup
+WORKFLOW_NAME="cylctb-x$(< /dev/urandom tr -dc _A-Z-a-z-0-9 | head -c6)"
+cp "${TEST_SOURCE_DIR}/vro_workflow/flow.cylc" .
+run_ok "setup (vip)" \
+    cylc vip --debug \
+    --workflow-name "${WORKFLOW_NAME}" \
+    --no-run-name \
+# Get the workflow into a paused state
+cylc pause "${WORKFLOW_NAME}"
+
+while [[ ! -n $(cylc scan --name "${WORKFLOW_NAME}" --states=paused) ]]; do
+    sleep 1
+done
+
+
+# It validates, reloads and resumes:
+
+# Change source workflow and run vro:
+sed -i 's@P1Y@P5Y@' flow.cylc
+run_ok "${TEST_NAME_BASE}-runs" cylc vro "${WORKFLOW_NAME}"
+
+# Grep for reporting of revalidation, reinstallation, reloading and playing:
+grep "\$" "${TEST_NAME_BASE}-runs.stdout" > VIPOUT.txt
+named_grep_ok "${TEST_NAME_BASE}-it-revalidated" "$ cylc validate --against-source" "VIPOUT.txt"
+named_grep_ok "${TEST_NAME_BASE}-it-installed" "$ cylc reinstall" "VIPOUT.txt"
+named_grep_ok "${TEST_NAME_BASE}-it-reloaded" "$ cylc reload" "VIPOUT.txt"
+
+
+# Clean Up:
+run_ok "teardown (stop workflow)" cylc stop "${WORKFLOW_NAME}" --now --now
+purge
+exit 0
diff --git a/tests/functional/cylc-combination-scripts/04-vro-fail-validate.t b/tests/functional/cylc-combination-scripts/04-vro-fail-validate.t
new file mode 100644
index 00000000000..42fb459d913
--- /dev/null
+++ b/tests/functional/cylc-combination-scripts/04-vro-fail-validate.t
@@ -0,0 +1,53 @@
+#!/usr/bin/env bash
+# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE.
+# Copyright (C) NIWA & British Crown (Met Office) & Contributors.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+#------------------------------------------------------------------------------
+# Test `cylc vro` (Validate Reinstall restart)
+# Changes to the source cause VRO to bail on validation.
+
+. "$(dirname "$0")/test_header"
+set_test_number 5
+
+# Setup
+WORKFLOW_NAME="cylctb-x$(< /dev/urandom tr -dc _A-Z-a-z-0-9 | head -c6)"
+cp "${TEST_SOURCE_DIR}/vro_workflow/flow.cylc" .
+run_ok "setup (vip)" \
+    cylc vip --debug \
+    --workflow-name "${WORKFLOW_NAME}" \
+    --no-run-name \
+
+
+# Change source workflow and run vro:
+
+# Cut the runtime section out of the source flow.
+cat flow.cylc | head -n 5 > tmp
+cat tmp > flow.cylc
+
+TEST_NAME="${TEST_NAME_BASE}"
+run_fail "${TEST_NAME}" cylc vro "${WORKFLOW_NAME}"
+
+# Grep for reporting of revalidation, reinstallation, reloading and playing:
+named_grep_ok "${TEST_NAME_BASE}-it-tried" \
+    "$ cylc validate --against-source" "${TEST_NAME}.stdout"
+named_grep_ok "${TEST_NAME_BASE}-it-failed" \
+    "WorkflowConfigError" "${TEST_NAME}.stderr"
+
+
+# Clean Up:
+run_ok "teardown (stop workflow)" cylc stop "${WORKFLOW_NAME}" --now --now
+purge
+exit 0
diff --git a/tests/functional/cylc-combination-scripts/05-vro-fail-is-running.t b/tests/functional/cylc-combination-scripts/05-vro-fail-is-running.t
new file mode 100644
index 00000000000..12c40eb33cc
--- /dev/null
+++ b/tests/functional/cylc-combination-scripts/05-vro-fail-is-running.t
@@ -0,0 +1,58 @@
+#!/usr/bin/env bash
+# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE.
+# Copyright (C) NIWA & British Crown (Met Office) & Contributors.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+#------------------------------------------------------------------------------
+# Test `cylc vro` (Validate Reinstall restart)
+# In this case the target workflow is in an abiguous state: We cannot tell
+# Whether it's running, paused or stopped. Cylc VRO should validate before
+# reinstall:
+
+. "$(dirname "$0")/test_header"
+set_test_number 4
+
+create_test_global_config "" """
+[scheduler]
+    [[main loop]]
+        plugins = reset bad hosts
+"""
+
+# Setup
+WORKFLOW_NAME="cylctb-x$(< /dev/urandom tr -dc _A-Z-a-z-0-9 | head -c6)"
+cp "${TEST_SOURCE_DIR}/vro_workflow/flow.cylc" .
+run_ok "setup (vip)" \
+    cylc vip --debug \
+    --workflow-name "${WORKFLOW_NAME}" \
+    --no-run-name
+# Get the workflow into an unreachable state
+
+CONTACTFILE="${RUN_DIR}/${WORKFLOW_NAME}/.service/contact"
+
+sed -i 's@CYLC_WORKFLOW_HOST=.*@CYLC_WORKFLOW_HOST=elephantshrew@' "${CONTACTFILE}"
+
+
+# It can't figure out whether the workflow is running:
+
+# Change source workflow and run vro:
+run_fail "${TEST_NAME_BASE}-runs" cylc vro "${WORKFLOW_NAME}"
+
+grep_ok "on elephantshrew." "${TEST_NAME_BASE}-runs.stderr"
+
+# Clean Up:
+sed -i "s@CYLC_WORKFLOW_HOST=elephantshrew@CYLC_WORKFLOW_HOST=$HOSTNAME@" "${CONTACTFILE}"
+run_ok "teardown (stop workflow)" cylc stop "${WORKFLOW_NAME}" --now --now
+purge
+exit 0
diff --git a/tests/functional/cylc-combination-scripts/vro_workflow/flow.cylc b/tests/functional/cylc-combination-scripts/vro_workflow/flow.cylc
new file mode 100644
index 00000000000..bda3da45ff3
--- /dev/null
+++ b/tests/functional/cylc-combination-scripts/vro_workflow/flow.cylc
@@ -0,0 +1,8 @@
+[scheduling]
+    initial cycle point = 1500
+    [[graph]]
+        P1Y = foo
+
+[runtime]
+    [[foo]]
+        script = sleep 500
\ No newline at end of file
diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py
index 65fdab9713a..c2ace181ede 100644
--- a/tests/integration/conftest.py
+++ b/tests/integration/conftest.py
@@ -149,6 +149,12 @@ def flow(run_dir, test_dir):
     yield partial(_make_flow, run_dir, test_dir)
 
 
+@pytest.fixture
+def flow_src(tmp_path):
+    """A function for creating function-level flows."""
+    yield partial(_make_src_flow, tmp_path)
+
+
 @pytest.fixture(scope='module')
 def mod_scheduler():
     """Return a Scheduler object for a flow.
diff --git a/tests/unit/scripts/test_validate.py b/tests/unit/scripts/test_validate.py
new file mode 100644
index 00000000000..e69de29bb2d
diff --git a/tests/unit/test_get_old_tvars_units.py b/tests/unit/test_get_old_tvars_units.py
new file mode 100644
index 00000000000..29d6cc85b8d
--- /dev/null
+++ b/tests/unit/test_get_old_tvars_units.py
@@ -0,0 +1,65 @@
+# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE.
+# Copyright (C) NIWA & British Crown (Met Office) & Contributors.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+
+from cylc.flow.templatevars import OldTemplateVars
+import sqlite3
+import pytest
+
+
+@pytest.fixture(scope='module')
+def _setup_db(tmp_path_factory):
+    tmp_path = tmp_path_factory.mktemp('test_get_old_tvars')
+    logfolder = tmp_path / "log/"
+    logfolder.mkdir()
+    db_path = logfolder / 'db'
+    conn = sqlite3.connect(db_path)
+    conn.execute(
+        r'''
+            CREATE TABLE workflow_template_vars (
+                key,
+                value
+            )
+        '''
+    )
+    conn.execute(
+        r'''
+            INSERT INTO workflow_template_vars
+            VALUES
+                ("FOO", "42"),
+                ("BAR", "'hello world'"),
+                ("BAZ", "'foo', 'bar', 48"),
+                ("QUX", "['foo', 'bar', 21]")
+        '''
+    )
+    conn.commit()
+    conn.close()
+    yield OldTemplateVars(tmp_path)
+
+
+@pytest.mark.parametrize(
+    'key, expect',
+    (
+        ('FOO', 42),
+        ('BAR', 'hello world'),
+        ('BAZ', ('foo', 'bar', 48)),
+        ('QUX', ['foo', 'bar', 21])
+    )
+)
+def test_OldTemplateVars(key, expect, _setup_db):
+    """It can extract a variety of items from a workflow database.
+    """
+    assert _setup_db.template_vars[key] == expect
diff --git a/tests/unit/test_pathutil.py b/tests/unit/test_pathutil.py
index d42b26a2dbe..14bf06c46d6 100644
--- a/tests/unit/test_pathutil.py
+++ b/tests/unit/test_pathutil.py
@@ -31,6 +31,7 @@
     get_next_rundir_number,
     get_remote_workflow_run_dir,
     get_remote_workflow_run_job_dir,
+    get_source_conf_from_id,
     get_workflow_run_dir,
     get_workflow_run_job_dir,
     get_workflow_run_scheduler_log_dir,
@@ -576,3 +577,43 @@ def test_get_workflow_name_from_id(
 
     result = get_workflow_name_from_id(id_)
     assert result == name
+
+
+@pytest.fixture
+def _setup_get_source_conf_from_id(tmp_path, monkeypatch):
+    run = tmp_path / 'cylc-run/run'
+    src = tmp_path / 'cylc-src/src'
+    src.mkdir(parents=True)
+    (run / '_cylc-install').mkdir(parents=True)
+    (run / '_cylc-install/source').symlink_to(src)
+    monkeypatch.setattr(
+        'cylc.flow.pathutil.get_workflow_run_dir',
+        lambda workflow_id: tmp_path / 'cylc-run/run'
+    )
+    monkeypatch.setattr(
+        'cylc.flow.pathutil.get_cylc_run_dir',
+        lambda: tmp_path / 'cylc-run'
+    )
+    yield tmp_path
+
+
+@pytest.mark.parametrize(
+    'conf_file',
+    (
+        param('flow.cylc', id='flow.cylc'),
+        param('suite.rc', id='flow.cylc'),
+        param(None, id='no file'),
+    )
+)
+def test_get_source_conf_from_id(
+    _setup_get_source_conf_from_id, conf_file
+):
+    """It locates a flow.cylc, suite.rc from a run dir, or fails nicely.
+    """
+    if conf_file:
+        expect = _setup_get_source_conf_from_id / f'cylc-src/src/{conf_file}'
+        expect.touch()
+        assert get_source_conf_from_id('run') == expect
+    else:
+        with pytest.raises(WorkflowFilesError):
+            get_source_conf_from_id('run')