diff --git a/CHANGES.md b/CHANGES.md index 77987787..9abd384f 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -6,6 +6,14 @@ creating a new release entry be sure to copy & paste the span tag with the updated. Only the first match gets replaced, so it's fine to leave the old ones in. --> +## __cylc-rose-1.4.0 (Upcoming)__ + +### Features + +[#269](https://github.com/cylc/cylc-rose/pull/269) - Allow environment variables +set in ``rose-suite.conf`` to be used when parsing ``global.cylc``. + + ## __cylc-rose-1.3.2 (Released 2024-01-18)__ [#284](https://github.com/cylc/cylc-rose/pull/284) - Allow use of Metomi-Rose 2.2.*. diff --git a/cylc/rose/__init__.py b/cylc/rose/__init__.py index 35eb03b4..b0e203b6 100644 --- a/cylc/rose/__init__.py +++ b/cylc/rose/__init__.py @@ -70,11 +70,37 @@ for ease of porting Cylc 7 workflows. +The ``global.cylc`` file +^^^^^^^^^^^^^^^^^^^^^^^^ + +The Cylc Rose Plugin forces the reloading of the ``global.cylc`` file +to allow environment variables set by Rose to change the global configuration. + +For example you could use ``CYLC_SYMLINKS`` as a variable to control +the behaviour of ``cylc install``: + +.. code-block:: cylc + + #!jinja2 + # part of a global.cylc file + [install] + [[symlink dirs]] + [[[hpc]]] + {% if environ["CYLC_SYMLINKS"] | default("x") == "A" %} + run = $LOCATION_A + {% elif environ["CYLC_SYMLINKS"] | default("x") == "B" %} + run = $LOCATION_B + {% else %} + run = $LOCATION_C + {% endif %} + + + Special Variables ----------------- The Cylc Rose plugin provides two environment/template variables -to the Cylc scheduler: +to the Cylc scheduler. ``ROSE_ORIG_HOST`` Cylc commands (such as ``cylc install``, ``cylc validate`` and @@ -111,6 +137,7 @@ ``CYLC_VERSION`` will be removed from your configuration by the Cylc-Rose plugin, as it is now set by Cylc. + Additional CLI options ---------------------- You can use command line options to set or override diff --git a/cylc/rose/utilities.py b/cylc/rose/utilities.py index e23f6e54..515356a5 100644 --- a/cylc/rose/utilities.py +++ b/cylc/rose/utilities.py @@ -27,6 +27,7 @@ from cylc.flow import LOG from cylc.flow.exceptions import CylcError from cylc.flow.flags import cylc7_back_compat +from cylc.flow.cfgspec.glbl_cfg import glbl_cfg from cylc.flow.hostuserutil import get_host from metomi.isodatetime.datetimeoper import DateTimeOperator from metomi.rose import __version__ as ROSE_VERSION @@ -869,6 +870,13 @@ def export_environment(environment: Dict[str, str]) -> None: for key, val in environment.items(): os.environ[key] = val + # If env vars have been set we want to force reload + # the global config so that the value of this vars + # can be used by Jinja2 in the global config. + # https://github.com/cylc/cylc-rose/issues/237 + if environment: + glbl_cfg().load() + def record_cylc_install_options( srcdir: Path, diff --git a/tests/conftest.py b/tests/conftest.py index 624d7c6a..5905240a 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -15,9 +15,11 @@ # along with this program. If not, see . from types import SimpleNamespace +from uuid import uuid4 from cylc.flow import __version__ as CYLC_VERSION from cylc.flow.option_parsers import Options +from cylc.flow.pathutil import get_workflow_run_dir from cylc.flow.scripts.install import get_option_parser as install_gop from cylc.flow.scripts.install import install_cli as cylc_install from cylc.flow.scripts.reinstall import get_option_parser as reinstall_gop @@ -27,6 +29,16 @@ import pytest +@pytest.fixture() +def workflow_name(): + return 'cylc-rose-test-' + str(uuid4())[:8] + + +@pytest.fixture(scope='module') +def mod_workflow_name(): + return 'cylc-rose-test-' + str(uuid4())[:8] + + @pytest.fixture(scope='module') def mod_capsys(request): from _pytest.capture import SysCapture @@ -108,7 +120,7 @@ def _inner(srcpath, args=None): return _inner -def _cylc_install_cli(capsys, caplog): +def _cylc_install_cli(capsys, caplog, workflow_name): """Access the install CLI""" def _inner(srcpath, args=None): """Install a workflow. @@ -119,9 +131,13 @@ def _inner(srcpath, args=None): """ options = Options(install_gop(), args)() output = SimpleNamespace() + if not options.workflow_name: + options.workflow_name = workflow_name + if not args or not args.get('no_run_name', ''): + options.no_run_name = True try: - cylc_install(options, str(srcpath)) + output.name, output.id = cylc_install(options, str(srcpath)) output.ret = 0 output.exc = '' except Exception as exc: @@ -129,6 +145,7 @@ def _inner(srcpath, args=None): output.exc = exc output.logging = '\n'.join([i.message for i in caplog.records]) output.out, output.err = capsys.readouterr() + output.run_dir = get_workflow_run_dir(output.id) return output return _inner @@ -159,13 +176,14 @@ def _inner(workflow_id, opts=None): @pytest.fixture -def cylc_install_cli(capsys, caplog): - return _cylc_install_cli(capsys, caplog) +def cylc_install_cli(capsys, caplog, workflow_name): + return _cylc_install_cli(capsys, caplog, workflow_name) @pytest.fixture(scope='module') -def mod_cylc_install_cli(mod_capsys, mod_caplog): - return _cylc_install_cli(mod_capsys, mod_caplog) +def mod_cylc_install_cli(mod_capsys, mod_caplog, mod_workflow_name): + return _cylc_install_cli( + mod_capsys, mod_caplog, mod_workflow_name) @pytest.fixture diff --git a/tests/functional/test_ROSE_ORIG_HOST.py b/tests/functional/test_ROSE_ORIG_HOST.py index 73d6fa1a..d97e221b 100644 --- a/tests/functional/test_ROSE_ORIG_HOST.py +++ b/tests/functional/test_ROSE_ORIG_HOST.py @@ -119,7 +119,7 @@ def fixture_install_flow( ) install_conf_path = ( fixture_provide_flow['flowpath'] / - 'runN/opt/rose-suite-cylc-install.conf' + 'opt/rose-suite-cylc-install.conf' ) text = install_conf_path.read_text() text = re.sub('ROSE_ORIG_HOST=.*', 'ROSE_ORIG_HOST=foo', text) @@ -142,7 +142,7 @@ def test_cylc_validate_srcdir(fixture_install_flow, mod_cylc_validate_cli): def test_cylc_validate_rundir(fixture_install_flow, mod_cylc_validate_cli): """Sanity check that workflow validates: """ - flowpath = fixture_install_flow['flowpath'] / 'runN' + flowpath = fixture_install_flow['flowpath'] result = mod_cylc_validate_cli(flowpath) assert 'ROSE_ORIG_HOST (env) is:' in result.logging diff --git a/tests/functional/test_reinstall.py b/tests/functional/test_reinstall.py index 65002f9a..32abc432 100644 --- a/tests/functional/test_reinstall.py +++ b/tests/functional/test_reinstall.py @@ -118,14 +118,14 @@ def test_cylc_install_run(fixture_install_flow): 'file_, expect', [ ( - 'run1/rose-suite.conf', ( + 'rose-suite.conf', ( '# Config Options \'b c (cylc-install)\' from CLI appended to' ' options already in `rose-suite.conf`.\n' 'opts=a b c (cylc-install)\n' ) ), ( - 'run1/opt/rose-suite-cylc-install.conf', ( + 'opt/rose-suite-cylc-install.conf', ( '# This file records CLI Options.\n\n' '!opts=b c\n' f'\n[env]\n#{ROHIOS}\nROSE_ORIG_HOST={HOST}\n' @@ -172,14 +172,14 @@ def test_cylc_reinstall_run(fixture_reinstall_flow): 'file_, expect', [ ( - 'run1/rose-suite.conf', ( + 'rose-suite.conf', ( '# Config Options \'b c d (cylc-install)\' from CLI appended ' 'to options already in `rose-suite.conf`.\n' 'opts=a b c d (cylc-install)\n' ) ), ( - 'run1/opt/rose-suite-cylc-install.conf', ( + 'opt/rose-suite-cylc-install.conf', ( '# This file records CLI Options.\n\n' '!opts=b c d\n' f'\n[env]\n#{ROHIOS}\nROSE_ORIG_HOST={HOST}\n' @@ -230,14 +230,14 @@ def test_cylc_reinstall_run2(fixture_reinstall_flow2): 'file_, expect', [ ( - 'run1/rose-suite.conf', ( + 'rose-suite.conf', ( '# Config Options \'b c d (cylc-install)\' from CLI appended ' 'to options already in `rose-suite.conf`.\n' 'opts=z b c d (cylc-install)\n' ) ), ( - 'run1/opt/rose-suite-cylc-install.conf', ( + 'opt/rose-suite-cylc-install.conf', ( '# This file records CLI Options.\n\n' '!opts=b c d\n' f'\n[env]\n#{ROHIOS}\nROSE_ORIG_HOST={HOST}\n' diff --git a/tests/functional/test_reinstall_clean.py b/tests/functional/test_reinstall_clean.py index 749778c4..0a207430 100644 --- a/tests/functional/test_reinstall_clean.py +++ b/tests/functional/test_reinstall_clean.py @@ -109,7 +109,7 @@ def test_cylc_install_run(fixture_install_flow): 'file_, expect', [ ( - 'run1/opt/rose-suite-cylc-install.conf', ( + 'opt/rose-suite-cylc-install.conf', ( '# This file records CLI Options.\n\n' '!opts=bar\n\n' '[env]\n' @@ -163,7 +163,7 @@ def test_cylc_reinstall_run(fixture_reinstall_flow): 'file_, expect', [ ( - 'run1/opt/rose-suite-cylc-install.conf', ( + 'opt/rose-suite-cylc-install.conf', ( '# This file records CLI Options.\n\n' '!opts=baz\n\n' '[env]\n' diff --git a/tests/functional/test_rose_fileinstall.py b/tests/functional/test_rose_fileinstall.py index d0bca3b3..ac99b3da 100644 --- a/tests/functional/test_rose_fileinstall.py +++ b/tests/functional/test_rose_fileinstall.py @@ -63,7 +63,7 @@ def fixture_install_flow(fixture_provide_flow, request, cylc_install_cli): yield srcpath, datapath, flow_name, result, destpath if not request.session.testsfailed: - shutil.rmtree(destpath) + shutil.rmtree(destpath, ignore_errors=True) def test_rose_fileinstall_validate(fixture_provide_flow, cylc_validate_cli): @@ -84,7 +84,7 @@ def test_rose_fileinstall_subfolders(fixture_install_flow): """File installed into a sub directory: """ _, datapath, _, _, destpath = fixture_install_flow - assert ((destpath / 'runN/lib/python/lion.py').read_text() == + assert ((destpath / 'lib/python/lion.py').read_text() == (datapath / 'lion.py').read_text()) @@ -92,7 +92,7 @@ def test_rose_fileinstall_concatenation(fixture_install_flow): """Multiple files concatenated on install(source contained wildcard): """ _, datapath, _, _, destpath = fixture_install_flow - assert ((destpath / 'runN/data').read_text() == + assert ((destpath / 'data').read_text() == ((datapath / 'randoms1.data').read_text() + (datapath / 'randoms3.data').read_text() )) diff --git a/tests/functional/test_utils.py b/tests/functional/test_utils.py index f2b9ef07..3642fda9 100644 --- a/tests/functional/test_utils.py +++ b/tests/functional/test_utils.py @@ -36,3 +36,94 @@ def test_basic(tmp_path): assert Path(tmp_path / 'src/rose-suite.conf').read_text() == ( Path(tmp_path / 'dest/rose-suite.conf').read_text() ) + + +def test_global_config_environment_validate( + monkeypatch, tmp_path, cylc_validate_cli +): + """It should reload the global config after exporting env variables. + + See: https://github.com/cylc/cylc-rose/issues/237 + """ + # Setup global config: + global_conf = """#!jinja2 + {% from "cylc.flow" import LOG %} + {% set cylc_symlinks = environ.get('CYLC_SYMLINKS', None) %} + {% do LOG.critical(cylc_symlinks) %} + """ + conf_path = tmp_path / 'conf' + conf_path.mkdir() + monkeypatch.setenv('CYLC_CONF_PATH', conf_path) + + # Setup workflow config: + (conf_path / 'global.cylc').write_text(global_conf) + (tmp_path / 'rose-suite.conf').write_text( + '[env]\nCYLC_SYMLINKS="Foo"\n') + (tmp_path / 'flow.cylc').write_text(""" + [scheduling] + initial cycle point = now + [[graph]] + R1 = x + [runtime] + [[x]] + """) + + # Validate the config: + output = cylc_validate_cli(tmp_path) + assert output.ret == 0 + + # CYLC_SYMLINKS == None the first time the global.cylc + # is loaded and "Foo" the second time. + assert output.logging.split('\n')[-1] == '"Foo"' + + +def test_global_config_environment_validate2( + monkeypatch, tmp_path, cylc_install_cli +): + """It should reload the global config after exporting env variables. + + See: https://github.com/cylc/cylc-rose/issues/237 + """ + # Setup global config: + global_conf = ( + '#!jinja2\n' + '[install]\n' + ' [[symlink dirs]]\n' + ' [[[localhost]]]\n' + '{% set cylc_symlinks = environ.get(\'CYLC_SYMLINKS\', None) %}\n' + '{% if cylc_symlinks == "foo" %}\n' + f'log = {str(tmp_path)}/foo\n' + '{% else %}\n' + f'log = {str(tmp_path)}/bar\n' + '{% endif %}\n' + ) + glbl_conf_path = tmp_path / 'conf' + glbl_conf_path.mkdir() + (glbl_conf_path / 'global.cylc').write_text(global_conf) + monkeypatch.setenv('CYLC_CONF_PATH', glbl_conf_path) + + # Setup workflow config: + (tmp_path / 'rose-suite.conf').write_text( + '[env]\nCYLC_SYMLINKS=foo\n') + (tmp_path / 'flow.cylc').write_text(""" + [scheduling] + initial cycle point = now + [[graph]] + R1 = x + [runtime] + [[x]] + """) + + # Install the config: + output = cylc_install_cli(tmp_path) + import sys + for i in output.logging.split('\n'): + print(i, file=sys.stderr) + assert output.ret == 0 + + # Assert symlink created back to test_path/foo: + expected_msg = ( + f'Symlink created: {output.run_dir}/log -> ' + f'{tmp_path}/foo/cylc-run/{output.id}/log' + ) + assert expected_msg in output.logging.split('\n')[0]