remote-install: add "ana/" to the default install list (#5137)

* The `ana/` directory is used by `rose_ana` to load comparison modules. * These are typically run where the data is generated which is often remote. * These directories typically contain a small number of Python files.
cylc · Sep 27, 2022 · c44563a · c44563a
1 parent 6d47966
commit c44563a
Show file tree

Hide file tree

Showing 5 changed files with 54 additions and 22 deletions.
diff --git a/CHANGES.md b/CHANGES.md
@@ -35,6 +35,10 @@ Maintenance release.
 
 ### Fixes
 
+
+[#5137](https://github.com/cylc/cylc-flow/pull/5137) -
+Install the `ana/` directory to remote platforms by default.
+
 [#5146](https://github.com/cylc/cylc-flow/pull/5146) - no-flow tasks should not
 retrigger incomplete children.
 

diff --git a/cylc/flow/cfgspec/workflow.py b/cylc/flow/cfgspec/workflow.py
@@ -222,10 +222,17 @@ def get_script_common_text(this: str, example: Optional[str] = None):
 
                The following directories already get installed by default:
 
-               * ``app/``
-               * ``bin/``
-               * ``etc/``
-               * ``lib/``
+               ``ana/``
+                  Rose ana analysis modules
+               ``app/``
+                  Rose applications
+               ``bin/``
+                  Cylc bin directory (added to ``PATH``)
+               ``etc/``
+                  Miscellaneous resources
+               ``lib/``
+                  Cylc lib directory (``lib/python`` added to ``PYTHONPATH``
+                  for workflow config)
 
                These should be located in the top level of your Cylc workflow,
                i.e. the directory that contains your ``flow.cylc`` file.

diff --git a/cylc/flow/remote.py b/cylc/flow/remote.py
@@ -172,6 +172,15 @@ def get_includes_to_rsync(rsync_includes=None):
     '--no-t'
 ]
 
+DEFAULT_INCLUDES = [
+    '/ana/***',  # Rose ana analysis modules
+    '/app/***',  # Rose applications
+    '/bin/***',  # Cylc bin directory (added to PATH)
+    '/etc/***',  # Miscellaneous resources
+    '/lib/***',  # Cylc lib directory (lib/python added to PYTHONPATH for
+                 # workflow config)
+]
+
 
 def construct_rsync_over_ssh_cmd(
     src_path: str, dst_path: str, platform: Dict[str, Any],
@@ -209,12 +218,7 @@ def construct_rsync_over_ssh_cmd(
     rsync_cmd.extend(rsync_options)
     for exclude in ['log', 'share', 'work']:
         rsync_cmd.append(f"--exclude={exclude}")
-    default_includes = [
-        '/app/***',
-        '/bin/***',
-        '/etc/***',
-        '/lib/***']
-    for include in default_includes:
+    for include in DEFAULT_INCLUDES:
         rsync_cmd.append(f"--include={include}")
     for include in get_includes_to_rsync(rsync_includes):
         rsync_cmd.append(f"--include={include}")

diff --git a/tests/functional/remote/01-file-install.t b/tests/functional/remote/01-file-install.t
@@ -23,7 +23,7 @@ set_test_number 6
 
 create_files () {
     # dump some files into the run dir
-    for DIR in "bin" "app" "etc" "lib" "dir1" "dir2"
+    for DIR in "bin" "ana" "app" "etc" "lib" "dir1" "dir2"
     do
         mkdir -p "${WORKFLOW_RUN_DIR}/${DIR}"
         touch "${WORKFLOW_RUN_DIR}/${DIR}/moo"
@@ -35,7 +35,7 @@ create_files () {
 }
 
 # Test configured files/directories along with default files/directories
-# (app, bin, etc, lib) are correctly installed on the remote platform.
+# (ana, app, bin, etc, lib) are correctly installed on the remote platform.
 TEST_NAME="${TEST_NAME_BASE}-default-paths"
 init_workflow "${TEST_NAME}" <<__FLOW_CONFIG__
 [scheduling]
@@ -59,8 +59,9 @@ workflow_run_ok "${TEST_NAME}-run1" cylc play "${WORKFLOW_NAME}" \
 # ensure these files get installed on the remote platform
 SSH="$(cylc config -d -i "[platforms][$CYLC_TEST_PLATFORM]ssh command")"
 ${SSH} "${CYLC_TEST_HOST}" \
-    find "${RUN_DIR_REL}/"{app,bin,etc,lib} -type f | sort > 'find.out'
+    find "${RUN_DIR_REL}/"{ana,app,bin,etc,lib} -type f | sort > 'find.out'
 cmp_ok 'find.out'  <<__OUT__
+${RUN_DIR_REL}/ana/moo
 ${RUN_DIR_REL}/app/moo
 ${RUN_DIR_REL}/bin/moo
 ${RUN_DIR_REL}/etc/moo
@@ -93,8 +94,9 @@ workflow_run_ok "${TEST_NAME}-run2" cylc play "${WORKFLOW_NAME}" \
     -s "CYLC_TEST_PLATFORM='${CYLC_TEST_PLATFORM}'"
 
 ${SSH} "${CYLC_TEST_HOST}" \
-    find "${RUN_DIR_REL}/"{app,bin,dir1,dir2,file1,file2,etc,lib} -type f | sort > 'find.out'
+    find "${RUN_DIR_REL}/"{ana,app,bin,dir1,dir2,file1,file2,etc,lib} -type f | sort > 'find.out'
 cmp_ok 'find.out'  <<__OUT__
+${RUN_DIR_REL}/ana/moo
 ${RUN_DIR_REL}/app/moo
 ${RUN_DIR_REL}/bin/moo
 ${RUN_DIR_REL}/dir1/moo

diff --git a/tests/unit/test_remote.py b/tests/unit/test_remote.py
@@ -63,11 +63,26 @@ def test_construct_rsync_over_ssh_cmd():
         }
     )
     assert host == 'miklegard'
-    assert ' '.join(cmd) == (
-        'rsync command --delete --rsh=strange_ssh --include=/.service/ '
-        '--include=/.service/server.key -a --checksum '
-        '--out-format=%o %n%L --no-t --exclude=log --exclude=share '
-        '--exclude=work --include=/app/*** --include=/bin/*** '
-        '--include=/etc/*** --include=/lib/*** --exclude=* '
-        '/foo/ miklegard:/bar/'
-    )
+    assert cmd == [
+        'rsync',
+        'command',
+        '--delete',
+        '--rsh=strange_ssh',
+        '--include=/.service/',
+        '--include=/.service/server.key',
+        '-a',
+        '--checksum',
+        '--out-format=%o %n%L',
+        '--no-t',
+        '--exclude=log',
+        '--exclude=share',
+        '--exclude=work',
+        '--include=/ana/***',
+        '--include=/app/***',
+        '--include=/bin/***',
+        '--include=/etc/***',
+        '--include=/lib/***',
+        '--exclude=*',
+        '/foo/',
+        'miklegard:/bar/',
+    ]