cat-log: view retrieved remote logs locally.

bin/cylc-cat-log

@@ -19,18 +19,22 @@
 """cylc [info] cat-log|log [OPTIONS] ARGS
 
 Print, edit, or tail-follow content, or print path or list directory, of local
-or remote task job logs and suite server logs. Batch-system-specific job cat or
-tail commands are used if defined in global config.
+or remote task job logs and suite server logs. Batch-system view commands (e.g.
+'qcat') are used if defined in global config and the job is running.
 
 For standard log types use the short-cut option argument or full filename (e.g.
 for job stdout "-f o" or "-f job.out" will do).
 
 To list the local job log directory of a remote task, choose "-m l" (directory
 list mode) and a local file, e.g. "-f a" (job-activity.log).
 
-Custom job logs are assumed to reside on the job host. These will be available
-to view from the GUI if listed in 'extra log files' in the suite definition.
-Here the file name must be given (but it can be discovered with list-dir mode).
+If remote job logs are retrieved to the suite host on completion (global config
+'[JOB-HOST]retrieve job logs = True') and the job is not currently running, the
+local (retrieved) log will be accessed unless '-o/--force-remote' is used.
+
+Custom job logs (written to $CYLC_TASK_LOG_ROOT on the job host) are available
+from the GUI if listed in 'extra log files' in the suite definition. The file
+name must be given here, but can be discovered with '--mode=l' (list-dir).
 
 The correct cycle point format of the suite must be for task job logs.
 
@@ -195,6 +199,12 @@ def get_option_parser():
         "next oldest, etc.",
         metavar="INT", action="store", dest="rotation_num")
 
+    parser.add_option(
+        "-o", "--force-remote",
+        help="View remote logs remotely even if they have been retrieved"
+        " to the suite host (default False).",
+        action="store_true", default=False, dest="force_remote")
+
     parser.add_option(
         "-s", "--submit-number", "-t", "--try-number",
         help="Job submit number (default=%s, i.e. latest)." % NN,
@@ -351,8 +361,8 @@ def main():
         user, host = split_user_at_host(user_at_host)
         batchview_cmd = None
         if live_job_id is not None:
-            # Get special batch system log viewer command (e.g. qcat) if one
-            # exists, the log is out or err, and the job is live.
+            # Job is currently running. Get special batch system log view
+            # command (e.g. qcat) if one exists, and the log is out or err.
             conf_key = None
             if options.filename == JOB_LOG_OUT:
                 if options.mode == 'c':
@@ -374,8 +384,12 @@ def main():
                 if batchview_cmd_tmpl is not None:
                     batchview_cmd = batchview_cmd_tmpl % {
                         "job_id": str(live_job_id)}
-        pout = None
-        if is_remote(user, host) and (options.filename not in JOB_LOGS_LOCAL):
+
+        log_is_remote = (is_remote(user, host)
+                         and (options.filename not in JOB_LOGS_LOCAL))
+        log_is_retrieved = (glbl_cfg().get_host_item('retrieve job logs', host)
+                            and live_job_id is None)
+        if log_is_remote and (not log_is_retrieved or options.force_remote):
             logpath = os.path.normpath(os.path.join(
                 glbl_cfg().get_derived_host_item(
                     suite_name, "suite job log directory", host, user),

tests/cylc-cat-log/10-remote-no-retrieve.t

@@ -0,0 +1,48 @@
+#!/bin/bash
+# THIS FILE IS PART OF THE CYLC SUITE ENGINE.
+# Copyright (C) 2008-2018 NIWA
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#-------------------------------------------------------------------------------
+
+# Test "cylc cat-log" for remote tasks with no auto-retrieval.
+
+CYLC_TEST_IS_GENERIC=false
+. $(dirname $0)/test_header
+
+set_test_remote
+set_test_number 5
+
+create_test_globalrc "" "
+[hosts]
+   [[${CYLC_TEST_HOST}]]
+       retrieve job logs = False"
+install_suite $TEST_NAME_BASE remote-simple
+
+TEST_NAME=${TEST_NAME_BASE}-validate
+run_ok $TEST_NAME cylc validate $SUITE_NAME
+
+TEST_NAME=$TEST_NAME_BASE-run
+suite_run_ok $TEST_NAME cylc run --debug --no-detach $SUITE_NAME
+
+# Local job.out should not exist (not retrieved).
+LOCAL_JOB_DIR=$(cylc cat-log -f a -m d $SUITE_NAME a-task.1)
+exists_fail "${LOCAL_JOB_DIR}/job.out"
+
+# Cat the remote one.
+TEST_NAME=${TEST_NAME_BASE}-task-out
+run_ok $TEST_NAME cylc cat-log -f o $SUITE_NAME a-task.1
+grep_ok '^the quick brown fox$' ${TEST_NAME}.stdout
+
+purge_suite $SUITE_NAME

tests/cylc-cat-log/11-remote-retrieve.t

@@ -0,0 +1,56 @@
+#!/bin/bash
+# THIS FILE IS PART OF THE CYLC SUITE ENGINE.
+# Copyright (C) 2008-2018 NIWA
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#-------------------------------------------------------------------------------
+
+# Test "cylc cat-log" for remote tasks with auto-retrieval.
+
+CYLC_TEST_IS_GENERIC=false
+. $(dirname $0)/test_header
+
+set_test_remote
+set_test_number 7
+
+create_test_globalrc "" "
+[hosts]
+   [[${CYLC_TEST_HOST}]]
+       retrieve job logs = True"
+install_suite $TEST_NAME_BASE remote-simple
+
+TEST_NAME=${TEST_NAME_BASE}-validate
+run_ok $TEST_NAME cylc validate $SUITE_NAME
+
+TEST_NAME=${TEST_NAME_BASE}-run
+suite_run_ok $TEST_NAME cylc run --debug --no-detach $SUITE_NAME
+
+# Local job.out should exist (retrieved).
+LOCAL_JOB_OUT=$(cylc cat-log -f a -m d $SUITE_NAME a-task.1)/job.out
+exists_ok "${LOCAL_JOB_OUT}"
+
+# Distinguish local from remote job.out.
+perl -pi -e 's/fox/FOX/' "${LOCAL_JOB_OUT}"
+
+# Cat the remote one.
+TEST_NAME=${TEST_NAME_BASE}-out-rem
+run_ok $TEST_NAME cylc cat-log --force-remote -f o $SUITE_NAME a-task.1
+grep_ok '^the quick brown fox$' ${TEST_NAME}.stdout
+
+# Cat the local one.
+TEST_NAME=${TEST_NAME_BASE}-out-loc
+run_ok $TEST_NAME cylc cat-log -f o $SUITE_NAME a-task.1
+grep_ok '^the quick brown FOX$' ${TEST_NAME}.stdout
+
+purge_suite $SUITE_NAME

tests/cylc-cat-log/remote-simple/suite.rc

@@ -0,0 +1,14 @@
+#!Jinja2
+[cylc]
+   [[events]]
+       abort on timeout = True
+       timeout = PT1M
+[scheduling]
+    [[dependencies]]
+        graph = a-task
+[runtime]
+    [[a-task]]
+        script = echo "the quick brown fox"
+        [[[remote]]]
+            host={{environ['CYLC_TEST_HOST'] | default("localhost")}}
+            owner={{environ['CYLC_TEST_OWNER'] | default(environ["USER"])}}