diff --git a/lib/cylc/cfgspec/site.py b/lib/cylc/cfgspec/site.py
index 8e5f8736f47..0f07420b9a7 100644
--- a/lib/cylc/cfgspec/site.py
+++ b/lib/cylc/cfgspec/site.py
@@ -100,6 +100,7 @@
},
'test battery' : {
+ 'remote host with shared fs' : vdr( vtype='string' ),
'remote host' : vdr( vtype='string' ),
'directives' : {
'loadleveler host' : vdr( vtype='string' ),
diff --git a/lib/cylc/job_submission/background.py b/lib/cylc/job_submission/background.py
index e6334b239d1..44443c51797 100644
--- a/lib/cylc/job_submission/background.py
+++ b/lib/cylc/job_submission/background.py
@@ -15,14 +15,16 @@
#C:
#C: You should have received a copy of the GNU General Public License
#C: along with this program. If not, see .
+"""Implement background job submission."""
-from job_submit import JobSubmit
+from cylc.job_submission.job_submit import JobSubmit
from cylc.command_env import pr_scripting_sl
import os
from signal import SIGKILL
from subprocess import Popen, PIPE
-class background( JobSubmit ):
+
+class background(JobSubmit):
"""
Background 'job submission' runs the task directly in the background
(with '&') so that we can get the job PID (with $!) but then uses
@@ -32,7 +34,7 @@ class background( JobSubmit ):
% ssh user@host 'job-script & echo $!; wait'
(We have to override the general command templates to achieve this)."""
- LOCAL_COMMAND_TEMPLATE = ( "( %(command)s & echo $!; wait )" )
+ LOCAL_COMMAND_TEMPLATE = "( %(command)s & echo $!; wait )"
REMOTE_COMMAND_TEMPLATE = (
" '" +
@@ -40,6 +42,8 @@ class background( JobSubmit ):
"; " +
# Retry "mkdir" once to avoid race to create log/job/CYCLE/
" (mkdir -p %(jobfile_dir)s || mkdir -p %(jobfile_dir)s)" +
+ " && rm -f $(dirname %(jobfile_dir)s)/NN"
+ " && ln -s $(basename %(jobfile_dir)s) $(dirname %(jobfile_dir)s)/NN"
" && cat >%(jobfile_path)s.tmp" +
" && mv %(jobfile_path)s.tmp %(jobfile_path)s" +
" && chmod +x %(jobfile_path)s" +
@@ -50,21 +54,20 @@ class background( JobSubmit ):
# N.B. The perl command ensures that the job script is executed in its own
# process group, which allows the job script and its child processes to be
# killed correctly.
- COMMAND_TEMPLATE = ("perl -e \"setpgrp(0,0);exec(@ARGV)\" %s " +
- "%s 2>%s")
+ COMMAND_TEMPLATE = (
+ "perl -e \"setpgrp(0,0);exec(@ARGV)\" %s %s 2>%s")
- def construct_job_submit_command( self ):
+ def construct_job_submit_command(self):
"""
Construct a command to submit this job to run.
"""
command_template = self.job_submit_command_template
if not command_template:
command_template = self.__class__.COMMAND_TEMPLATE
- self.command = command_template % ( self.jobfile_path,
- self.stdout_file,
- self.stderr_file )
+ self.command = command_template % (
+ self.jobfile_path, self.stdout_file, self.stderr_file)
- def get_id( self, out, err ):
+ def get_id(self, out, err):
"""
Extract the job process ID from job submission command
output. For background jobs the submission command simply
@@ -72,10 +75,12 @@ def get_id( self, out, err ):
"""
return out.strip()
- def kill( self, jid, st_file=None ):
+ @classmethod
+ def kill(cls, jid, _=None):
"""Kill the job."""
os.killpg(int(jid), SIGKILL)
- def poll( self, jid ):
+ @classmethod
+ def poll(cls, jid):
"""Return 0 if jid is in the queueing system, 1 otherwise."""
return Popen(["ps", jid], stdout=PIPE).wait()
diff --git a/lib/cylc/job_submission/job_submit.py b/lib/cylc/job_submission/job_submit.py
index 3d3756016df..11997a43643 100644
--- a/lib/cylc/job_submission/job_submit.py
+++ b/lib/cylc/job_submission/job_submit.py
@@ -55,6 +55,8 @@ class JobSubmit(object):
"; " +
# Retry "mkdir" once to avoid race to create log/job/CYCLE/
" (mkdir -p %(jobfile_dir)s || mkdir -p %(jobfile_dir)s)" +
+ " && rm -f $(dirname %(jobfile_dir)s)/NN"
+ " && ln -s $(basename %(jobfile_dir)s) $(dirname %(jobfile_dir)s)/NN"
" && cat >%(jobfile_path)s.tmp" +
" && mv %(jobfile_path)s.tmp %(jobfile_path)s" +
" && chmod +x %(jobfile_path)s" +
diff --git a/tests/job-submission/00-user.t b/tests/job-submission/00-user.t
old mode 100644
new mode 100755
diff --git a/tests/job-submission/01-job-nn-localhost.t b/tests/job-submission/01-job-nn-localhost.t
new file mode 100755
index 00000000000..de9ac2a639d
--- /dev/null
+++ b/tests/job-submission/01-job-nn-localhost.t
@@ -0,0 +1,32 @@
+#!/bin/bash
+#C: THIS FILE IS PART OF THE CYLC SUITE ENGINE.
+#C: Copyright (C) 2008-2014 Hilary Oliver, NIWA
+#C:
+#C: This program is free software: you can redistribute it and/or modify
+#C: it under the terms of the GNU General Public License as published by
+#C: the Free Software Foundation, either version 3 of the License, or
+#C: (at your option) any later version.
+#C:
+#C: This program is distributed in the hope that it will be useful,
+#C: but WITHOUT ANY WARRANTY; without even the implied warranty of
+#C: MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+#C: GNU General Public License for more details.
+#C:
+#C: You should have received a copy of the GNU General Public License
+#C: along with this program. If not, see .
+#-------------------------------------------------------------------------------
+# Test localhost job log NN link correctness.
+. $(dirname $0)/test_header
+#-------------------------------------------------------------------------------
+set_test_number 2
+#-------------------------------------------------------------------------------
+install_suite "$TEST_NAME_BASE" "$TEST_NAME_BASE"
+#-------------------------------------------------------------------------------
+TEST_NAME="$TEST_NAME_BASE-validate"
+run_ok "$TEST_NAME" cylc validate "$SUITE_NAME"
+#-------------------------------------------------------------------------------
+TEST_NAME="$TEST_NAME_BASE-run"
+suite_run_ok "$TEST_NAME" cylc run --reference-test --debug "$SUITE_NAME"
+#-------------------------------------------------------------------------------
+purge_suite "$SUITE_NAME"
+exit
diff --git a/tests/job-submission/01-job-nn-localhost/reference.log b/tests/job-submission/01-job-nn-localhost/reference.log
new file mode 100644
index 00000000000..0e56bcf442e
--- /dev/null
+++ b/tests/job-submission/01-job-nn-localhost/reference.log
@@ -0,0 +1,711 @@
+2014-08-14T20:15:16Z INFO - port:7766
+2014-08-14T20:15:16Z INFO - Suite starting at 2014-08-14T20:15:16Z
+2014-08-14T20:15:16Z INFO - Log event clock: real time
+2014-08-14T20:15:16Z INFO - Run mode: live
+2014-08-14T20:15:16Z INFO - Initial point: 1
+2014-08-14T20:15:16Z INFO - Final point: 1
+2014-08-14T20:15:16Z INFO - Cold Start 1
+2014-08-14T20:15:16Z INFO - [foo.1] -triggered off []
+2014-08-14T20:15:17Z INFO - [foo.1] -submit_method_id=20205
+2014-08-14T20:15:17Z INFO - [foo.1] -submission succeeded
+2014-08-14T20:15:17Z INFO - [foo.1] -(current:submitted)> foo.1 started at 2014-08-14T20:15:16Z
+2014-08-14T20:15:17Z CRITICAL - [foo.1] -(current:running)> Task job script received signal ERR at 2014-08-14T20:15:16Z
+2014-08-14T20:15:17Z CRITICAL - [foo.1] -(current:running)> foo.1 failed at 2014-08-14T20:15:16Z
+2014-08-14T20:15:17Z INFO - [foo.1] -job failed, retrying in P0Y
+2014-08-14T20:15:18Z INFO - [foo.1] -triggered off []
+2014-08-14T20:15:19Z INFO - [foo.1] -submit_method_id=20248
+2014-08-14T20:15:19Z INFO - [foo.1] -submission succeeded
+2014-08-14T20:15:20Z INFO - [foo.1] -(current:submitted)> foo.1 started at 2014-08-14T20:15:19Z
+2014-08-14T20:15:20Z CRITICAL - [foo.1] -(current:running)> Task job script received signal ERR at 2014-08-14T20:15:19Z
+2014-08-14T20:15:20Z CRITICAL - [foo.1] -(current:running)> foo.1 failed at 2014-08-14T20:15:19Z
+2014-08-14T20:15:20Z INFO - [foo.1] -job failed, retrying in P0Y
+2014-08-14T20:15:21Z INFO - [foo.1] -triggered off []
+2014-08-14T20:15:22Z INFO - [foo.1] -submit_method_id=20291
+2014-08-14T20:15:22Z INFO - [foo.1] -submission succeeded
+2014-08-14T20:15:22Z INFO - [foo.1] -(current:submitted)> foo.1 started at 2014-08-14T20:15:21Z
+2014-08-14T20:15:22Z CRITICAL - [foo.1] -(current:running)> Task job script received signal ERR at 2014-08-14T20:15:21Z
+2014-08-14T20:15:22Z CRITICAL - [foo.1] -(current:running)> foo.1 failed at 2014-08-14T20:15:21Z
+2014-08-14T20:15:22Z INFO - [foo.1] -job failed, retrying in P0Y
+2014-08-14T20:15:23Z INFO - [foo.1] -triggered off []
+2014-08-14T20:15:24Z INFO - [foo.1] -submit_method_id=20343
+2014-08-14T20:15:24Z INFO - [foo.1] -submission succeeded
+2014-08-14T20:15:24Z INFO - [foo.1] -(current:submitted)> foo.1 started at 2014-08-14T20:15:23Z
+2014-08-14T20:15:24Z CRITICAL - [foo.1] -(current:running)> Task job script received signal ERR at 2014-08-14T20:15:23Z
+2014-08-14T20:15:24Z CRITICAL - [foo.1] -(current:running)> foo.1 failed at 2014-08-14T20:15:23Z
+2014-08-14T20:15:24Z INFO - [foo.1] -job failed, retrying in P0Y
+2014-08-14T20:15:25Z INFO - [foo.1] -triggered off []
+2014-08-14T20:15:26Z INFO - [foo.1] -submit_method_id=20386
+2014-08-14T20:15:26Z INFO - [foo.1] -submission succeeded
+2014-08-14T20:15:26Z INFO - [foo.1] -(current:submitted)> foo.1 started at 2014-08-14T20:15:25Z
+2014-08-14T20:15:26Z CRITICAL - [foo.1] -(current:running)> Task job script received signal ERR at 2014-08-14T20:15:26Z
+2014-08-14T20:15:26Z CRITICAL - [foo.1] -(current:running)> foo.1 failed at 2014-08-14T20:15:26Z
+2014-08-14T20:15:26Z INFO - [foo.1] -job failed, retrying in P0Y
+2014-08-14T20:15:27Z INFO - [foo.1] -triggered off []
+2014-08-14T20:15:29Z INFO - [foo.1] -submit_method_id=20429
+2014-08-14T20:15:29Z INFO - [foo.1] -submission succeeded
+2014-08-14T20:15:29Z INFO - [foo.1] -(current:submitted)> foo.1 started at 2014-08-14T20:15:28Z
+2014-08-14T20:15:29Z CRITICAL - [foo.1] -(current:running)> Task job script received signal ERR at 2014-08-14T20:15:28Z
+2014-08-14T20:15:29Z CRITICAL - [foo.1] -(current:running)> foo.1 failed at 2014-08-14T20:15:28Z
+2014-08-14T20:15:29Z INFO - [foo.1] -job failed, retrying in P0Y
+2014-08-14T20:15:30Z INFO - [foo.1] -triggered off []
+2014-08-14T20:15:31Z INFO - [foo.1] -submit_method_id=20472
+2014-08-14T20:15:31Z INFO - [foo.1] -submission succeeded
+2014-08-14T20:15:31Z INFO - [foo.1] -(current:submitted)> foo.1 started at 2014-08-14T20:15:30Z
+2014-08-14T20:15:31Z CRITICAL - [foo.1] -(current:running)> Task job script received signal ERR at 2014-08-14T20:15:30Z
+2014-08-14T20:15:31Z CRITICAL - [foo.1] -(current:running)> foo.1 failed at 2014-08-14T20:15:30Z
+2014-08-14T20:15:31Z INFO - [foo.1] -job failed, retrying in P0Y
+2014-08-14T20:15:32Z INFO - [foo.1] -triggered off []
+2014-08-14T20:15:33Z INFO - [foo.1] -submit_method_id=20515
+2014-08-14T20:15:33Z INFO - [foo.1] -submission succeeded
+2014-08-14T20:15:33Z INFO - [foo.1] -(current:submitted)> foo.1 started at 2014-08-14T20:15:32Z
+2014-08-14T20:15:33Z CRITICAL - [foo.1] -(current:running)> Task job script received signal ERR at 2014-08-14T20:15:32Z
+2014-08-14T20:15:33Z CRITICAL - [foo.1] -(current:running)> foo.1 failed at 2014-08-14T20:15:32Z
+2014-08-14T20:15:33Z INFO - [foo.1] -job failed, retrying in P0Y
+2014-08-14T20:15:34Z INFO - [foo.1] -triggered off []
+2014-08-14T20:15:35Z INFO - [foo.1] -submit_method_id=20558
+2014-08-14T20:15:35Z INFO - [foo.1] -submission succeeded
+2014-08-14T20:15:35Z INFO - [foo.1] -(current:submitted)> foo.1 started at 2014-08-14T20:15:34Z
+2014-08-14T20:15:35Z CRITICAL - [foo.1] -(current:running)> Task job script received signal ERR at 2014-08-14T20:15:35Z
+2014-08-14T20:15:35Z CRITICAL - [foo.1] -(current:running)> foo.1 failed at 2014-08-14T20:15:35Z
+2014-08-14T20:15:35Z INFO - [foo.1] -job failed, retrying in P0Y
+2014-08-14T20:15:36Z INFO - [foo.1] -triggered off []
+2014-08-14T20:15:38Z INFO - [foo.1] -submit_method_id=20601
+2014-08-14T20:15:38Z INFO - [foo.1] -submission succeeded
+2014-08-14T20:15:38Z INFO - [foo.1] -(current:submitted)> foo.1 started at 2014-08-14T20:15:37Z
+2014-08-14T20:15:38Z CRITICAL - [foo.1] -(current:running)> Task job script received signal ERR at 2014-08-14T20:15:37Z
+2014-08-14T20:15:38Z CRITICAL - [foo.1] -(current:running)> foo.1 failed at 2014-08-14T20:15:37Z
+2014-08-14T20:15:38Z INFO - [foo.1] -job failed, retrying in P0Y
+2014-08-14T20:15:39Z INFO - [foo.1] -triggered off []
+2014-08-14T20:15:40Z INFO - [foo.1] -submit_method_id=20644
+2014-08-14T20:15:40Z INFO - [foo.1] -submission succeeded
+2014-08-14T20:15:40Z INFO - [foo.1] -(current:submitted)> foo.1 started at 2014-08-14T20:15:39Z
+2014-08-14T20:15:40Z CRITICAL - [foo.1] -(current:running)> Task job script received signal ERR at 2014-08-14T20:15:39Z
+2014-08-14T20:15:40Z CRITICAL - [foo.1] -(current:running)> foo.1 failed at 2014-08-14T20:15:39Z
+2014-08-14T20:15:40Z INFO - [foo.1] -job failed, retrying in P0Y
+2014-08-14T20:15:41Z INFO - [foo.1] -triggered off []
+2014-08-14T20:15:42Z INFO - [foo.1] -submit_method_id=20687
+2014-08-14T20:15:42Z INFO - [foo.1] -submission succeeded
+2014-08-14T20:15:42Z INFO - [foo.1] -(current:submitted)> foo.1 started at 2014-08-14T20:15:41Z
+2014-08-14T20:15:42Z CRITICAL - [foo.1] -(current:running)> Task job script received signal ERR at 2014-08-14T20:15:41Z
+2014-08-14T20:15:42Z CRITICAL - [foo.1] -(current:running)> foo.1 failed at 2014-08-14T20:15:41Z
+2014-08-14T20:15:42Z INFO - [foo.1] -job failed, retrying in P0Y
+2014-08-14T20:15:43Z INFO - [foo.1] -triggered off []
+2014-08-14T20:15:44Z INFO - [foo.1] -(current:ready)> foo.1 started at 2014-08-14T20:15:43Z
+2014-08-14T20:15:44Z CRITICAL - [foo.1] -(current:running)> Task job script received signal ERR at 2014-08-14T20:15:44Z
+2014-08-14T20:15:44Z CRITICAL - [foo.1] -(current:running)> foo.1 failed at 2014-08-14T20:15:44Z
+2014-08-14T20:15:44Z INFO - [foo.1] -job failed, retrying in P0Y
+2014-08-14T20:15:45Z INFO - [foo.1] -submit_method_id=20738
+2014-08-14T20:15:45Z INFO - [foo.1] -submission succeeded
+2014-08-14T20:15:45Z INFO - [foo.1] -triggered off []
+2014-08-14T20:15:46Z INFO - [foo.1] -(current:ready)> foo.1 started at 2014-08-14T20:15:45Z
+2014-08-14T20:15:46Z CRITICAL - [foo.1] -(current:running)> Task job script received signal ERR at 2014-08-14T20:15:45Z
+2014-08-14T20:15:46Z CRITICAL - [foo.1] -(current:running)> foo.1 failed at 2014-08-14T20:15:45Z
+2014-08-14T20:15:46Z INFO - [foo.1] -job failed, retrying in P0Y
+2014-08-14T20:15:47Z INFO - [foo.1] -submit_method_id=20781
+2014-08-14T20:15:47Z INFO - [foo.1] -submission succeeded
+2014-08-14T20:15:47Z INFO - [foo.1] -triggered off []
+2014-08-14T20:15:47Z INFO - [foo.1] -(current:ready)> foo.1 started at 2014-08-14T20:15:47Z
+2014-08-14T20:15:47Z CRITICAL - [foo.1] -(current:running)> Task job script received signal ERR at 2014-08-14T20:15:47Z
+2014-08-14T20:15:47Z CRITICAL - [foo.1] -(current:running)> foo.1 failed at 2014-08-14T20:15:47Z
+2014-08-14T20:15:47Z INFO - [foo.1] -job failed, retrying in P0Y
+2014-08-14T20:15:48Z INFO - [foo.1] -submit_method_id=20825
+2014-08-14T20:15:48Z INFO - [foo.1] -submission succeeded
+2014-08-14T20:15:48Z INFO - [foo.1] -triggered off []
+2014-08-14T20:15:49Z INFO - [foo.1] -(current:ready)> foo.1 started at 2014-08-14T20:15:49Z
+2014-08-14T20:15:50Z INFO - [foo.1] -submit_method_id=20868
+2014-08-14T20:15:50Z INFO - [foo.1] -submission succeeded
+2014-08-14T20:15:50Z CRITICAL - [foo.1] -(current:running)> Task job script received signal ERR at 2014-08-14T20:15:49Z
+2014-08-14T20:15:50Z CRITICAL - [foo.1] -(current:running)> foo.1 failed at 2014-08-14T20:15:49Z
+2014-08-14T20:15:50Z INFO - [foo.1] -job failed, retrying in P0Y
+2014-08-14T20:15:51Z INFO - [foo.1] -triggered off []
+2014-08-14T20:15:52Z INFO - [foo.1] -submit_method_id=20911
+2014-08-14T20:15:52Z INFO - [foo.1] -submission succeeded
+2014-08-14T20:15:53Z INFO - [foo.1] -(current:submitted)> foo.1 started at 2014-08-14T20:15:51Z
+2014-08-14T20:15:53Z CRITICAL - [foo.1] -(current:running)> Task job script received signal ERR at 2014-08-14T20:15:52Z
+2014-08-14T20:15:53Z CRITICAL - [foo.1] -(current:running)> foo.1 failed at 2014-08-14T20:15:52Z
+2014-08-14T20:15:53Z INFO - [foo.1] -job failed, retrying in P0Y
+2014-08-14T20:15:54Z INFO - [foo.1] -triggered off []
+2014-08-14T20:15:54Z INFO - [foo.1] -(current:ready)> foo.1 started at 2014-08-14T20:15:54Z
+2014-08-14T20:15:54Z CRITICAL - [foo.1] -(current:running)> Task job script received signal ERR at 2014-08-14T20:15:54Z
+2014-08-14T20:15:54Z CRITICAL - [foo.1] -(current:running)> foo.1 failed at 2014-08-14T20:15:54Z
+2014-08-14T20:15:54Z INFO - [foo.1] -job failed, retrying in P0Y
+2014-08-14T20:15:55Z INFO - [foo.1] -submit_method_id=20954
+2014-08-14T20:15:55Z INFO - [foo.1] -submission succeeded
+2014-08-14T20:15:55Z INFO - [foo.1] -triggered off []
+2014-08-14T20:15:56Z INFO - [foo.1] -(current:ready)> foo.1 started at 2014-08-14T20:15:55Z
+2014-08-14T20:15:57Z INFO - [foo.1] -submit_method_id=20997
+2014-08-14T20:15:57Z INFO - [foo.1] -submission succeeded
+2014-08-14T20:15:57Z CRITICAL - [foo.1] -(current:running)> Task job script received signal ERR at 2014-08-14T20:15:56Z
+2014-08-14T20:15:57Z CRITICAL - [foo.1] -(current:running)> foo.1 failed at 2014-08-14T20:15:56Z
+2014-08-14T20:15:57Z INFO - [foo.1] -job failed, retrying in P0Y
+2014-08-14T20:15:58Z INFO - [foo.1] -triggered off []
+2014-08-14T20:15:59Z INFO - [foo.1] -(current:ready)> foo.1 started at 2014-08-14T20:15:58Z
+2014-08-14T20:15:59Z CRITICAL - [foo.1] -(current:running)> Task job script received signal ERR at 2014-08-14T20:15:58Z
+2014-08-14T20:15:59Z CRITICAL - [foo.1] -(current:running)> foo.1 failed at 2014-08-14T20:15:58Z
+2014-08-14T20:15:59Z INFO - [foo.1] -job failed, retrying in P0Y
+2014-08-14T20:16:00Z INFO - [foo.1] -submit_method_id=21040
+2014-08-14T20:16:00Z INFO - [foo.1] -submission succeeded
+2014-08-14T20:16:00Z INFO - [foo.1] -triggered off []
+2014-08-14T20:16:01Z INFO - [foo.1] -submit_method_id=21084
+2014-08-14T20:16:01Z INFO - [foo.1] -submission succeeded
+2014-08-14T20:16:01Z INFO - [foo.1] -(current:submitted)> foo.1 started at 2014-08-14T20:16:00Z
+2014-08-14T20:16:01Z CRITICAL - [foo.1] -(current:running)> Task job script received signal ERR at 2014-08-14T20:16:00Z
+2014-08-14T20:16:01Z CRITICAL - [foo.1] -(current:running)> foo.1 failed at 2014-08-14T20:16:00Z
+2014-08-14T20:16:01Z INFO - [foo.1] -job failed, retrying in P0Y
+2014-08-14T20:16:02Z INFO - [foo.1] -triggered off []
+2014-08-14T20:16:02Z INFO - [foo.1] -(current:ready)> foo.1 started at 2014-08-14T20:16:02Z
+2014-08-14T20:16:03Z INFO - [foo.1] -submit_method_id=21127
+2014-08-14T20:16:03Z INFO - [foo.1] -submission succeeded
+2014-08-14T20:16:04Z CRITICAL - [foo.1] -(current:running)> Task job script received signal ERR at 2014-08-14T20:16:03Z
+2014-08-14T20:16:04Z CRITICAL - [foo.1] -(current:running)> foo.1 failed at 2014-08-14T20:16:03Z
+2014-08-14T20:16:04Z INFO - [foo.1] -job failed, retrying in P0Y
+2014-08-14T20:16:05Z INFO - [foo.1] -triggered off []
+2014-08-14T20:16:05Z INFO - [foo.1] -(current:ready)> foo.1 started at 2014-08-14T20:16:05Z
+2014-08-14T20:16:06Z INFO - [foo.1] -submit_method_id=21170
+2014-08-14T20:16:06Z INFO - [foo.1] -submission succeeded
+2014-08-14T20:16:06Z CRITICAL - [foo.1] -(current:running)> Task job script received signal ERR at 2014-08-14T20:16:05Z
+2014-08-14T20:16:06Z CRITICAL - [foo.1] -(current:running)> foo.1 failed at 2014-08-14T20:16:05Z
+2014-08-14T20:16:06Z INFO - [foo.1] -job failed, retrying in P0Y
+2014-08-14T20:16:07Z INFO - [foo.1] -triggered off []
+2014-08-14T20:16:08Z INFO - [foo.1] -submit_method_id=21213
+2014-08-14T20:16:08Z INFO - [foo.1] -submission succeeded
+2014-08-14T20:16:08Z INFO - [foo.1] -(current:submitted)> foo.1 started at 2014-08-14T20:16:07Z
+2014-08-14T20:16:08Z CRITICAL - [foo.1] -(current:running)> Task job script received signal ERR at 2014-08-14T20:16:08Z
+2014-08-14T20:16:08Z CRITICAL - [foo.1] -(current:running)> foo.1 failed at 2014-08-14T20:16:08Z
+2014-08-14T20:16:08Z INFO - [foo.1] -job failed, retrying in P0Y
+2014-08-14T20:16:09Z INFO - [foo.1] -triggered off []
+2014-08-14T20:16:10Z INFO - [foo.1] -submit_method_id=21256
+2014-08-14T20:16:10Z INFO - [foo.1] -submission succeeded
+2014-08-14T20:16:11Z INFO - [foo.1] -(current:submitted)> foo.1 started at 2014-08-14T20:16:10Z
+2014-08-14T20:16:11Z CRITICAL - [foo.1] -(current:running)> Task job script received signal ERR at 2014-08-14T20:16:10Z
+2014-08-14T20:16:11Z CRITICAL - [foo.1] -(current:running)> foo.1 failed at 2014-08-14T20:16:10Z
+2014-08-14T20:16:11Z INFO - [foo.1] -job failed, retrying in P0Y
+2014-08-14T20:16:12Z INFO - [foo.1] -triggered off []
+2014-08-14T20:16:12Z INFO - [foo.1] -(current:ready)> foo.1 started at 2014-08-14T20:16:12Z
+2014-08-14T20:16:13Z INFO - [foo.1] -submit_method_id=21300
+2014-08-14T20:16:13Z INFO - [foo.1] -submission succeeded
+2014-08-14T20:16:15Z CRITICAL - [foo.1] -(current:running)> Task job script received signal ERR at 2014-08-14T20:16:12Z
+2014-08-14T20:16:15Z CRITICAL - [foo.1] -(current:running)> foo.1 failed at 2014-08-14T20:16:12Z
+2014-08-14T20:16:15Z INFO - [foo.1] -job failed, retrying in P0Y
+2014-08-14T20:16:16Z INFO - [foo.1] -triggered off []
+2014-08-14T20:16:17Z INFO - [foo.1] -submit_method_id=21344
+2014-08-14T20:16:17Z INFO - [foo.1] -submission succeeded
+2014-08-14T20:16:18Z INFO - [foo.1] -(current:submitted)> foo.1 started at 2014-08-14T20:16:16Z
+2014-08-14T20:16:18Z CRITICAL - [foo.1] -(current:running)> Task job script received signal ERR at 2014-08-14T20:16:16Z
+2014-08-14T20:16:18Z CRITICAL - [foo.1] -(current:running)> foo.1 failed at 2014-08-14T20:16:16Z
+2014-08-14T20:16:18Z INFO - [foo.1] -job failed, retrying in P0Y
+2014-08-14T20:16:19Z INFO - [foo.1] -triggered off []
+2014-08-14T20:16:19Z INFO - [foo.1] -(current:ready)> foo.1 started at 2014-08-14T20:16:19Z
+2014-08-14T20:16:20Z INFO - [foo.1] -submit_method_id=21388
+2014-08-14T20:16:20Z INFO - [foo.1] -submission succeeded
+2014-08-14T20:16:20Z CRITICAL - [foo.1] -(current:running)> Task job script received signal ERR at 2014-08-14T20:16:19Z
+2014-08-14T20:16:20Z CRITICAL - [foo.1] -(current:running)> foo.1 failed at 2014-08-14T20:16:19Z
+2014-08-14T20:16:20Z INFO - [foo.1] -job failed, retrying in P0Y
+2014-08-14T20:16:21Z INFO - [foo.1] -triggered off []
+2014-08-14T20:16:22Z INFO - [foo.1] -submit_method_id=21434
+2014-08-14T20:16:22Z INFO - [foo.1] -submission succeeded
+2014-08-14T20:16:22Z INFO - [foo.1] -(current:submitted)> foo.1 started at 2014-08-14T20:16:21Z
+2014-08-14T20:16:22Z CRITICAL - [foo.1] -(current:running)> Task job script received signal ERR at 2014-08-14T20:16:21Z
+2014-08-14T20:16:22Z CRITICAL - [foo.1] -(current:running)> foo.1 failed at 2014-08-14T20:16:21Z
+2014-08-14T20:16:22Z INFO - [foo.1] -job failed, retrying in P0Y
+2014-08-14T20:16:23Z INFO - [foo.1] -triggered off []
+2014-08-14T20:16:24Z INFO - [foo.1] -submit_method_id=21477
+2014-08-14T20:16:24Z INFO - [foo.1] -submission succeeded
+2014-08-14T20:16:25Z INFO - [foo.1] -(current:submitted)> foo.1 started at 2014-08-14T20:16:23Z
+2014-08-14T20:16:25Z CRITICAL - [foo.1] -(current:running)> Task job script received signal ERR at 2014-08-14T20:16:24Z
+2014-08-14T20:16:25Z CRITICAL - [foo.1] -(current:running)> foo.1 failed at 2014-08-14T20:16:24Z
+2014-08-14T20:16:25Z INFO - [foo.1] -job failed, retrying in P0Y
+2014-08-14T20:16:26Z INFO - [foo.1] -triggered off []
+2014-08-14T20:16:27Z INFO - [foo.1] -submit_method_id=21520
+2014-08-14T20:16:27Z INFO - [foo.1] -submission succeeded
+2014-08-14T20:16:27Z INFO - [foo.1] -(current:submitted)> foo.1 started at 2014-08-14T20:16:26Z
+2014-08-14T20:16:27Z CRITICAL - [foo.1] -(current:running)> Task job script received signal ERR at 2014-08-14T20:16:26Z
+2014-08-14T20:16:27Z CRITICAL - [foo.1] -(current:running)> foo.1 failed at 2014-08-14T20:16:26Z
+2014-08-14T20:16:27Z INFO - [foo.1] -job failed, retrying in P0Y
+2014-08-14T20:16:28Z INFO - [foo.1] -triggered off []
+2014-08-14T20:16:29Z INFO - [foo.1] -submit_method_id=21563
+2014-08-14T20:16:29Z INFO - [foo.1] -submission succeeded
+2014-08-14T20:16:29Z INFO - [foo.1] -(current:submitted)> foo.1 started at 2014-08-14T20:16:28Z
+2014-08-14T20:16:29Z CRITICAL - [foo.1] -(current:running)> Task job script received signal ERR at 2014-08-14T20:16:28Z
+2014-08-14T20:16:29Z CRITICAL - [foo.1] -(current:running)> foo.1 failed at 2014-08-14T20:16:28Z
+2014-08-14T20:16:29Z INFO - [foo.1] -job failed, retrying in P0Y
+2014-08-14T20:16:30Z INFO - [foo.1] -triggered off []
+2014-08-14T20:16:32Z INFO - [foo.1] -submit_method_id=21606
+2014-08-14T20:16:32Z INFO - [foo.1] -submission succeeded
+2014-08-14T20:16:32Z INFO - [foo.1] -(current:submitted)> foo.1 started at 2014-08-14T20:16:31Z
+2014-08-14T20:16:32Z CRITICAL - [foo.1] -(current:running)> Task job script received signal ERR at 2014-08-14T20:16:31Z
+2014-08-14T20:16:32Z CRITICAL - [foo.1] -(current:running)> foo.1 failed at 2014-08-14T20:16:31Z
+2014-08-14T20:16:32Z INFO - [foo.1] -job failed, retrying in P0Y
+2014-08-14T20:16:33Z INFO - [foo.1] -triggered off []
+2014-08-14T20:16:34Z INFO - [foo.1] -submit_method_id=21649
+2014-08-14T20:16:34Z INFO - [foo.1] -submission succeeded
+2014-08-14T20:16:34Z INFO - [foo.1] -(current:submitted)> foo.1 started at 2014-08-14T20:16:33Z
+2014-08-14T20:16:34Z CRITICAL - [foo.1] -(current:running)> Task job script received signal ERR at 2014-08-14T20:16:33Z
+2014-08-14T20:16:34Z CRITICAL - [foo.1] -(current:running)> foo.1 failed at 2014-08-14T20:16:33Z
+2014-08-14T20:16:34Z INFO - [foo.1] -job failed, retrying in P0Y
+2014-08-14T20:16:35Z INFO - [foo.1] -triggered off []
+2014-08-14T20:16:36Z INFO - [foo.1] -submit_method_id=21692
+2014-08-14T20:16:36Z INFO - [foo.1] -submission succeeded
+2014-08-14T20:16:36Z INFO - [foo.1] -(current:submitted)> foo.1 started at 2014-08-14T20:16:35Z
+2014-08-14T20:16:36Z CRITICAL - [foo.1] -(current:running)> Task job script received signal ERR at 2014-08-14T20:16:35Z
+2014-08-14T20:16:36Z CRITICAL - [foo.1] -(current:running)> foo.1 failed at 2014-08-14T20:16:35Z
+2014-08-14T20:16:36Z INFO - [foo.1] -job failed, retrying in P0Y
+2014-08-14T20:16:37Z INFO - [foo.1] -triggered off []
+2014-08-14T20:16:38Z INFO - [foo.1] -submit_method_id=21735
+2014-08-14T20:16:38Z INFO - [foo.1] -submission succeeded
+2014-08-14T20:16:38Z INFO - [foo.1] -(current:submitted)> foo.1 started at 2014-08-14T20:16:37Z
+2014-08-14T20:16:38Z CRITICAL - [foo.1] -(current:running)> Task job script received signal ERR at 2014-08-14T20:16:37Z
+2014-08-14T20:16:38Z CRITICAL - [foo.1] -(current:running)> foo.1 failed at 2014-08-14T20:16:37Z
+2014-08-14T20:16:38Z INFO - [foo.1] -job failed, retrying in P0Y
+2014-08-14T20:16:39Z INFO - [foo.1] -triggered off []
+2014-08-14T20:16:40Z INFO - [foo.1] -submit_method_id=21778
+2014-08-14T20:16:40Z INFO - [foo.1] -submission succeeded
+2014-08-14T20:16:40Z INFO - [foo.1] -(current:submitted)> foo.1 started at 2014-08-14T20:16:39Z
+2014-08-14T20:16:40Z CRITICAL - [foo.1] -(current:running)> Task job script received signal ERR at 2014-08-14T20:16:40Z
+2014-08-14T20:16:40Z CRITICAL - [foo.1] -(current:running)> foo.1 failed at 2014-08-14T20:16:40Z
+2014-08-14T20:16:40Z INFO - [foo.1] -job failed, retrying in P0Y
+2014-08-14T20:16:41Z INFO - [foo.1] -triggered off []
+2014-08-14T20:16:43Z INFO - [foo.1] -submit_method_id=21821
+2014-08-14T20:16:43Z INFO - [foo.1] -submission succeeded
+2014-08-14T20:16:43Z INFO - [foo.1] -(current:submitted)> foo.1 started at 2014-08-14T20:16:42Z
+2014-08-14T20:16:43Z CRITICAL - [foo.1] -(current:running)> Task job script received signal ERR at 2014-08-14T20:16:42Z
+2014-08-14T20:16:43Z CRITICAL - [foo.1] -(current:running)> foo.1 failed at 2014-08-14T20:16:42Z
+2014-08-14T20:16:43Z INFO - [foo.1] -job failed, retrying in P0Y
+2014-08-14T20:16:44Z INFO - [foo.1] -triggered off []
+2014-08-14T20:16:45Z INFO - [foo.1] -submit_method_id=21864
+2014-08-14T20:16:45Z INFO - [foo.1] -submission succeeded
+2014-08-14T20:16:45Z INFO - [foo.1] -(current:submitted)> foo.1 started at 2014-08-14T20:16:44Z
+2014-08-14T20:16:45Z CRITICAL - [foo.1] -(current:running)> Task job script received signal ERR at 2014-08-14T20:16:44Z
+2014-08-14T20:16:45Z CRITICAL - [foo.1] -(current:running)> foo.1 failed at 2014-08-14T20:16:44Z
+2014-08-14T20:16:45Z INFO - [foo.1] -job failed, retrying in P0Y
+2014-08-14T20:16:46Z INFO - [foo.1] -triggered off []
+2014-08-14T20:16:47Z INFO - [foo.1] -submit_method_id=21907
+2014-08-14T20:16:47Z INFO - [foo.1] -submission succeeded
+2014-08-14T20:16:47Z INFO - [foo.1] -(current:submitted)> foo.1 started at 2014-08-14T20:16:46Z
+2014-08-14T20:16:47Z CRITICAL - [foo.1] -(current:running)> Task job script received signal ERR at 2014-08-14T20:16:46Z
+2014-08-14T20:16:47Z CRITICAL - [foo.1] -(current:running)> foo.1 failed at 2014-08-14T20:16:46Z
+2014-08-14T20:16:47Z INFO - [foo.1] -job failed, retrying in P0Y
+2014-08-14T20:16:48Z INFO - [foo.1] -triggered off []
+2014-08-14T20:16:49Z INFO - [foo.1] -submit_method_id=21951
+2014-08-14T20:16:49Z INFO - [foo.1] -submission succeeded
+2014-08-14T20:16:49Z INFO - [foo.1] -(current:submitted)> foo.1 started at 2014-08-14T20:16:48Z
+2014-08-14T20:16:49Z CRITICAL - [foo.1] -(current:running)> Task job script received signal ERR at 2014-08-14T20:16:48Z
+2014-08-14T20:16:49Z CRITICAL - [foo.1] -(current:running)> foo.1 failed at 2014-08-14T20:16:48Z
+2014-08-14T20:16:49Z INFO - [foo.1] -job failed, retrying in P0Y
+2014-08-14T20:16:50Z INFO - [foo.1] -triggered off []
+2014-08-14T20:16:51Z INFO - [foo.1] -submit_method_id=21994
+2014-08-14T20:16:51Z INFO - [foo.1] -submission succeeded
+2014-08-14T20:16:51Z INFO - [foo.1] -(current:submitted)> foo.1 started at 2014-08-14T20:16:50Z
+2014-08-14T20:16:51Z CRITICAL - [foo.1] -(current:running)> Task job script received signal ERR at 2014-08-14T20:16:51Z
+2014-08-14T20:16:51Z CRITICAL - [foo.1] -(current:running)> foo.1 failed at 2014-08-14T20:16:51Z
+2014-08-14T20:16:51Z INFO - [foo.1] -job failed, retrying in P0Y
+2014-08-14T20:16:52Z INFO - [foo.1] -triggered off []
+2014-08-14T20:16:53Z INFO - [foo.1] -submit_method_id=22037
+2014-08-14T20:16:53Z INFO - [foo.1] -submission succeeded
+2014-08-14T20:16:54Z INFO - [foo.1] -(current:submitted)> foo.1 started at 2014-08-14T20:16:53Z
+2014-08-14T20:16:54Z CRITICAL - [foo.1] -(current:running)> Task job script received signal ERR at 2014-08-14T20:16:53Z
+2014-08-14T20:16:54Z CRITICAL - [foo.1] -(current:running)> foo.1 failed at 2014-08-14T20:16:53Z
+2014-08-14T20:16:54Z INFO - [foo.1] -job failed, retrying in P0Y
+2014-08-14T20:16:55Z INFO - [foo.1] -triggered off []
+2014-08-14T20:16:56Z INFO - [foo.1] -submit_method_id=22080
+2014-08-14T20:16:56Z INFO - [foo.1] -submission succeeded
+2014-08-14T20:16:56Z INFO - [foo.1] -(current:submitted)> foo.1 started at 2014-08-14T20:16:55Z
+2014-08-14T20:16:56Z CRITICAL - [foo.1] -(current:running)> Task job script received signal ERR at 2014-08-14T20:16:55Z
+2014-08-14T20:16:56Z CRITICAL - [foo.1] -(current:running)> foo.1 failed at 2014-08-14T20:16:55Z
+2014-08-14T20:16:56Z INFO - [foo.1] -job failed, retrying in P0Y
+2014-08-14T20:16:57Z INFO - [foo.1] -triggered off []
+2014-08-14T20:16:58Z INFO - [foo.1] -submit_method_id=22123
+2014-08-14T20:16:58Z INFO - [foo.1] -submission succeeded
+2014-08-14T20:16:58Z INFO - [foo.1] -(current:submitted)> foo.1 started at 2014-08-14T20:16:57Z
+2014-08-14T20:16:58Z CRITICAL - [foo.1] -(current:running)> Task job script received signal ERR at 2014-08-14T20:16:57Z
+2014-08-14T20:16:58Z CRITICAL - [foo.1] -(current:running)> foo.1 failed at 2014-08-14T20:16:57Z
+2014-08-14T20:16:58Z INFO - [foo.1] -job failed, retrying in P0Y
+2014-08-14T20:16:59Z INFO - [foo.1] -triggered off []
+2014-08-14T20:17:00Z INFO - [foo.1] -submit_method_id=22166
+2014-08-14T20:17:00Z INFO - [foo.1] -submission succeeded
+2014-08-14T20:17:00Z INFO - [foo.1] -(current:submitted)> foo.1 started at 2014-08-14T20:16:59Z
+2014-08-14T20:17:00Z CRITICAL - [foo.1] -(current:running)> Task job script received signal ERR at 2014-08-14T20:16:59Z
+2014-08-14T20:17:00Z CRITICAL - [foo.1] -(current:running)> foo.1 failed at 2014-08-14T20:16:59Z
+2014-08-14T20:17:00Z INFO - [foo.1] -job failed, retrying in P0Y
+2014-08-14T20:17:01Z INFO - [foo.1] -triggered off []
+2014-08-14T20:17:02Z INFO - [foo.1] -submit_method_id=22212
+2014-08-14T20:17:02Z INFO - [foo.1] -submission succeeded
+2014-08-14T20:17:02Z INFO - [foo.1] -(current:submitted)> foo.1 started at 2014-08-14T20:17:01Z
+2014-08-14T20:17:02Z CRITICAL - [foo.1] -(current:running)> Task job script received signal ERR at 2014-08-14T20:17:01Z
+2014-08-14T20:17:02Z CRITICAL - [foo.1] -(current:running)> foo.1 failed at 2014-08-14T20:17:01Z
+2014-08-14T20:17:02Z INFO - [foo.1] -job failed, retrying in P0Y
+2014-08-14T20:17:03Z INFO - [foo.1] -triggered off []
+2014-08-14T20:17:04Z INFO - [foo.1] -submit_method_id=22255
+2014-08-14T20:17:04Z INFO - [foo.1] -submission succeeded
+2014-08-14T20:17:04Z INFO - [foo.1] -(current:submitted)> foo.1 started at 2014-08-14T20:17:03Z
+2014-08-14T20:17:04Z CRITICAL - [foo.1] -(current:running)> Task job script received signal ERR at 2014-08-14T20:17:04Z
+2014-08-14T20:17:04Z CRITICAL - [foo.1] -(current:running)> foo.1 failed at 2014-08-14T20:17:04Z
+2014-08-14T20:17:04Z INFO - [foo.1] -job failed, retrying in P0Y
+2014-08-14T20:17:05Z INFO - [foo.1] -triggered off []
+2014-08-14T20:17:07Z INFO - [foo.1] -submit_method_id=22298
+2014-08-14T20:17:07Z INFO - [foo.1] -submission succeeded
+2014-08-14T20:17:07Z INFO - [foo.1] -(current:submitted)> foo.1 started at 2014-08-14T20:17:06Z
+2014-08-14T20:17:07Z CRITICAL - [foo.1] -(current:running)> Task job script received signal ERR at 2014-08-14T20:17:06Z
+2014-08-14T20:17:07Z CRITICAL - [foo.1] -(current:running)> foo.1 failed at 2014-08-14T20:17:06Z
+2014-08-14T20:17:07Z INFO - [foo.1] -job failed, retrying in P0Y
+2014-08-14T20:17:08Z INFO - [foo.1] -triggered off []
+2014-08-14T20:17:09Z INFO - [foo.1] -submit_method_id=22341
+2014-08-14T20:17:09Z INFO - [foo.1] -submission succeeded
+2014-08-14T20:17:09Z INFO - [foo.1] -(current:submitted)> foo.1 started at 2014-08-14T20:17:08Z
+2014-08-14T20:17:09Z CRITICAL - [foo.1] -(current:running)> Task job script received signal ERR at 2014-08-14T20:17:08Z
+2014-08-14T20:17:09Z CRITICAL - [foo.1] -(current:running)> foo.1 failed at 2014-08-14T20:17:08Z
+2014-08-14T20:17:09Z INFO - [foo.1] -job failed, retrying in P0Y
+2014-08-14T20:17:10Z INFO - [foo.1] -triggered off []
+2014-08-14T20:17:11Z INFO - [foo.1] -submit_method_id=22384
+2014-08-14T20:17:11Z INFO - [foo.1] -submission succeeded
+2014-08-14T20:17:11Z INFO - [foo.1] -(current:submitted)> foo.1 started at 2014-08-14T20:17:10Z
+2014-08-14T20:17:11Z CRITICAL - [foo.1] -(current:running)> Task job script received signal ERR at 2014-08-14T20:17:10Z
+2014-08-14T20:17:11Z CRITICAL - [foo.1] -(current:running)> foo.1 failed at 2014-08-14T20:17:10Z
+2014-08-14T20:17:11Z INFO - [foo.1] -job failed, retrying in P0Y
+2014-08-14T20:17:12Z INFO - [foo.1] -triggered off []
+2014-08-14T20:17:13Z INFO - [foo.1] -submit_method_id=22427
+2014-08-14T20:17:13Z INFO - [foo.1] -submission succeeded
+2014-08-14T20:17:13Z INFO - [foo.1] -(current:submitted)> foo.1 started at 2014-08-14T20:17:12Z
+2014-08-14T20:17:13Z CRITICAL - [foo.1] -(current:running)> Task job script received signal ERR at 2014-08-14T20:17:12Z
+2014-08-14T20:17:13Z CRITICAL - [foo.1] -(current:running)> foo.1 failed at 2014-08-14T20:17:12Z
+2014-08-14T20:17:13Z INFO - [foo.1] -job failed, retrying in P0Y
+2014-08-14T20:17:14Z INFO - [foo.1] -triggered off []
+2014-08-14T20:17:15Z INFO - [foo.1] -submit_method_id=22470
+2014-08-14T20:17:15Z INFO - [foo.1] -submission succeeded
+2014-08-14T20:17:15Z INFO - [foo.1] -(current:submitted)> foo.1 started at 2014-08-14T20:17:14Z
+2014-08-14T20:17:15Z CRITICAL - [foo.1] -(current:running)> Task job script received signal ERR at 2014-08-14T20:17:14Z
+2014-08-14T20:17:15Z CRITICAL - [foo.1] -(current:running)> foo.1 failed at 2014-08-14T20:17:14Z
+2014-08-14T20:17:15Z INFO - [foo.1] -job failed, retrying in P0Y
+2014-08-14T20:17:16Z INFO - [foo.1] -triggered off []
+2014-08-14T20:17:17Z INFO - [foo.1] -submit_method_id=22513
+2014-08-14T20:17:17Z INFO - [foo.1] -submission succeeded
+2014-08-14T20:17:17Z INFO - [foo.1] -(current:submitted)> foo.1 started at 2014-08-14T20:17:16Z
+2014-08-14T20:17:17Z CRITICAL - [foo.1] -(current:running)> Task job script received signal ERR at 2014-08-14T20:17:17Z
+2014-08-14T20:17:17Z CRITICAL - [foo.1] -(current:running)> foo.1 failed at 2014-08-14T20:17:17Z
+2014-08-14T20:17:17Z INFO - [foo.1] -job failed, retrying in P0Y
+2014-08-14T20:17:18Z INFO - [foo.1] -triggered off []
+2014-08-14T20:17:20Z INFO - [foo.1] -submit_method_id=22556
+2014-08-14T20:17:20Z INFO - [foo.1] -submission succeeded
+2014-08-14T20:17:20Z INFO - [foo.1] -(current:submitted)> foo.1 started at 2014-08-14T20:17:19Z
+2014-08-14T20:17:20Z CRITICAL - [foo.1] -(current:running)> Task job script received signal ERR at 2014-08-14T20:17:19Z
+2014-08-14T20:17:20Z CRITICAL - [foo.1] -(current:running)> foo.1 failed at 2014-08-14T20:17:19Z
+2014-08-14T20:17:20Z INFO - [foo.1] -job failed, retrying in P0Y
+2014-08-14T20:17:21Z INFO - [foo.1] -triggered off []
+2014-08-14T20:17:22Z INFO - [foo.1] -submit_method_id=22599
+2014-08-14T20:17:22Z INFO - [foo.1] -submission succeeded
+2014-08-14T20:17:22Z INFO - [foo.1] -(current:submitted)> foo.1 started at 2014-08-14T20:17:21Z
+2014-08-14T20:17:22Z CRITICAL - [foo.1] -(current:running)> Task job script received signal ERR at 2014-08-14T20:17:21Z
+2014-08-14T20:17:22Z CRITICAL - [foo.1] -(current:running)> foo.1 failed at 2014-08-14T20:17:21Z
+2014-08-14T20:17:22Z INFO - [foo.1] -job failed, retrying in P0Y
+2014-08-14T20:17:23Z INFO - [foo.1] -triggered off []
+2014-08-14T20:17:24Z INFO - [foo.1] -submit_method_id=22643
+2014-08-14T20:17:24Z INFO - [foo.1] -submission succeeded
+2014-08-14T20:17:24Z INFO - [foo.1] -(current:submitted)> foo.1 started at 2014-08-14T20:17:23Z
+2014-08-14T20:17:24Z CRITICAL - [foo.1] -(current:running)> Task job script received signal ERR at 2014-08-14T20:17:23Z
+2014-08-14T20:17:24Z CRITICAL - [foo.1] -(current:running)> foo.1 failed at 2014-08-14T20:17:23Z
+2014-08-14T20:17:24Z INFO - [foo.1] -job failed, retrying in P0Y
+2014-08-14T20:17:25Z INFO - [foo.1] -triggered off []
+2014-08-14T20:17:26Z INFO - [foo.1] -submit_method_id=22686
+2014-08-14T20:17:26Z INFO - [foo.1] -submission succeeded
+2014-08-14T20:17:26Z INFO - [foo.1] -(current:submitted)> foo.1 started at 2014-08-14T20:17:25Z
+2014-08-14T20:17:26Z CRITICAL - [foo.1] -(current:running)> Task job script received signal ERR at 2014-08-14T20:17:25Z
+2014-08-14T20:17:26Z CRITICAL - [foo.1] -(current:running)> foo.1 failed at 2014-08-14T20:17:25Z
+2014-08-14T20:17:26Z INFO - [foo.1] -job failed, retrying in P0Y
+2014-08-14T20:17:27Z INFO - [foo.1] -triggered off []
+2014-08-14T20:17:28Z INFO - [foo.1] -submit_method_id=22729
+2014-08-14T20:17:28Z INFO - [foo.1] -submission succeeded
+2014-08-14T20:17:28Z INFO - [foo.1] -(current:submitted)> foo.1 started at 2014-08-14T20:17:27Z
+2014-08-14T20:17:28Z CRITICAL - [foo.1] -(current:running)> Task job script received signal ERR at 2014-08-14T20:17:27Z
+2014-08-14T20:17:28Z CRITICAL - [foo.1] -(current:running)> foo.1 failed at 2014-08-14T20:17:27Z
+2014-08-14T20:17:28Z INFO - [foo.1] -job failed, retrying in P0Y
+2014-08-14T20:17:29Z INFO - [foo.1] -triggered off []
+2014-08-14T20:17:30Z INFO - [foo.1] -submit_method_id=22772
+2014-08-14T20:17:30Z INFO - [foo.1] -submission succeeded
+2014-08-14T20:17:30Z INFO - [foo.1] -(current:submitted)> foo.1 started at 2014-08-14T20:17:29Z
+2014-08-14T20:17:30Z CRITICAL - [foo.1] -(current:running)> Task job script received signal ERR at 2014-08-14T20:17:30Z
+2014-08-14T20:17:30Z CRITICAL - [foo.1] -(current:running)> foo.1 failed at 2014-08-14T20:17:30Z
+2014-08-14T20:17:30Z INFO - [foo.1] -job failed, retrying in P0Y
+2014-08-14T20:17:31Z INFO - [foo.1] -triggered off []
+2014-08-14T20:17:32Z INFO - [foo.1] -submit_method_id=22815
+2014-08-14T20:17:32Z INFO - [foo.1] -submission succeeded
+2014-08-14T20:17:33Z INFO - [foo.1] -(current:submitted)> foo.1 started at 2014-08-14T20:17:32Z
+2014-08-14T20:17:33Z CRITICAL - [foo.1] -(current:running)> Task job script received signal ERR at 2014-08-14T20:17:32Z
+2014-08-14T20:17:33Z CRITICAL - [foo.1] -(current:running)> foo.1 failed at 2014-08-14T20:17:32Z
+2014-08-14T20:17:33Z INFO - [foo.1] -job failed, retrying in P0Y
+2014-08-14T20:17:34Z INFO - [foo.1] -triggered off []
+2014-08-14T20:17:35Z INFO - [foo.1] -submit_method_id=22858
+2014-08-14T20:17:35Z INFO - [foo.1] -submission succeeded
+2014-08-14T20:17:35Z INFO - [foo.1] -(current:submitted)> foo.1 started at 2014-08-14T20:17:34Z
+2014-08-14T20:17:35Z CRITICAL - [foo.1] -(current:running)> Task job script received signal ERR at 2014-08-14T20:17:34Z
+2014-08-14T20:17:35Z CRITICAL - [foo.1] -(current:running)> foo.1 failed at 2014-08-14T20:17:34Z
+2014-08-14T20:17:35Z INFO - [foo.1] -job failed, retrying in P0Y
+2014-08-14T20:17:36Z INFO - [foo.1] -triggered off []
+2014-08-14T20:17:37Z INFO - [foo.1] -submit_method_id=22901
+2014-08-14T20:17:37Z INFO - [foo.1] -submission succeeded
+2014-08-14T20:17:37Z INFO - [foo.1] -(current:submitted)> foo.1 started at 2014-08-14T20:17:36Z
+2014-08-14T20:17:37Z CRITICAL - [foo.1] -(current:running)> Task job script received signal ERR at 2014-08-14T20:17:36Z
+2014-08-14T20:17:37Z CRITICAL - [foo.1] -(current:running)> foo.1 failed at 2014-08-14T20:17:36Z
+2014-08-14T20:17:37Z INFO - [foo.1] -job failed, retrying in P0Y
+2014-08-14T20:17:38Z INFO - [foo.1] -triggered off []
+2014-08-14T20:17:39Z INFO - [foo.1] -submit_method_id=22944
+2014-08-14T20:17:39Z INFO - [foo.1] -submission succeeded
+2014-08-14T20:17:39Z INFO - [foo.1] -(current:submitted)> foo.1 started at 2014-08-14T20:17:38Z
+2014-08-14T20:17:39Z CRITICAL - [foo.1] -(current:running)> Task job script received signal ERR at 2014-08-14T20:17:38Z
+2014-08-14T20:17:39Z CRITICAL - [foo.1] -(current:running)> foo.1 failed at 2014-08-14T20:17:38Z
+2014-08-14T20:17:39Z INFO - [foo.1] -job failed, retrying in P0Y
+2014-08-14T20:17:40Z INFO - [foo.1] -triggered off []
+2014-08-14T20:17:41Z INFO - [foo.1] -submit_method_id=22987
+2014-08-14T20:17:41Z INFO - [foo.1] -submission succeeded
+2014-08-14T20:17:41Z INFO - [foo.1] -(current:submitted)> foo.1 started at 2014-08-14T20:17:40Z
+2014-08-14T20:17:41Z CRITICAL - [foo.1] -(current:running)> Task job script received signal ERR at 2014-08-14T20:17:40Z
+2014-08-14T20:17:41Z CRITICAL - [foo.1] -(current:running)> foo.1 failed at 2014-08-14T20:17:40Z
+2014-08-14T20:17:41Z INFO - [foo.1] -job failed, retrying in P0Y
+2014-08-14T20:17:42Z INFO - [foo.1] -triggered off []
+2014-08-14T20:17:43Z INFO - [foo.1] -submit_method_id=23030
+2014-08-14T20:17:43Z INFO - [foo.1] -submission succeeded
+2014-08-14T20:17:43Z INFO - [foo.1] -(current:submitted)> foo.1 started at 2014-08-14T20:17:42Z
+2014-08-14T20:17:43Z CRITICAL - [foo.1] -(current:running)> Task job script received signal ERR at 2014-08-14T20:17:43Z
+2014-08-14T20:17:43Z CRITICAL - [foo.1] -(current:running)> foo.1 failed at 2014-08-14T20:17:43Z
+2014-08-14T20:17:43Z INFO - [foo.1] -job failed, retrying in P0Y
+2014-08-14T20:17:44Z INFO - [foo.1] -triggered off []
+2014-08-14T20:17:46Z INFO - [foo.1] -submit_method_id=23073
+2014-08-14T20:17:46Z INFO - [foo.1] -submission succeeded
+2014-08-14T20:17:46Z INFO - [foo.1] -(current:submitted)> foo.1 started at 2014-08-14T20:17:45Z
+2014-08-14T20:17:46Z CRITICAL - [foo.1] -(current:running)> Task job script received signal ERR at 2014-08-14T20:17:45Z
+2014-08-14T20:17:46Z CRITICAL - [foo.1] -(current:running)> foo.1 failed at 2014-08-14T20:17:45Z
+2014-08-14T20:17:46Z INFO - [foo.1] -job failed, retrying in P0Y
+2014-08-14T20:17:47Z INFO - [foo.1] -triggered off []
+2014-08-14T20:17:48Z INFO - [foo.1] -submit_method_id=23117
+2014-08-14T20:17:48Z INFO - [foo.1] -submission succeeded
+2014-08-14T20:17:48Z INFO - [foo.1] -(current:submitted)> foo.1 started at 2014-08-14T20:17:47Z
+2014-08-14T20:17:48Z CRITICAL - [foo.1] -(current:running)> Task job script received signal ERR at 2014-08-14T20:17:47Z
+2014-08-14T20:17:48Z CRITICAL - [foo.1] -(current:running)> foo.1 failed at 2014-08-14T20:17:47Z
+2014-08-14T20:17:48Z INFO - [foo.1] -job failed, retrying in P0Y
+2014-08-14T20:17:49Z INFO - [foo.1] -triggered off []
+2014-08-14T20:17:50Z INFO - [foo.1] -submit_method_id=23160
+2014-08-14T20:17:50Z INFO - [foo.1] -submission succeeded
+2014-08-14T20:17:50Z INFO - [foo.1] -(current:submitted)> foo.1 started at 2014-08-14T20:17:49Z
+2014-08-14T20:17:50Z CRITICAL - [foo.1] -(current:running)> Task job script received signal ERR at 2014-08-14T20:17:49Z
+2014-08-14T20:17:50Z CRITICAL - [foo.1] -(current:running)> foo.1 failed at 2014-08-14T20:17:49Z
+2014-08-14T20:17:50Z INFO - [foo.1] -job failed, retrying in P0Y
+2014-08-14T20:17:51Z INFO - [foo.1] -triggered off []
+2014-08-14T20:17:52Z INFO - [foo.1] -submit_method_id=23203
+2014-08-14T20:17:52Z INFO - [foo.1] -submission succeeded
+2014-08-14T20:17:52Z INFO - [foo.1] -(current:submitted)> foo.1 started at 2014-08-14T20:17:51Z
+2014-08-14T20:17:52Z CRITICAL - [foo.1] -(current:running)> Task job script received signal ERR at 2014-08-14T20:17:51Z
+2014-08-14T20:17:52Z CRITICAL - [foo.1] -(current:running)> foo.1 failed at 2014-08-14T20:17:51Z
+2014-08-14T20:17:52Z INFO - [foo.1] -job failed, retrying in P0Y
+2014-08-14T20:17:53Z INFO - [foo.1] -triggered off []
+2014-08-14T20:17:54Z INFO - [foo.1] -submit_method_id=23246
+2014-08-14T20:17:54Z INFO - [foo.1] -submission succeeded
+2014-08-14T20:17:54Z INFO - [foo.1] -(current:submitted)> foo.1 started at 2014-08-14T20:17:53Z
+2014-08-14T20:17:54Z CRITICAL - [foo.1] -(current:running)> Task job script received signal ERR at 2014-08-14T20:17:53Z
+2014-08-14T20:17:54Z CRITICAL - [foo.1] -(current:running)> foo.1 failed at 2014-08-14T20:17:53Z
+2014-08-14T20:17:54Z INFO - [foo.1] -job failed, retrying in P0Y
+2014-08-14T20:17:55Z INFO - [foo.1] -triggered off []
+2014-08-14T20:17:56Z INFO - [foo.1] -submit_method_id=23289
+2014-08-14T20:17:56Z INFO - [foo.1] -submission succeeded
+2014-08-14T20:17:56Z INFO - [foo.1] -(current:submitted)> foo.1 started at 2014-08-14T20:17:55Z
+2014-08-14T20:17:56Z CRITICAL - [foo.1] -(current:running)> Task job script received signal ERR at 2014-08-14T20:17:56Z
+2014-08-14T20:17:56Z CRITICAL - [foo.1] -(current:running)> foo.1 failed at 2014-08-14T20:17:56Z
+2014-08-14T20:17:56Z INFO - [foo.1] -job failed, retrying in P0Y
+2014-08-14T20:17:57Z INFO - [foo.1] -triggered off []
+2014-08-14T20:17:58Z INFO - [foo.1] -submit_method_id=23332
+2014-08-14T20:17:58Z INFO - [foo.1] -submission succeeded
+2014-08-14T20:17:58Z INFO - [foo.1] -(current:submitted)> foo.1 started at 2014-08-14T20:17:57Z
+2014-08-14T20:17:58Z CRITICAL - [foo.1] -(current:running)> Task job script received signal ERR at 2014-08-14T20:17:58Z
+2014-08-14T20:17:58Z CRITICAL - [foo.1] -(current:running)> foo.1 failed at 2014-08-14T20:17:58Z
+2014-08-14T20:17:58Z INFO - [foo.1] -job failed, retrying in P0Y
+2014-08-14T20:18:00Z INFO - [foo.1] -triggered off []
+2014-08-14T20:18:01Z INFO - [foo.1] -submit_method_id=23375
+2014-08-14T20:18:01Z INFO - [foo.1] -submission succeeded
+2014-08-14T20:18:01Z INFO - [foo.1] -(current:submitted)> foo.1 started at 2014-08-14T20:18:00Z
+2014-08-14T20:18:01Z CRITICAL - [foo.1] -(current:running)> Task job script received signal ERR at 2014-08-14T20:18:00Z
+2014-08-14T20:18:01Z CRITICAL - [foo.1] -(current:running)> foo.1 failed at 2014-08-14T20:18:00Z
+2014-08-14T20:18:01Z INFO - [foo.1] -job failed, retrying in P0Y
+2014-08-14T20:18:02Z INFO - [foo.1] -triggered off []
+2014-08-14T20:18:03Z INFO - [foo.1] -submit_method_id=23418
+2014-08-14T20:18:03Z INFO - [foo.1] -submission succeeded
+2014-08-14T20:18:03Z INFO - [foo.1] -(current:submitted)> foo.1 started at 2014-08-14T20:18:02Z
+2014-08-14T20:18:03Z CRITICAL - [foo.1] -(current:running)> Task job script received signal ERR at 2014-08-14T20:18:02Z
+2014-08-14T20:18:03Z CRITICAL - [foo.1] -(current:running)> foo.1 failed at 2014-08-14T20:18:02Z
+2014-08-14T20:18:03Z INFO - [foo.1] -job failed, retrying in P0Y
+2014-08-14T20:18:04Z INFO - [foo.1] -triggered off []
+2014-08-14T20:18:05Z INFO - [foo.1] -submit_method_id=23461
+2014-08-14T20:18:05Z INFO - [foo.1] -submission succeeded
+2014-08-14T20:18:05Z INFO - [foo.1] -(current:submitted)> foo.1 started at 2014-08-14T20:18:04Z
+2014-08-14T20:18:05Z CRITICAL - [foo.1] -(current:running)> Task job script received signal ERR at 2014-08-14T20:18:04Z
+2014-08-14T20:18:05Z CRITICAL - [foo.1] -(current:running)> foo.1 failed at 2014-08-14T20:18:04Z
+2014-08-14T20:18:05Z INFO - [foo.1] -job failed, retrying in P0Y
+2014-08-14T20:18:06Z INFO - [foo.1] -triggered off []
+2014-08-14T20:18:07Z INFO - [foo.1] -submit_method_id=23504
+2014-08-14T20:18:07Z INFO - [foo.1] -submission succeeded
+2014-08-14T20:18:07Z INFO - [foo.1] -(current:submitted)> foo.1 started at 2014-08-14T20:18:06Z
+2014-08-14T20:18:07Z CRITICAL - [foo.1] -(current:running)> Task job script received signal ERR at 2014-08-14T20:18:06Z
+2014-08-14T20:18:07Z CRITICAL - [foo.1] -(current:running)> foo.1 failed at 2014-08-14T20:18:06Z
+2014-08-14T20:18:07Z INFO - [foo.1] -job failed, retrying in P0Y
+2014-08-14T20:18:08Z INFO - [foo.1] -triggered off []
+2014-08-14T20:18:09Z INFO - [foo.1] -submit_method_id=23547
+2014-08-14T20:18:09Z INFO - [foo.1] -submission succeeded
+2014-08-14T20:18:09Z INFO - [foo.1] -(current:submitted)> foo.1 started at 2014-08-14T20:18:08Z
+2014-08-14T20:18:09Z CRITICAL - [foo.1] -(current:running)> Task job script received signal ERR at 2014-08-14T20:18:09Z
+2014-08-14T20:18:09Z CRITICAL - [foo.1] -(current:running)> foo.1 failed at 2014-08-14T20:18:09Z
+2014-08-14T20:18:09Z INFO - [foo.1] -job failed, retrying in P0Y
+2014-08-14T20:18:10Z INFO - [foo.1] -triggered off []
+2014-08-14T20:18:11Z INFO - [foo.1] -submit_method_id=23590
+2014-08-14T20:18:11Z INFO - [foo.1] -submission succeeded
+2014-08-14T20:18:12Z INFO - [foo.1] -(current:submitted)> foo.1 started at 2014-08-14T20:18:11Z
+2014-08-14T20:18:12Z CRITICAL - [foo.1] -(current:running)> Task job script received signal ERR at 2014-08-14T20:18:11Z
+2014-08-14T20:18:12Z CRITICAL - [foo.1] -(current:running)> foo.1 failed at 2014-08-14T20:18:11Z
+2014-08-14T20:18:12Z INFO - [foo.1] -job failed, retrying in P0Y
+2014-08-14T20:18:13Z INFO - [foo.1] -triggered off []
+2014-08-14T20:18:14Z INFO - [foo.1] -submit_method_id=23633
+2014-08-14T20:18:14Z INFO - [foo.1] -submission succeeded
+2014-08-14T20:18:14Z INFO - [foo.1] -(current:submitted)> foo.1 started at 2014-08-14T20:18:13Z
+2014-08-14T20:18:14Z CRITICAL - [foo.1] -(current:running)> Task job script received signal ERR at 2014-08-14T20:18:13Z
+2014-08-14T20:18:14Z CRITICAL - [foo.1] -(current:running)> foo.1 failed at 2014-08-14T20:18:13Z
+2014-08-14T20:18:14Z INFO - [foo.1] -job failed, retrying in P0Y
+2014-08-14T20:18:15Z INFO - [foo.1] -triggered off []
+2014-08-14T20:18:16Z INFO - [foo.1] -submit_method_id=23676
+2014-08-14T20:18:16Z INFO - [foo.1] -submission succeeded
+2014-08-14T20:18:16Z INFO - [foo.1] -(current:submitted)> foo.1 started at 2014-08-14T20:18:15Z
+2014-08-14T20:18:16Z CRITICAL - [foo.1] -(current:running)> Task job script received signal ERR at 2014-08-14T20:18:15Z
+2014-08-14T20:18:16Z CRITICAL - [foo.1] -(current:running)> foo.1 failed at 2014-08-14T20:18:15Z
+2014-08-14T20:18:16Z INFO - [foo.1] -job failed, retrying in P0Y
+2014-08-14T20:18:17Z INFO - [foo.1] -triggered off []
+2014-08-14T20:18:18Z INFO - [foo.1] -submit_method_id=23719
+2014-08-14T20:18:18Z INFO - [foo.1] -submission succeeded
+2014-08-14T20:18:18Z INFO - [foo.1] -(current:submitted)> foo.1 started at 2014-08-14T20:18:17Z
+2014-08-14T20:18:18Z CRITICAL - [foo.1] -(current:running)> Task job script received signal ERR at 2014-08-14T20:18:17Z
+2014-08-14T20:18:18Z CRITICAL - [foo.1] -(current:running)> foo.1 failed at 2014-08-14T20:18:17Z
+2014-08-14T20:18:18Z INFO - [foo.1] -job failed, retrying in P0Y
+2014-08-14T20:18:19Z INFO - [foo.1] -triggered off []
+2014-08-14T20:18:20Z INFO - [foo.1] -submit_method_id=23762
+2014-08-14T20:18:20Z INFO - [foo.1] -submission succeeded
+2014-08-14T20:18:20Z INFO - [foo.1] -(current:submitted)> foo.1 started at 2014-08-14T20:18:19Z
+2014-08-14T20:18:20Z CRITICAL - [foo.1] -(current:running)> Task job script received signal ERR at 2014-08-14T20:18:19Z
+2014-08-14T20:18:20Z CRITICAL - [foo.1] -(current:running)> foo.1 failed at 2014-08-14T20:18:19Z
+2014-08-14T20:18:20Z INFO - [foo.1] -job failed, retrying in P0Y
+2014-08-14T20:18:21Z INFO - [foo.1] -triggered off []
+2014-08-14T20:18:22Z INFO - [foo.1] -submit_method_id=23805
+2014-08-14T20:18:22Z INFO - [foo.1] -submission succeeded
+2014-08-14T20:18:22Z INFO - [foo.1] -(current:submitted)> foo.1 started at 2014-08-14T20:18:21Z
+2014-08-14T20:18:22Z CRITICAL - [foo.1] -(current:running)> Task job script received signal ERR at 2014-08-14T20:18:22Z
+2014-08-14T20:18:22Z CRITICAL - [foo.1] -(current:running)> foo.1 failed at 2014-08-14T20:18:22Z
+2014-08-14T20:18:22Z INFO - [foo.1] -job failed, retrying in P0Y
+2014-08-14T20:18:23Z INFO - [foo.1] -triggered off []
+2014-08-14T20:18:24Z INFO - [foo.1] -submit_method_id=23848
+2014-08-14T20:18:24Z INFO - [foo.1] -submission succeeded
+2014-08-14T20:18:25Z INFO - [foo.1] -(current:submitted)> foo.1 started at 2014-08-14T20:18:24Z
+2014-08-14T20:18:25Z CRITICAL - [foo.1] -(current:running)> Task job script received signal ERR at 2014-08-14T20:18:24Z
+2014-08-14T20:18:25Z CRITICAL - [foo.1] -(current:running)> foo.1 failed at 2014-08-14T20:18:24Z
+2014-08-14T20:18:25Z INFO - [foo.1] -job failed, retrying in P0Y
+2014-08-14T20:18:26Z INFO - [foo.1] -triggered off []
+2014-08-14T20:18:27Z INFO - [foo.1] -submit_method_id=23891
+2014-08-14T20:18:27Z INFO - [foo.1] -submission succeeded
+2014-08-14T20:18:27Z INFO - [foo.1] -(current:submitted)> foo.1 started at 2014-08-14T20:18:26Z
+2014-08-14T20:18:27Z CRITICAL - [foo.1] -(current:running)> Task job script received signal ERR at 2014-08-14T20:18:26Z
+2014-08-14T20:18:27Z CRITICAL - [foo.1] -(current:running)> foo.1 failed at 2014-08-14T20:18:26Z
+2014-08-14T20:18:27Z INFO - [foo.1] -job failed, retrying in P0Y
+2014-08-14T20:18:28Z INFO - [foo.1] -triggered off []
+2014-08-14T20:18:29Z INFO - [foo.1] -submit_method_id=23934
+2014-08-14T20:18:29Z INFO - [foo.1] -submission succeeded
+2014-08-14T20:18:29Z INFO - [foo.1] -(current:submitted)> foo.1 started at 2014-08-14T20:18:28Z
+2014-08-14T20:18:29Z CRITICAL - [foo.1] -(current:running)> Task job script received signal ERR at 2014-08-14T20:18:28Z
+2014-08-14T20:18:29Z CRITICAL - [foo.1] -(current:running)> foo.1 failed at 2014-08-14T20:18:28Z
+2014-08-14T20:18:29Z INFO - [foo.1] -job failed, retrying in P0Y
+2014-08-14T20:18:30Z INFO - [foo.1] -triggered off []
+2014-08-14T20:18:31Z INFO - [foo.1] -submit_method_id=23977
+2014-08-14T20:18:31Z INFO - [foo.1] -submission succeeded
+2014-08-14T20:18:31Z INFO - [foo.1] -(current:submitted)> foo.1 started at 2014-08-14T20:18:30Z
+2014-08-14T20:18:31Z CRITICAL - [foo.1] -(current:running)> Task job script received signal ERR at 2014-08-14T20:18:30Z
+2014-08-14T20:18:31Z CRITICAL - [foo.1] -(current:running)> foo.1 failed at 2014-08-14T20:18:30Z
+2014-08-14T20:18:31Z INFO - [foo.1] -job failed, retrying in P0Y
+2014-08-14T20:18:32Z INFO - [foo.1] -triggered off []
+2014-08-14T20:18:33Z INFO - [foo.1] -submit_method_id=24020
+2014-08-14T20:18:33Z INFO - [foo.1] -submission succeeded
+2014-08-14T20:18:33Z INFO - [foo.1] -(current:submitted)> foo.1 started at 2014-08-14T20:18:32Z
+2014-08-14T20:18:33Z CRITICAL - [foo.1] -(current:running)> Task job script received signal ERR at 2014-08-14T20:18:32Z
+2014-08-14T20:18:33Z CRITICAL - [foo.1] -(current:running)> foo.1 failed at 2014-08-14T20:18:32Z
+2014-08-14T20:18:33Z INFO - [foo.1] -job failed, retrying in P0Y
+2014-08-14T20:18:34Z INFO - [foo.1] -triggered off []
+2014-08-14T20:18:35Z INFO - [foo.1] -submit_method_id=24063
+2014-08-14T20:18:35Z INFO - [foo.1] -submission succeeded
+2014-08-14T20:18:35Z INFO - [foo.1] -(current:submitted)> foo.1 started at 2014-08-14T20:18:34Z
+2014-08-14T20:18:35Z CRITICAL - [foo.1] -(current:running)> Task job script received signal ERR at 2014-08-14T20:18:35Z
+2014-08-14T20:18:35Z CRITICAL - [foo.1] -(current:running)> foo.1 failed at 2014-08-14T20:18:35Z
+2014-08-14T20:18:35Z INFO - [foo.1] -job failed, retrying in P0Y
+2014-08-14T20:18:36Z INFO - [foo.1] -triggered off []
+2014-08-14T20:18:37Z INFO - [foo.1] -submit_method_id=24106
+2014-08-14T20:18:37Z INFO - [foo.1] -submission succeeded
+2014-08-14T20:18:37Z INFO - [foo.1] -(current:submitted)> foo.1 started at 2014-08-14T20:18:36Z
+2014-08-14T20:18:37Z CRITICAL - [foo.1] -(current:running)> Task job script received signal ERR at 2014-08-14T20:18:37Z
+2014-08-14T20:18:37Z CRITICAL - [foo.1] -(current:running)> foo.1 failed at 2014-08-14T20:18:37Z
+2014-08-14T20:18:37Z INFO - [foo.1] -job failed, retrying in P0Y
+2014-08-14T20:18:38Z INFO - [foo.1] -triggered off []
+2014-08-14T20:18:39Z INFO - [foo.1] -(current:ready)> foo.1 started at 2014-08-14T20:18:39Z
+2014-08-14T20:18:40Z INFO - [foo.1] -submit_method_id=24149
+2014-08-14T20:18:40Z INFO - [foo.1] -submission succeeded
+2014-08-14T20:18:40Z CRITICAL - [foo.1] -(current:running)> Task job script received signal ERR at 2014-08-14T20:18:39Z
+2014-08-14T20:18:40Z CRITICAL - [foo.1] -(current:running)> foo.1 failed at 2014-08-14T20:18:39Z
+2014-08-14T20:18:40Z INFO - [foo.1] -job failed, retrying in P0Y
+2014-08-14T20:18:41Z INFO - [foo.1] -triggered off []
+2014-08-14T20:18:42Z INFO - [foo.1] -submit_method_id=24192
+2014-08-14T20:18:42Z INFO - [foo.1] -submission succeeded
+2014-08-14T20:18:42Z INFO - [foo.1] -(current:submitted)> foo.1 started at 2014-08-14T20:18:41Z
+2014-08-14T20:18:42Z CRITICAL - [foo.1] -(current:running)> Task job script received signal ERR at 2014-08-14T20:18:41Z
+2014-08-14T20:18:42Z CRITICAL - [foo.1] -(current:running)> foo.1 failed at 2014-08-14T20:18:41Z
+2014-08-14T20:18:42Z INFO - [foo.1] -job failed, retrying in P0Y
+2014-08-14T20:18:43Z INFO - [foo.1] -triggered off []
+2014-08-14T20:18:44Z INFO - [foo.1] -submit_method_id=24245
+2014-08-14T20:18:44Z INFO - [foo.1] -submission succeeded
+2014-08-14T20:18:44Z INFO - [foo.1] -(current:submitted)> foo.1 started at 2014-08-14T20:18:43Z
+2014-08-14T20:18:44Z CRITICAL - [foo.1] -(current:running)> Task job script received signal ERR at 2014-08-14T20:18:43Z
+2014-08-14T20:18:44Z CRITICAL - [foo.1] -(current:running)> foo.1 failed at 2014-08-14T20:18:43Z
+2014-08-14T20:18:44Z INFO - [foo.1] -job failed, retrying in P0Y
+2014-08-14T20:18:45Z INFO - [foo.1] -triggered off []
+2014-08-14T20:18:46Z INFO - [foo.1] -submit_method_id=24288
+2014-08-14T20:18:46Z INFO - [foo.1] -submission succeeded
+2014-08-14T20:18:46Z INFO - [foo.1] -(current:submitted)> foo.1 started at 2014-08-14T20:18:45Z
+2014-08-14T20:18:46Z CRITICAL - [foo.1] -(current:running)> Task job script received signal ERR at 2014-08-14T20:18:46Z
+2014-08-14T20:18:46Z CRITICAL - [foo.1] -(current:running)> foo.1 failed at 2014-08-14T20:18:46Z
+2014-08-14T20:18:46Z INFO - [foo.1] -job failed, retrying in P0Y
+2014-08-14T20:18:47Z INFO - [foo.1] -triggered off []
+2014-08-14T20:18:48Z INFO - [foo.1] -submit_method_id=24332
+2014-08-14T20:18:48Z INFO - [foo.1] -submission succeeded
+2014-08-14T20:18:49Z INFO - [foo.1] -(current:submitted)> foo.1 started at 2014-08-14T20:18:48Z
+2014-08-14T20:18:49Z CRITICAL - [foo.1] -(current:running)> Task job script received signal ERR at 2014-08-14T20:18:48Z
+2014-08-14T20:18:49Z CRITICAL - [foo.1] -(current:running)> foo.1 failed at 2014-08-14T20:18:48Z
+2014-08-14T20:18:49Z INFO - [foo.1] -job failed, retrying in P0Y
+2014-08-14T20:18:50Z INFO - [foo.1] -triggered off []
+2014-08-14T20:18:51Z INFO - [foo.1] -submit_method_id=24375
+2014-08-14T20:18:51Z INFO - [foo.1] -submission succeeded
+2014-08-14T20:18:51Z INFO - [foo.1] -(current:submitted)> foo.1 started at 2014-08-14T20:18:50Z
+2014-08-14T20:18:51Z CRITICAL - [foo.1] -(current:running)> Task job script received signal ERR at 2014-08-14T20:18:50Z
+2014-08-14T20:18:51Z CRITICAL - [foo.1] -(current:running)> foo.1 failed at 2014-08-14T20:18:50Z
+2014-08-14T20:18:51Z INFO - [foo.1] -job failed, retrying in P0Y
+2014-08-14T20:18:52Z INFO - [foo.1] -triggered off []
+2014-08-14T20:18:53Z INFO - [foo.1] -submit_method_id=24418
+2014-08-14T20:18:53Z INFO - [foo.1] -submission succeeded
+2014-08-14T20:18:53Z INFO - [foo.1] -(current:submitted)> foo.1 started at 2014-08-14T20:18:52Z
+2014-08-14T20:18:53Z CRITICAL - [foo.1] -(current:running)> Task job script received signal ERR at 2014-08-14T20:18:52Z
+2014-08-14T20:18:53Z CRITICAL - [foo.1] -(current:running)> foo.1 failed at 2014-08-14T20:18:52Z
+2014-08-14T20:18:53Z INFO - [foo.1] -job failed, retrying in P0Y
+2014-08-14T20:18:54Z INFO - [foo.1] -triggered off []
+2014-08-14T20:18:55Z INFO - [foo.1] -submit_method_id=24461
+2014-08-14T20:18:55Z INFO - [foo.1] -submission succeeded
+2014-08-14T20:18:55Z INFO - [foo.1] -(current:submitted)> foo.1 started at 2014-08-14T20:18:54Z
+2014-08-14T20:18:55Z CRITICAL - [foo.1] -(current:running)> Task job script received signal ERR at 2014-08-14T20:18:54Z
+2014-08-14T20:18:55Z CRITICAL - [foo.1] -(current:running)> foo.1 failed at 2014-08-14T20:18:54Z
+2014-08-14T20:18:55Z INFO - [foo.1] -job failed, retrying in P0Y
+2014-08-14T20:18:56Z INFO - [foo.1] -triggered off []
+2014-08-14T20:18:57Z INFO - [foo.1] -submit_method_id=24504
+2014-08-14T20:18:57Z INFO - [foo.1] -submission succeeded
+2014-08-14T20:18:57Z INFO - [foo.1] -(current:submitted)> foo.1 started at 2014-08-14T20:18:56Z
+2014-08-14T20:18:57Z INFO - [foo.1] -(current:running)> foo.1 succeeded at 2014-08-14T20:18:56Z
+2014-08-14T20:18:58Z INFO - [bar.1] -triggered off ['foo.1']
+2014-08-14T20:18:59Z INFO - [bar.1] -submit_method_id=24547
+2014-08-14T20:18:59Z INFO - [bar.1] -submission succeeded
+2014-08-14T20:18:59Z INFO - [bar.1] -(current:submitted)> bar.1 started at 2014-08-14T20:18:58Z
+2014-08-14T20:18:59Z INFO - [bar.1] -(current:running)> bar.1 succeeded at 2014-08-14T20:18:59Z
+2014-08-14T20:19:00Z INFO - Suite shutting down at 2014-08-14T20:19:00Z
diff --git a/tests/job-submission/01-job-nn-localhost/suite.rc b/tests/job-submission/01-job-nn-localhost/suite.rc
new file mode 100644
index 00000000000..e8de571c259
--- /dev/null
+++ b/tests/job-submission/01-job-nn-localhost/suite.rc
@@ -0,0 +1,28 @@
+#!jinja2
+[cylc]
+ UTC mode = True # Ignore DST
+ [[reference test]]
+ live mode suite timeout = PT10M
+
+[scheduling]
+ [[dependencies]]
+ graph="""foo => bar"""
+
+[runtime]
+{% if "CYLC_TEST_HOST" in environ %}
+ [[root]]
+ [[[remote]]]
+ host = {{environ["CYLC_TEST_HOST"]}}
+{% endif %}
+ [[foo]]
+ command scripting="""
+JOB_LOG_DIR=$(dirname $(dirname "$CYLC_TASK_LOG_ROOT"))
+readlink "$JOB_LOG_DIR/NN" >>what-is-nn.txt
+echo "$JOB_LOG_DIR/NN"
+((CYLC_TASK_TRY_NUMBER == 100))
+"""
+ retry delays=99*PT0S
+ [[bar]]
+ command scripting="""
+printf "%02d\n" {1..100} | cmp -s ../foo/what-is-nn.txt -
+"""
diff --git a/tests/job-submission/02-job-nn-remote-host b/tests/job-submission/02-job-nn-remote-host
new file mode 120000
index 00000000000..6808ad34c93
--- /dev/null
+++ b/tests/job-submission/02-job-nn-remote-host
@@ -0,0 +1 @@
+01-job-nn-localhost
\ No newline at end of file
diff --git a/tests/job-submission/02-job-nn-remote-host.t b/tests/job-submission/02-job-nn-remote-host.t
new file mode 100755
index 00000000000..930d9da4978
--- /dev/null
+++ b/tests/job-submission/02-job-nn-remote-host.t
@@ -0,0 +1,48 @@
+#!/bin/bash
+#C: THIS FILE IS PART OF THE CYLC SUITE ENGINE.
+#C: Copyright (C) 2008-2014 Hilary Oliver, NIWA
+#C:
+#C: This program is free software: you can redistribute it and/or modify
+#C: it under the terms of the GNU General Public License as published by
+#C: the Free Software Foundation, either version 3 of the License, or
+#C: (at your option) any later version.
+#C:
+#C: This program is distributed in the hope that it will be useful,
+#C: but WITHOUT ANY WARRANTY; without even the implied warranty of
+#C: MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+#C: GNU General Public License for more details.
+#C:
+#C: You should have received a copy of the GNU General Public License
+#C: along with this program. If not, see .
+#-------------------------------------------------------------------------------
+# Test remote host job log NN link correctness.
+. $(dirname $0)/test_header
+#-------------------------------------------------------------------------------
+export CYLC_TEST_HOST=$( \
+ cylc get-global-config -i '[test battery]remote host')
+if [[ -z "$CYLC_TEST_HOST" ]]; then
+ skip_all '[test battery]remote host: not defined'
+fi
+set_test_number 2
+#-------------------------------------------------------------------------------
+install_suite "$TEST_NAME_BASE" "$TEST_NAME_BASE"
+set -eu
+if [[ $CYLC_TEST_HOST != 'localhost' ]]; then
+ SSH='ssh -oBatchMode=yes -oConnectTimeout=5'
+ $SSH "$CYLC_TEST_HOST" \
+ "mkdir -p .cylc/$SUITE_NAME/ && cat >.cylc/$SUITE_NAME/passphrase" \
+ <"$TEST_DIR/$SUITE_NAME/passphrase"
+fi
+set +eu
+#-------------------------------------------------------------------------------
+TEST_NAME="$TEST_NAME_BASE-validate"
+run_ok "$TEST_NAME" cylc validate "$SUITE_NAME"
+#-------------------------------------------------------------------------------
+TEST_NAME="$TEST_NAME_BASE-run"
+suite_run_ok "$TEST_NAME" cylc run --reference-test --debug "$SUITE_NAME"
+#-------------------------------------------------------------------------------
+if [[ "$CYLC_TEST_HOST" != 'localhost' ]]; then
+ $SSH "$CYLC_TEST_HOST" "rm -rf '.cylc/$SUITE_NAME' 'cylc-run/$SUITE_NAME'"
+fi
+purge_suite "$SUITE_NAME"
+exit
diff --git a/tests/job-submission/03-job-nn-remote-host-with-shared-fs b/tests/job-submission/03-job-nn-remote-host-with-shared-fs
new file mode 120000
index 00000000000..6808ad34c93
--- /dev/null
+++ b/tests/job-submission/03-job-nn-remote-host-with-shared-fs
@@ -0,0 +1 @@
+01-job-nn-localhost
\ No newline at end of file
diff --git a/tests/job-submission/03-job-nn-remote-host-with-shared-fs.t b/tests/job-submission/03-job-nn-remote-host-with-shared-fs.t
new file mode 100755
index 00000000000..1b9c8d299ad
--- /dev/null
+++ b/tests/job-submission/03-job-nn-remote-host-with-shared-fs.t
@@ -0,0 +1,48 @@
+#!/bin/bash
+#C: THIS FILE IS PART OF THE CYLC SUITE ENGINE.
+#C: Copyright (C) 2008-2014 Hilary Oliver, NIWA
+#C:
+#C: This program is free software: you can redistribute it and/or modify
+#C: it under the terms of the GNU General Public License as published by
+#C: the Free Software Foundation, either version 3 of the License, or
+#C: (at your option) any later version.
+#C:
+#C: This program is distributed in the hope that it will be useful,
+#C: but WITHOUT ANY WARRANTY; without even the implied warranty of
+#C: MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+#C: GNU General Public License for more details.
+#C:
+#C: You should have received a copy of the GNU General Public License
+#C: along with this program. If not, see .
+#-------------------------------------------------------------------------------
+# Test remote host (with shared fs) job log NN link correctness.
+. $(dirname $0)/test_header
+#-------------------------------------------------------------------------------
+export CYLC_TEST_HOST=$( \
+ cylc get-global-config -i '[test battery]remote host with shared fs')
+if [[ -z "$CYLC_TEST_HOST" ]]; then
+ skip_all '[test battery]remote host with shared fs: not defined'
+fi
+set_test_number 2
+#-------------------------------------------------------------------------------
+install_suite "$TEST_NAME_BASE" "$TEST_NAME_BASE"
+set -eu
+if [[ $CYLC_TEST_HOST != 'localhost' ]]; then
+ SSH='ssh -oBatchMode=yes -oConnectTimeout=5'
+ $SSH "$CYLC_TEST_HOST" \
+ "mkdir -p .cylc/$SUITE_NAME/ && cat >.cylc/$SUITE_NAME/passphrase" \
+ <"$TEST_DIR/$SUITE_NAME/passphrase"
+fi
+set +eu
+#-------------------------------------------------------------------------------
+TEST_NAME="$TEST_NAME_BASE-validate"
+run_ok "$TEST_NAME" cylc validate "$SUITE_NAME"
+#-------------------------------------------------------------------------------
+TEST_NAME="$TEST_NAME_BASE-run"
+suite_run_ok "$TEST_NAME" cylc run --reference-test --debug "$SUITE_NAME"
+#-------------------------------------------------------------------------------
+if [[ "$CYLC_TEST_HOST" != 'localhost' ]]; then
+ $SSH "$CYLC_TEST_HOST" "rm -rf '.cylc/$SUITE_NAME' 'cylc-run/$SUITE_NAME'"
+fi
+purge_suite "$SUITE_NAME"
+exit