diff --git a/lib/cylc/cfgspec/site.py b/lib/cylc/cfgspec/site.py index 8e5f8736f47..0f07420b9a7 100644 --- a/lib/cylc/cfgspec/site.py +++ b/lib/cylc/cfgspec/site.py @@ -100,6 +100,7 @@ }, 'test battery' : { + 'remote host with shared fs' : vdr( vtype='string' ), 'remote host' : vdr( vtype='string' ), 'directives' : { 'loadleveler host' : vdr( vtype='string' ), diff --git a/lib/cylc/job_submission/background.py b/lib/cylc/job_submission/background.py index e6334b239d1..44443c51797 100644 --- a/lib/cylc/job_submission/background.py +++ b/lib/cylc/job_submission/background.py @@ -15,14 +15,16 @@ #C: #C: You should have received a copy of the GNU General Public License #C: along with this program. If not, see . +"""Implement background job submission.""" -from job_submit import JobSubmit +from cylc.job_submission.job_submit import JobSubmit from cylc.command_env import pr_scripting_sl import os from signal import SIGKILL from subprocess import Popen, PIPE -class background( JobSubmit ): + +class background(JobSubmit): """ Background 'job submission' runs the task directly in the background (with '&') so that we can get the job PID (with $!) but then uses @@ -32,7 +34,7 @@ class background( JobSubmit ): % ssh user@host 'job-script & echo $!; wait' (We have to override the general command templates to achieve this).""" - LOCAL_COMMAND_TEMPLATE = ( "( %(command)s & echo $!; wait )" ) + LOCAL_COMMAND_TEMPLATE = "( %(command)s & echo $!; wait )" REMOTE_COMMAND_TEMPLATE = ( " '" + @@ -40,6 +42,8 @@ class background( JobSubmit ): "; " + # Retry "mkdir" once to avoid race to create log/job/CYCLE/ " (mkdir -p %(jobfile_dir)s || mkdir -p %(jobfile_dir)s)" + + " && rm -f $(dirname %(jobfile_dir)s)/NN" + " && ln -s $(basename %(jobfile_dir)s) $(dirname %(jobfile_dir)s)/NN" " && cat >%(jobfile_path)s.tmp" + " && mv %(jobfile_path)s.tmp %(jobfile_path)s" + " && chmod +x %(jobfile_path)s" + @@ -50,21 +54,20 @@ class background( JobSubmit ): # N.B. The perl command ensures that the job script is executed in its own # process group, which allows the job script and its child processes to be # killed correctly. - COMMAND_TEMPLATE = ("perl -e \"setpgrp(0,0);exec(@ARGV)\" %s " + - "%s 2>%s") + COMMAND_TEMPLATE = ( + "perl -e \"setpgrp(0,0);exec(@ARGV)\" %s %s 2>%s") - def construct_job_submit_command( self ): + def construct_job_submit_command(self): """ Construct a command to submit this job to run. """ command_template = self.job_submit_command_template if not command_template: command_template = self.__class__.COMMAND_TEMPLATE - self.command = command_template % ( self.jobfile_path, - self.stdout_file, - self.stderr_file ) + self.command = command_template % ( + self.jobfile_path, self.stdout_file, self.stderr_file) - def get_id( self, out, err ): + def get_id(self, out, err): """ Extract the job process ID from job submission command output. For background jobs the submission command simply @@ -72,10 +75,12 @@ def get_id( self, out, err ): """ return out.strip() - def kill( self, jid, st_file=None ): + @classmethod + def kill(cls, jid, _=None): """Kill the job.""" os.killpg(int(jid), SIGKILL) - def poll( self, jid ): + @classmethod + def poll(cls, jid): """Return 0 if jid is in the queueing system, 1 otherwise.""" return Popen(["ps", jid], stdout=PIPE).wait() diff --git a/lib/cylc/job_submission/job_submit.py b/lib/cylc/job_submission/job_submit.py index 3d3756016df..11997a43643 100644 --- a/lib/cylc/job_submission/job_submit.py +++ b/lib/cylc/job_submission/job_submit.py @@ -55,6 +55,8 @@ class JobSubmit(object): "; " + # Retry "mkdir" once to avoid race to create log/job/CYCLE/ " (mkdir -p %(jobfile_dir)s || mkdir -p %(jobfile_dir)s)" + + " && rm -f $(dirname %(jobfile_dir)s)/NN" + " && ln -s $(basename %(jobfile_dir)s) $(dirname %(jobfile_dir)s)/NN" " && cat >%(jobfile_path)s.tmp" + " && mv %(jobfile_path)s.tmp %(jobfile_path)s" + " && chmod +x %(jobfile_path)s" + diff --git a/tests/job-submission/00-user.t b/tests/job-submission/00-user.t old mode 100644 new mode 100755 diff --git a/tests/job-submission/01-job-nn-localhost.t b/tests/job-submission/01-job-nn-localhost.t new file mode 100755 index 00000000000..de9ac2a639d --- /dev/null +++ b/tests/job-submission/01-job-nn-localhost.t @@ -0,0 +1,32 @@ +#!/bin/bash +#C: THIS FILE IS PART OF THE CYLC SUITE ENGINE. +#C: Copyright (C) 2008-2014 Hilary Oliver, NIWA +#C: +#C: This program is free software: you can redistribute it and/or modify +#C: it under the terms of the GNU General Public License as published by +#C: the Free Software Foundation, either version 3 of the License, or +#C: (at your option) any later version. +#C: +#C: This program is distributed in the hope that it will be useful, +#C: but WITHOUT ANY WARRANTY; without even the implied warranty of +#C: MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +#C: GNU General Public License for more details. +#C: +#C: You should have received a copy of the GNU General Public License +#C: along with this program. If not, see . +#------------------------------------------------------------------------------- +# Test localhost job log NN link correctness. +. $(dirname $0)/test_header +#------------------------------------------------------------------------------- +set_test_number 2 +#------------------------------------------------------------------------------- +install_suite "$TEST_NAME_BASE" "$TEST_NAME_BASE" +#------------------------------------------------------------------------------- +TEST_NAME="$TEST_NAME_BASE-validate" +run_ok "$TEST_NAME" cylc validate "$SUITE_NAME" +#------------------------------------------------------------------------------- +TEST_NAME="$TEST_NAME_BASE-run" +suite_run_ok "$TEST_NAME" cylc run --reference-test --debug "$SUITE_NAME" +#------------------------------------------------------------------------------- +purge_suite "$SUITE_NAME" +exit diff --git a/tests/job-submission/01-job-nn-localhost/reference.log b/tests/job-submission/01-job-nn-localhost/reference.log new file mode 100644 index 00000000000..0e56bcf442e --- /dev/null +++ b/tests/job-submission/01-job-nn-localhost/reference.log @@ -0,0 +1,711 @@ +2014-08-14T20:15:16Z INFO - port:7766 +2014-08-14T20:15:16Z INFO - Suite starting at 2014-08-14T20:15:16Z +2014-08-14T20:15:16Z INFO - Log event clock: real time +2014-08-14T20:15:16Z INFO - Run mode: live +2014-08-14T20:15:16Z INFO - Initial point: 1 +2014-08-14T20:15:16Z INFO - Final point: 1 +2014-08-14T20:15:16Z INFO - Cold Start 1 +2014-08-14T20:15:16Z INFO - [foo.1] -triggered off [] +2014-08-14T20:15:17Z INFO - [foo.1] -submit_method_id=20205 +2014-08-14T20:15:17Z INFO - [foo.1] -submission succeeded +2014-08-14T20:15:17Z INFO - [foo.1] -(current:submitted)> foo.1 started at 2014-08-14T20:15:16Z +2014-08-14T20:15:17Z CRITICAL - [foo.1] -(current:running)> Task job script received signal ERR at 2014-08-14T20:15:16Z +2014-08-14T20:15:17Z CRITICAL - [foo.1] -(current:running)> foo.1 failed at 2014-08-14T20:15:16Z +2014-08-14T20:15:17Z INFO - [foo.1] -job failed, retrying in P0Y +2014-08-14T20:15:18Z INFO - [foo.1] -triggered off [] +2014-08-14T20:15:19Z INFO - [foo.1] -submit_method_id=20248 +2014-08-14T20:15:19Z INFO - [foo.1] -submission succeeded +2014-08-14T20:15:20Z INFO - [foo.1] -(current:submitted)> foo.1 started at 2014-08-14T20:15:19Z +2014-08-14T20:15:20Z CRITICAL - [foo.1] -(current:running)> Task job script received signal ERR at 2014-08-14T20:15:19Z +2014-08-14T20:15:20Z CRITICAL - [foo.1] -(current:running)> foo.1 failed at 2014-08-14T20:15:19Z +2014-08-14T20:15:20Z INFO - [foo.1] -job failed, retrying in P0Y +2014-08-14T20:15:21Z INFO - [foo.1] -triggered off [] +2014-08-14T20:15:22Z INFO - [foo.1] -submit_method_id=20291 +2014-08-14T20:15:22Z INFO - [foo.1] -submission succeeded +2014-08-14T20:15:22Z INFO - [foo.1] -(current:submitted)> foo.1 started at 2014-08-14T20:15:21Z +2014-08-14T20:15:22Z CRITICAL - [foo.1] -(current:running)> Task job script received signal ERR at 2014-08-14T20:15:21Z +2014-08-14T20:15:22Z CRITICAL - [foo.1] -(current:running)> foo.1 failed at 2014-08-14T20:15:21Z +2014-08-14T20:15:22Z INFO - [foo.1] -job failed, retrying in P0Y +2014-08-14T20:15:23Z INFO - [foo.1] -triggered off [] +2014-08-14T20:15:24Z INFO - [foo.1] -submit_method_id=20343 +2014-08-14T20:15:24Z INFO - [foo.1] -submission succeeded +2014-08-14T20:15:24Z INFO - [foo.1] -(current:submitted)> foo.1 started at 2014-08-14T20:15:23Z +2014-08-14T20:15:24Z CRITICAL - [foo.1] -(current:running)> Task job script received signal ERR at 2014-08-14T20:15:23Z +2014-08-14T20:15:24Z CRITICAL - [foo.1] -(current:running)> foo.1 failed at 2014-08-14T20:15:23Z +2014-08-14T20:15:24Z INFO - [foo.1] -job failed, retrying in P0Y +2014-08-14T20:15:25Z INFO - [foo.1] -triggered off [] +2014-08-14T20:15:26Z INFO - [foo.1] -submit_method_id=20386 +2014-08-14T20:15:26Z INFO - [foo.1] -submission succeeded +2014-08-14T20:15:26Z INFO - [foo.1] -(current:submitted)> foo.1 started at 2014-08-14T20:15:25Z +2014-08-14T20:15:26Z CRITICAL - [foo.1] -(current:running)> Task job script received signal ERR at 2014-08-14T20:15:26Z +2014-08-14T20:15:26Z CRITICAL - [foo.1] -(current:running)> foo.1 failed at 2014-08-14T20:15:26Z +2014-08-14T20:15:26Z INFO - [foo.1] -job failed, retrying in P0Y +2014-08-14T20:15:27Z INFO - [foo.1] -triggered off [] +2014-08-14T20:15:29Z INFO - [foo.1] -submit_method_id=20429 +2014-08-14T20:15:29Z INFO - [foo.1] -submission succeeded +2014-08-14T20:15:29Z INFO - [foo.1] -(current:submitted)> foo.1 started at 2014-08-14T20:15:28Z +2014-08-14T20:15:29Z CRITICAL - [foo.1] -(current:running)> Task job script received signal ERR at 2014-08-14T20:15:28Z +2014-08-14T20:15:29Z CRITICAL - [foo.1] -(current:running)> foo.1 failed at 2014-08-14T20:15:28Z +2014-08-14T20:15:29Z INFO - [foo.1] -job failed, retrying in P0Y +2014-08-14T20:15:30Z INFO - [foo.1] -triggered off [] +2014-08-14T20:15:31Z INFO - [foo.1] -submit_method_id=20472 +2014-08-14T20:15:31Z INFO - [foo.1] -submission succeeded +2014-08-14T20:15:31Z INFO - [foo.1] -(current:submitted)> foo.1 started at 2014-08-14T20:15:30Z +2014-08-14T20:15:31Z CRITICAL - [foo.1] -(current:running)> Task job script received signal ERR at 2014-08-14T20:15:30Z +2014-08-14T20:15:31Z CRITICAL - [foo.1] -(current:running)> foo.1 failed at 2014-08-14T20:15:30Z +2014-08-14T20:15:31Z INFO - [foo.1] -job failed, retrying in P0Y +2014-08-14T20:15:32Z INFO - [foo.1] -triggered off [] +2014-08-14T20:15:33Z INFO - [foo.1] -submit_method_id=20515 +2014-08-14T20:15:33Z INFO - [foo.1] -submission succeeded +2014-08-14T20:15:33Z INFO - [foo.1] -(current:submitted)> foo.1 started at 2014-08-14T20:15:32Z +2014-08-14T20:15:33Z CRITICAL - [foo.1] -(current:running)> Task job script received signal ERR at 2014-08-14T20:15:32Z +2014-08-14T20:15:33Z CRITICAL - [foo.1] -(current:running)> foo.1 failed at 2014-08-14T20:15:32Z +2014-08-14T20:15:33Z INFO - [foo.1] -job failed, retrying in P0Y +2014-08-14T20:15:34Z INFO - [foo.1] -triggered off [] +2014-08-14T20:15:35Z INFO - [foo.1] -submit_method_id=20558 +2014-08-14T20:15:35Z INFO - [foo.1] -submission succeeded +2014-08-14T20:15:35Z INFO - [foo.1] -(current:submitted)> foo.1 started at 2014-08-14T20:15:34Z +2014-08-14T20:15:35Z CRITICAL - [foo.1] -(current:running)> Task job script received signal ERR at 2014-08-14T20:15:35Z +2014-08-14T20:15:35Z CRITICAL - [foo.1] -(current:running)> foo.1 failed at 2014-08-14T20:15:35Z +2014-08-14T20:15:35Z INFO - [foo.1] -job failed, retrying in P0Y +2014-08-14T20:15:36Z INFO - [foo.1] -triggered off [] +2014-08-14T20:15:38Z INFO - [foo.1] -submit_method_id=20601 +2014-08-14T20:15:38Z INFO - [foo.1] -submission succeeded +2014-08-14T20:15:38Z INFO - [foo.1] -(current:submitted)> foo.1 started at 2014-08-14T20:15:37Z +2014-08-14T20:15:38Z CRITICAL - [foo.1] -(current:running)> Task job script received signal ERR at 2014-08-14T20:15:37Z +2014-08-14T20:15:38Z CRITICAL - [foo.1] -(current:running)> foo.1 failed at 2014-08-14T20:15:37Z +2014-08-14T20:15:38Z INFO - [foo.1] -job failed, retrying in P0Y +2014-08-14T20:15:39Z INFO - [foo.1] -triggered off [] +2014-08-14T20:15:40Z INFO - [foo.1] -submit_method_id=20644 +2014-08-14T20:15:40Z INFO - [foo.1] -submission succeeded +2014-08-14T20:15:40Z INFO - [foo.1] -(current:submitted)> foo.1 started at 2014-08-14T20:15:39Z +2014-08-14T20:15:40Z CRITICAL - [foo.1] -(current:running)> Task job script received signal ERR at 2014-08-14T20:15:39Z +2014-08-14T20:15:40Z CRITICAL - [foo.1] -(current:running)> foo.1 failed at 2014-08-14T20:15:39Z +2014-08-14T20:15:40Z INFO - [foo.1] -job failed, retrying in P0Y +2014-08-14T20:15:41Z INFO - [foo.1] -triggered off [] +2014-08-14T20:15:42Z INFO - [foo.1] -submit_method_id=20687 +2014-08-14T20:15:42Z INFO - [foo.1] -submission succeeded +2014-08-14T20:15:42Z INFO - [foo.1] -(current:submitted)> foo.1 started at 2014-08-14T20:15:41Z +2014-08-14T20:15:42Z CRITICAL - [foo.1] -(current:running)> Task job script received signal ERR at 2014-08-14T20:15:41Z +2014-08-14T20:15:42Z CRITICAL - [foo.1] -(current:running)> foo.1 failed at 2014-08-14T20:15:41Z +2014-08-14T20:15:42Z INFO - [foo.1] -job failed, retrying in P0Y +2014-08-14T20:15:43Z INFO - [foo.1] -triggered off [] +2014-08-14T20:15:44Z INFO - [foo.1] -(current:ready)> foo.1 started at 2014-08-14T20:15:43Z +2014-08-14T20:15:44Z CRITICAL - [foo.1] -(current:running)> Task job script received signal ERR at 2014-08-14T20:15:44Z +2014-08-14T20:15:44Z CRITICAL - [foo.1] -(current:running)> foo.1 failed at 2014-08-14T20:15:44Z +2014-08-14T20:15:44Z INFO - [foo.1] -job failed, retrying in P0Y +2014-08-14T20:15:45Z INFO - [foo.1] -submit_method_id=20738 +2014-08-14T20:15:45Z INFO - [foo.1] -submission succeeded +2014-08-14T20:15:45Z INFO - [foo.1] -triggered off [] +2014-08-14T20:15:46Z INFO - [foo.1] -(current:ready)> foo.1 started at 2014-08-14T20:15:45Z +2014-08-14T20:15:46Z CRITICAL - [foo.1] -(current:running)> Task job script received signal ERR at 2014-08-14T20:15:45Z +2014-08-14T20:15:46Z CRITICAL - [foo.1] -(current:running)> foo.1 failed at 2014-08-14T20:15:45Z +2014-08-14T20:15:46Z INFO - [foo.1] -job failed, retrying in P0Y +2014-08-14T20:15:47Z INFO - [foo.1] -submit_method_id=20781 +2014-08-14T20:15:47Z INFO - [foo.1] -submission succeeded +2014-08-14T20:15:47Z INFO - [foo.1] -triggered off [] +2014-08-14T20:15:47Z INFO - [foo.1] -(current:ready)> foo.1 started at 2014-08-14T20:15:47Z +2014-08-14T20:15:47Z CRITICAL - [foo.1] -(current:running)> Task job script received signal ERR at 2014-08-14T20:15:47Z +2014-08-14T20:15:47Z CRITICAL - [foo.1] -(current:running)> foo.1 failed at 2014-08-14T20:15:47Z +2014-08-14T20:15:47Z INFO - [foo.1] -job failed, retrying in P0Y +2014-08-14T20:15:48Z INFO - [foo.1] -submit_method_id=20825 +2014-08-14T20:15:48Z INFO - [foo.1] -submission succeeded +2014-08-14T20:15:48Z INFO - [foo.1] -triggered off [] +2014-08-14T20:15:49Z INFO - [foo.1] -(current:ready)> foo.1 started at 2014-08-14T20:15:49Z +2014-08-14T20:15:50Z INFO - [foo.1] -submit_method_id=20868 +2014-08-14T20:15:50Z INFO - [foo.1] -submission succeeded +2014-08-14T20:15:50Z CRITICAL - [foo.1] -(current:running)> Task job script received signal ERR at 2014-08-14T20:15:49Z +2014-08-14T20:15:50Z CRITICAL - [foo.1] -(current:running)> foo.1 failed at 2014-08-14T20:15:49Z +2014-08-14T20:15:50Z INFO - [foo.1] -job failed, retrying in P0Y +2014-08-14T20:15:51Z INFO - [foo.1] -triggered off [] +2014-08-14T20:15:52Z INFO - [foo.1] -submit_method_id=20911 +2014-08-14T20:15:52Z INFO - [foo.1] -submission succeeded +2014-08-14T20:15:53Z INFO - [foo.1] -(current:submitted)> foo.1 started at 2014-08-14T20:15:51Z +2014-08-14T20:15:53Z CRITICAL - [foo.1] -(current:running)> Task job script received signal ERR at 2014-08-14T20:15:52Z +2014-08-14T20:15:53Z CRITICAL - [foo.1] -(current:running)> foo.1 failed at 2014-08-14T20:15:52Z +2014-08-14T20:15:53Z INFO - [foo.1] -job failed, retrying in P0Y +2014-08-14T20:15:54Z INFO - [foo.1] -triggered off [] +2014-08-14T20:15:54Z INFO - [foo.1] -(current:ready)> foo.1 started at 2014-08-14T20:15:54Z +2014-08-14T20:15:54Z CRITICAL - [foo.1] -(current:running)> Task job script received signal ERR at 2014-08-14T20:15:54Z +2014-08-14T20:15:54Z CRITICAL - [foo.1] -(current:running)> foo.1 failed at 2014-08-14T20:15:54Z +2014-08-14T20:15:54Z INFO - [foo.1] -job failed, retrying in P0Y +2014-08-14T20:15:55Z INFO - [foo.1] -submit_method_id=20954 +2014-08-14T20:15:55Z INFO - [foo.1] -submission succeeded +2014-08-14T20:15:55Z INFO - [foo.1] -triggered off [] +2014-08-14T20:15:56Z INFO - [foo.1] -(current:ready)> foo.1 started at 2014-08-14T20:15:55Z +2014-08-14T20:15:57Z INFO - [foo.1] -submit_method_id=20997 +2014-08-14T20:15:57Z INFO - [foo.1] -submission succeeded +2014-08-14T20:15:57Z CRITICAL - [foo.1] -(current:running)> Task job script received signal ERR at 2014-08-14T20:15:56Z +2014-08-14T20:15:57Z CRITICAL - [foo.1] -(current:running)> foo.1 failed at 2014-08-14T20:15:56Z +2014-08-14T20:15:57Z INFO - [foo.1] -job failed, retrying in P0Y +2014-08-14T20:15:58Z INFO - [foo.1] -triggered off [] +2014-08-14T20:15:59Z INFO - [foo.1] -(current:ready)> foo.1 started at 2014-08-14T20:15:58Z +2014-08-14T20:15:59Z CRITICAL - [foo.1] -(current:running)> Task job script received signal ERR at 2014-08-14T20:15:58Z +2014-08-14T20:15:59Z CRITICAL - [foo.1] -(current:running)> foo.1 failed at 2014-08-14T20:15:58Z +2014-08-14T20:15:59Z INFO - [foo.1] -job failed, retrying in P0Y +2014-08-14T20:16:00Z INFO - [foo.1] -submit_method_id=21040 +2014-08-14T20:16:00Z INFO - [foo.1] -submission succeeded +2014-08-14T20:16:00Z INFO - [foo.1] -triggered off [] +2014-08-14T20:16:01Z INFO - [foo.1] -submit_method_id=21084 +2014-08-14T20:16:01Z INFO - [foo.1] -submission succeeded +2014-08-14T20:16:01Z INFO - [foo.1] -(current:submitted)> foo.1 started at 2014-08-14T20:16:00Z +2014-08-14T20:16:01Z CRITICAL - [foo.1] -(current:running)> Task job script received signal ERR at 2014-08-14T20:16:00Z +2014-08-14T20:16:01Z CRITICAL - [foo.1] -(current:running)> foo.1 failed at 2014-08-14T20:16:00Z +2014-08-14T20:16:01Z INFO - [foo.1] -job failed, retrying in P0Y +2014-08-14T20:16:02Z INFO - [foo.1] -triggered off [] +2014-08-14T20:16:02Z INFO - [foo.1] -(current:ready)> foo.1 started at 2014-08-14T20:16:02Z +2014-08-14T20:16:03Z INFO - [foo.1] -submit_method_id=21127 +2014-08-14T20:16:03Z INFO - [foo.1] -submission succeeded +2014-08-14T20:16:04Z CRITICAL - [foo.1] -(current:running)> Task job script received signal ERR at 2014-08-14T20:16:03Z +2014-08-14T20:16:04Z CRITICAL - [foo.1] -(current:running)> foo.1 failed at 2014-08-14T20:16:03Z +2014-08-14T20:16:04Z INFO - [foo.1] -job failed, retrying in P0Y +2014-08-14T20:16:05Z INFO - [foo.1] -triggered off [] +2014-08-14T20:16:05Z INFO - [foo.1] -(current:ready)> foo.1 started at 2014-08-14T20:16:05Z +2014-08-14T20:16:06Z INFO - [foo.1] -submit_method_id=21170 +2014-08-14T20:16:06Z INFO - [foo.1] -submission succeeded +2014-08-14T20:16:06Z CRITICAL - [foo.1] -(current:running)> Task job script received signal ERR at 2014-08-14T20:16:05Z +2014-08-14T20:16:06Z CRITICAL - [foo.1] -(current:running)> foo.1 failed at 2014-08-14T20:16:05Z +2014-08-14T20:16:06Z INFO - [foo.1] -job failed, retrying in P0Y +2014-08-14T20:16:07Z INFO - [foo.1] -triggered off [] +2014-08-14T20:16:08Z INFO - [foo.1] -submit_method_id=21213 +2014-08-14T20:16:08Z INFO - [foo.1] -submission succeeded +2014-08-14T20:16:08Z INFO - [foo.1] -(current:submitted)> foo.1 started at 2014-08-14T20:16:07Z +2014-08-14T20:16:08Z CRITICAL - [foo.1] -(current:running)> Task job script received signal ERR at 2014-08-14T20:16:08Z +2014-08-14T20:16:08Z CRITICAL - [foo.1] -(current:running)> foo.1 failed at 2014-08-14T20:16:08Z +2014-08-14T20:16:08Z INFO - [foo.1] -job failed, retrying in P0Y +2014-08-14T20:16:09Z INFO - [foo.1] -triggered off [] +2014-08-14T20:16:10Z INFO - [foo.1] -submit_method_id=21256 +2014-08-14T20:16:10Z INFO - [foo.1] -submission succeeded +2014-08-14T20:16:11Z INFO - [foo.1] -(current:submitted)> foo.1 started at 2014-08-14T20:16:10Z +2014-08-14T20:16:11Z CRITICAL - [foo.1] -(current:running)> Task job script received signal ERR at 2014-08-14T20:16:10Z +2014-08-14T20:16:11Z CRITICAL - [foo.1] -(current:running)> foo.1 failed at 2014-08-14T20:16:10Z +2014-08-14T20:16:11Z INFO - [foo.1] -job failed, retrying in P0Y +2014-08-14T20:16:12Z INFO - [foo.1] -triggered off [] +2014-08-14T20:16:12Z INFO - [foo.1] -(current:ready)> foo.1 started at 2014-08-14T20:16:12Z +2014-08-14T20:16:13Z INFO - [foo.1] -submit_method_id=21300 +2014-08-14T20:16:13Z INFO - [foo.1] -submission succeeded +2014-08-14T20:16:15Z CRITICAL - [foo.1] -(current:running)> Task job script received signal ERR at 2014-08-14T20:16:12Z +2014-08-14T20:16:15Z CRITICAL - [foo.1] -(current:running)> foo.1 failed at 2014-08-14T20:16:12Z +2014-08-14T20:16:15Z INFO - [foo.1] -job failed, retrying in P0Y +2014-08-14T20:16:16Z INFO - [foo.1] -triggered off [] +2014-08-14T20:16:17Z INFO - [foo.1] -submit_method_id=21344 +2014-08-14T20:16:17Z INFO - [foo.1] -submission succeeded +2014-08-14T20:16:18Z INFO - [foo.1] -(current:submitted)> foo.1 started at 2014-08-14T20:16:16Z +2014-08-14T20:16:18Z CRITICAL - [foo.1] -(current:running)> Task job script received signal ERR at 2014-08-14T20:16:16Z +2014-08-14T20:16:18Z CRITICAL - [foo.1] -(current:running)> foo.1 failed at 2014-08-14T20:16:16Z +2014-08-14T20:16:18Z INFO - [foo.1] -job failed, retrying in P0Y +2014-08-14T20:16:19Z INFO - [foo.1] -triggered off [] +2014-08-14T20:16:19Z INFO - [foo.1] -(current:ready)> foo.1 started at 2014-08-14T20:16:19Z +2014-08-14T20:16:20Z INFO - [foo.1] -submit_method_id=21388 +2014-08-14T20:16:20Z INFO - [foo.1] -submission succeeded +2014-08-14T20:16:20Z CRITICAL - [foo.1] -(current:running)> Task job script received signal ERR at 2014-08-14T20:16:19Z +2014-08-14T20:16:20Z CRITICAL - [foo.1] -(current:running)> foo.1 failed at 2014-08-14T20:16:19Z +2014-08-14T20:16:20Z INFO - [foo.1] -job failed, retrying in P0Y +2014-08-14T20:16:21Z INFO - [foo.1] -triggered off [] +2014-08-14T20:16:22Z INFO - [foo.1] -submit_method_id=21434 +2014-08-14T20:16:22Z INFO - [foo.1] -submission succeeded +2014-08-14T20:16:22Z INFO - [foo.1] -(current:submitted)> foo.1 started at 2014-08-14T20:16:21Z +2014-08-14T20:16:22Z CRITICAL - [foo.1] -(current:running)> Task job script received signal ERR at 2014-08-14T20:16:21Z +2014-08-14T20:16:22Z CRITICAL - [foo.1] -(current:running)> foo.1 failed at 2014-08-14T20:16:21Z +2014-08-14T20:16:22Z INFO - [foo.1] -job failed, retrying in P0Y +2014-08-14T20:16:23Z INFO - [foo.1] -triggered off [] +2014-08-14T20:16:24Z INFO - [foo.1] -submit_method_id=21477 +2014-08-14T20:16:24Z INFO - [foo.1] -submission succeeded +2014-08-14T20:16:25Z INFO - [foo.1] -(current:submitted)> foo.1 started at 2014-08-14T20:16:23Z +2014-08-14T20:16:25Z CRITICAL - [foo.1] -(current:running)> Task job script received signal ERR at 2014-08-14T20:16:24Z +2014-08-14T20:16:25Z CRITICAL - [foo.1] -(current:running)> foo.1 failed at 2014-08-14T20:16:24Z +2014-08-14T20:16:25Z INFO - [foo.1] -job failed, retrying in P0Y +2014-08-14T20:16:26Z INFO - [foo.1] -triggered off [] +2014-08-14T20:16:27Z INFO - [foo.1] -submit_method_id=21520 +2014-08-14T20:16:27Z INFO - [foo.1] -submission succeeded +2014-08-14T20:16:27Z INFO - [foo.1] -(current:submitted)> foo.1 started at 2014-08-14T20:16:26Z +2014-08-14T20:16:27Z CRITICAL - [foo.1] -(current:running)> Task job script received signal ERR at 2014-08-14T20:16:26Z +2014-08-14T20:16:27Z CRITICAL - [foo.1] -(current:running)> foo.1 failed at 2014-08-14T20:16:26Z +2014-08-14T20:16:27Z INFO - [foo.1] -job failed, retrying in P0Y +2014-08-14T20:16:28Z INFO - [foo.1] -triggered off [] +2014-08-14T20:16:29Z INFO - [foo.1] -submit_method_id=21563 +2014-08-14T20:16:29Z INFO - [foo.1] -submission succeeded +2014-08-14T20:16:29Z INFO - [foo.1] -(current:submitted)> foo.1 started at 2014-08-14T20:16:28Z +2014-08-14T20:16:29Z CRITICAL - [foo.1] -(current:running)> Task job script received signal ERR at 2014-08-14T20:16:28Z +2014-08-14T20:16:29Z CRITICAL - [foo.1] -(current:running)> foo.1 failed at 2014-08-14T20:16:28Z +2014-08-14T20:16:29Z INFO - [foo.1] -job failed, retrying in P0Y +2014-08-14T20:16:30Z INFO - [foo.1] -triggered off [] +2014-08-14T20:16:32Z INFO - [foo.1] -submit_method_id=21606 +2014-08-14T20:16:32Z INFO - [foo.1] -submission succeeded +2014-08-14T20:16:32Z INFO - [foo.1] -(current:submitted)> foo.1 started at 2014-08-14T20:16:31Z +2014-08-14T20:16:32Z CRITICAL - [foo.1] -(current:running)> Task job script received signal ERR at 2014-08-14T20:16:31Z +2014-08-14T20:16:32Z CRITICAL - [foo.1] -(current:running)> foo.1 failed at 2014-08-14T20:16:31Z +2014-08-14T20:16:32Z INFO - [foo.1] -job failed, retrying in P0Y +2014-08-14T20:16:33Z INFO - [foo.1] -triggered off [] +2014-08-14T20:16:34Z INFO - [foo.1] -submit_method_id=21649 +2014-08-14T20:16:34Z INFO - [foo.1] -submission succeeded +2014-08-14T20:16:34Z INFO - [foo.1] -(current:submitted)> foo.1 started at 2014-08-14T20:16:33Z +2014-08-14T20:16:34Z CRITICAL - [foo.1] -(current:running)> Task job script received signal ERR at 2014-08-14T20:16:33Z +2014-08-14T20:16:34Z CRITICAL - [foo.1] -(current:running)> foo.1 failed at 2014-08-14T20:16:33Z +2014-08-14T20:16:34Z INFO - [foo.1] -job failed, retrying in P0Y +2014-08-14T20:16:35Z INFO - [foo.1] -triggered off [] +2014-08-14T20:16:36Z INFO - [foo.1] -submit_method_id=21692 +2014-08-14T20:16:36Z INFO - [foo.1] -submission succeeded +2014-08-14T20:16:36Z INFO - [foo.1] -(current:submitted)> foo.1 started at 2014-08-14T20:16:35Z +2014-08-14T20:16:36Z CRITICAL - [foo.1] -(current:running)> Task job script received signal ERR at 2014-08-14T20:16:35Z +2014-08-14T20:16:36Z CRITICAL - [foo.1] -(current:running)> foo.1 failed at 2014-08-14T20:16:35Z +2014-08-14T20:16:36Z INFO - [foo.1] -job failed, retrying in P0Y +2014-08-14T20:16:37Z INFO - [foo.1] -triggered off [] +2014-08-14T20:16:38Z INFO - [foo.1] -submit_method_id=21735 +2014-08-14T20:16:38Z INFO - [foo.1] -submission succeeded +2014-08-14T20:16:38Z INFO - [foo.1] -(current:submitted)> foo.1 started at 2014-08-14T20:16:37Z +2014-08-14T20:16:38Z CRITICAL - [foo.1] -(current:running)> Task job script received signal ERR at 2014-08-14T20:16:37Z +2014-08-14T20:16:38Z CRITICAL - [foo.1] -(current:running)> foo.1 failed at 2014-08-14T20:16:37Z +2014-08-14T20:16:38Z INFO - [foo.1] -job failed, retrying in P0Y +2014-08-14T20:16:39Z INFO - [foo.1] -triggered off [] +2014-08-14T20:16:40Z INFO - [foo.1] -submit_method_id=21778 +2014-08-14T20:16:40Z INFO - [foo.1] -submission succeeded +2014-08-14T20:16:40Z INFO - [foo.1] -(current:submitted)> foo.1 started at 2014-08-14T20:16:39Z +2014-08-14T20:16:40Z CRITICAL - [foo.1] -(current:running)> Task job script received signal ERR at 2014-08-14T20:16:40Z +2014-08-14T20:16:40Z CRITICAL - [foo.1] -(current:running)> foo.1 failed at 2014-08-14T20:16:40Z +2014-08-14T20:16:40Z INFO - [foo.1] -job failed, retrying in P0Y +2014-08-14T20:16:41Z INFO - [foo.1] -triggered off [] +2014-08-14T20:16:43Z INFO - [foo.1] -submit_method_id=21821 +2014-08-14T20:16:43Z INFO - [foo.1] -submission succeeded +2014-08-14T20:16:43Z INFO - [foo.1] -(current:submitted)> foo.1 started at 2014-08-14T20:16:42Z +2014-08-14T20:16:43Z CRITICAL - [foo.1] -(current:running)> Task job script received signal ERR at 2014-08-14T20:16:42Z +2014-08-14T20:16:43Z CRITICAL - [foo.1] -(current:running)> foo.1 failed at 2014-08-14T20:16:42Z +2014-08-14T20:16:43Z INFO - [foo.1] -job failed, retrying in P0Y +2014-08-14T20:16:44Z INFO - [foo.1] -triggered off [] +2014-08-14T20:16:45Z INFO - [foo.1] -submit_method_id=21864 +2014-08-14T20:16:45Z INFO - [foo.1] -submission succeeded +2014-08-14T20:16:45Z INFO - [foo.1] -(current:submitted)> foo.1 started at 2014-08-14T20:16:44Z +2014-08-14T20:16:45Z CRITICAL - [foo.1] -(current:running)> Task job script received signal ERR at 2014-08-14T20:16:44Z +2014-08-14T20:16:45Z CRITICAL - [foo.1] -(current:running)> foo.1 failed at 2014-08-14T20:16:44Z +2014-08-14T20:16:45Z INFO - [foo.1] -job failed, retrying in P0Y +2014-08-14T20:16:46Z INFO - [foo.1] -triggered off [] +2014-08-14T20:16:47Z INFO - [foo.1] -submit_method_id=21907 +2014-08-14T20:16:47Z INFO - [foo.1] -submission succeeded +2014-08-14T20:16:47Z INFO - [foo.1] -(current:submitted)> foo.1 started at 2014-08-14T20:16:46Z +2014-08-14T20:16:47Z CRITICAL - [foo.1] -(current:running)> Task job script received signal ERR at 2014-08-14T20:16:46Z +2014-08-14T20:16:47Z CRITICAL - [foo.1] -(current:running)> foo.1 failed at 2014-08-14T20:16:46Z +2014-08-14T20:16:47Z INFO - [foo.1] -job failed, retrying in P0Y +2014-08-14T20:16:48Z INFO - [foo.1] -triggered off [] +2014-08-14T20:16:49Z INFO - [foo.1] -submit_method_id=21951 +2014-08-14T20:16:49Z INFO - [foo.1] -submission succeeded +2014-08-14T20:16:49Z INFO - [foo.1] -(current:submitted)> foo.1 started at 2014-08-14T20:16:48Z +2014-08-14T20:16:49Z CRITICAL - [foo.1] -(current:running)> Task job script received signal ERR at 2014-08-14T20:16:48Z +2014-08-14T20:16:49Z CRITICAL - [foo.1] -(current:running)> foo.1 failed at 2014-08-14T20:16:48Z +2014-08-14T20:16:49Z INFO - [foo.1] -job failed, retrying in P0Y +2014-08-14T20:16:50Z INFO - [foo.1] -triggered off [] +2014-08-14T20:16:51Z INFO - [foo.1] -submit_method_id=21994 +2014-08-14T20:16:51Z INFO - [foo.1] -submission succeeded +2014-08-14T20:16:51Z INFO - [foo.1] -(current:submitted)> foo.1 started at 2014-08-14T20:16:50Z +2014-08-14T20:16:51Z CRITICAL - [foo.1] -(current:running)> Task job script received signal ERR at 2014-08-14T20:16:51Z +2014-08-14T20:16:51Z CRITICAL - [foo.1] -(current:running)> foo.1 failed at 2014-08-14T20:16:51Z +2014-08-14T20:16:51Z INFO - [foo.1] -job failed, retrying in P0Y +2014-08-14T20:16:52Z INFO - [foo.1] -triggered off [] +2014-08-14T20:16:53Z INFO - [foo.1] -submit_method_id=22037 +2014-08-14T20:16:53Z INFO - [foo.1] -submission succeeded +2014-08-14T20:16:54Z INFO - [foo.1] -(current:submitted)> foo.1 started at 2014-08-14T20:16:53Z +2014-08-14T20:16:54Z CRITICAL - [foo.1] -(current:running)> Task job script received signal ERR at 2014-08-14T20:16:53Z +2014-08-14T20:16:54Z CRITICAL - [foo.1] -(current:running)> foo.1 failed at 2014-08-14T20:16:53Z +2014-08-14T20:16:54Z INFO - [foo.1] -job failed, retrying in P0Y +2014-08-14T20:16:55Z INFO - [foo.1] -triggered off [] +2014-08-14T20:16:56Z INFO - [foo.1] -submit_method_id=22080 +2014-08-14T20:16:56Z INFO - [foo.1] -submission succeeded +2014-08-14T20:16:56Z INFO - [foo.1] -(current:submitted)> foo.1 started at 2014-08-14T20:16:55Z +2014-08-14T20:16:56Z CRITICAL - [foo.1] -(current:running)> Task job script received signal ERR at 2014-08-14T20:16:55Z +2014-08-14T20:16:56Z CRITICAL - [foo.1] -(current:running)> foo.1 failed at 2014-08-14T20:16:55Z +2014-08-14T20:16:56Z INFO - [foo.1] -job failed, retrying in P0Y +2014-08-14T20:16:57Z INFO - [foo.1] -triggered off [] +2014-08-14T20:16:58Z INFO - [foo.1] -submit_method_id=22123 +2014-08-14T20:16:58Z INFO - [foo.1] -submission succeeded +2014-08-14T20:16:58Z INFO - [foo.1] -(current:submitted)> foo.1 started at 2014-08-14T20:16:57Z +2014-08-14T20:16:58Z CRITICAL - [foo.1] -(current:running)> Task job script received signal ERR at 2014-08-14T20:16:57Z +2014-08-14T20:16:58Z CRITICAL - [foo.1] -(current:running)> foo.1 failed at 2014-08-14T20:16:57Z +2014-08-14T20:16:58Z INFO - [foo.1] -job failed, retrying in P0Y +2014-08-14T20:16:59Z INFO - [foo.1] -triggered off [] +2014-08-14T20:17:00Z INFO - [foo.1] -submit_method_id=22166 +2014-08-14T20:17:00Z INFO - [foo.1] -submission succeeded +2014-08-14T20:17:00Z INFO - [foo.1] -(current:submitted)> foo.1 started at 2014-08-14T20:16:59Z +2014-08-14T20:17:00Z CRITICAL - [foo.1] -(current:running)> Task job script received signal ERR at 2014-08-14T20:16:59Z +2014-08-14T20:17:00Z CRITICAL - [foo.1] -(current:running)> foo.1 failed at 2014-08-14T20:16:59Z +2014-08-14T20:17:00Z INFO - [foo.1] -job failed, retrying in P0Y +2014-08-14T20:17:01Z INFO - [foo.1] -triggered off [] +2014-08-14T20:17:02Z INFO - [foo.1] -submit_method_id=22212 +2014-08-14T20:17:02Z INFO - [foo.1] -submission succeeded +2014-08-14T20:17:02Z INFO - [foo.1] -(current:submitted)> foo.1 started at 2014-08-14T20:17:01Z +2014-08-14T20:17:02Z CRITICAL - [foo.1] -(current:running)> Task job script received signal ERR at 2014-08-14T20:17:01Z +2014-08-14T20:17:02Z CRITICAL - [foo.1] -(current:running)> foo.1 failed at 2014-08-14T20:17:01Z +2014-08-14T20:17:02Z INFO - [foo.1] -job failed, retrying in P0Y +2014-08-14T20:17:03Z INFO - [foo.1] -triggered off [] +2014-08-14T20:17:04Z INFO - [foo.1] -submit_method_id=22255 +2014-08-14T20:17:04Z INFO - [foo.1] -submission succeeded +2014-08-14T20:17:04Z INFO - [foo.1] -(current:submitted)> foo.1 started at 2014-08-14T20:17:03Z +2014-08-14T20:17:04Z CRITICAL - [foo.1] -(current:running)> Task job script received signal ERR at 2014-08-14T20:17:04Z +2014-08-14T20:17:04Z CRITICAL - [foo.1] -(current:running)> foo.1 failed at 2014-08-14T20:17:04Z +2014-08-14T20:17:04Z INFO - [foo.1] -job failed, retrying in P0Y +2014-08-14T20:17:05Z INFO - [foo.1] -triggered off [] +2014-08-14T20:17:07Z INFO - [foo.1] -submit_method_id=22298 +2014-08-14T20:17:07Z INFO - [foo.1] -submission succeeded +2014-08-14T20:17:07Z INFO - [foo.1] -(current:submitted)> foo.1 started at 2014-08-14T20:17:06Z +2014-08-14T20:17:07Z CRITICAL - [foo.1] -(current:running)> Task job script received signal ERR at 2014-08-14T20:17:06Z +2014-08-14T20:17:07Z CRITICAL - [foo.1] -(current:running)> foo.1 failed at 2014-08-14T20:17:06Z +2014-08-14T20:17:07Z INFO - [foo.1] -job failed, retrying in P0Y +2014-08-14T20:17:08Z INFO - [foo.1] -triggered off [] +2014-08-14T20:17:09Z INFO - [foo.1] -submit_method_id=22341 +2014-08-14T20:17:09Z INFO - [foo.1] -submission succeeded +2014-08-14T20:17:09Z INFO - [foo.1] -(current:submitted)> foo.1 started at 2014-08-14T20:17:08Z +2014-08-14T20:17:09Z CRITICAL - [foo.1] -(current:running)> Task job script received signal ERR at 2014-08-14T20:17:08Z +2014-08-14T20:17:09Z CRITICAL - [foo.1] -(current:running)> foo.1 failed at 2014-08-14T20:17:08Z +2014-08-14T20:17:09Z INFO - [foo.1] -job failed, retrying in P0Y +2014-08-14T20:17:10Z INFO - [foo.1] -triggered off [] +2014-08-14T20:17:11Z INFO - [foo.1] -submit_method_id=22384 +2014-08-14T20:17:11Z INFO - [foo.1] -submission succeeded +2014-08-14T20:17:11Z INFO - [foo.1] -(current:submitted)> foo.1 started at 2014-08-14T20:17:10Z +2014-08-14T20:17:11Z CRITICAL - [foo.1] -(current:running)> Task job script received signal ERR at 2014-08-14T20:17:10Z +2014-08-14T20:17:11Z CRITICAL - [foo.1] -(current:running)> foo.1 failed at 2014-08-14T20:17:10Z +2014-08-14T20:17:11Z INFO - [foo.1] -job failed, retrying in P0Y +2014-08-14T20:17:12Z INFO - [foo.1] -triggered off [] +2014-08-14T20:17:13Z INFO - [foo.1] -submit_method_id=22427 +2014-08-14T20:17:13Z INFO - [foo.1] -submission succeeded +2014-08-14T20:17:13Z INFO - [foo.1] -(current:submitted)> foo.1 started at 2014-08-14T20:17:12Z +2014-08-14T20:17:13Z CRITICAL - [foo.1] -(current:running)> Task job script received signal ERR at 2014-08-14T20:17:12Z +2014-08-14T20:17:13Z CRITICAL - [foo.1] -(current:running)> foo.1 failed at 2014-08-14T20:17:12Z +2014-08-14T20:17:13Z INFO - [foo.1] -job failed, retrying in P0Y +2014-08-14T20:17:14Z INFO - [foo.1] -triggered off [] +2014-08-14T20:17:15Z INFO - [foo.1] -submit_method_id=22470 +2014-08-14T20:17:15Z INFO - [foo.1] -submission succeeded +2014-08-14T20:17:15Z INFO - [foo.1] -(current:submitted)> foo.1 started at 2014-08-14T20:17:14Z +2014-08-14T20:17:15Z CRITICAL - [foo.1] -(current:running)> Task job script received signal ERR at 2014-08-14T20:17:14Z +2014-08-14T20:17:15Z CRITICAL - [foo.1] -(current:running)> foo.1 failed at 2014-08-14T20:17:14Z +2014-08-14T20:17:15Z INFO - [foo.1] -job failed, retrying in P0Y +2014-08-14T20:17:16Z INFO - [foo.1] -triggered off [] +2014-08-14T20:17:17Z INFO - [foo.1] -submit_method_id=22513 +2014-08-14T20:17:17Z INFO - [foo.1] -submission succeeded +2014-08-14T20:17:17Z INFO - [foo.1] -(current:submitted)> foo.1 started at 2014-08-14T20:17:16Z +2014-08-14T20:17:17Z CRITICAL - [foo.1] -(current:running)> Task job script received signal ERR at 2014-08-14T20:17:17Z +2014-08-14T20:17:17Z CRITICAL - [foo.1] -(current:running)> foo.1 failed at 2014-08-14T20:17:17Z +2014-08-14T20:17:17Z INFO - [foo.1] -job failed, retrying in P0Y +2014-08-14T20:17:18Z INFO - [foo.1] -triggered off [] +2014-08-14T20:17:20Z INFO - [foo.1] -submit_method_id=22556 +2014-08-14T20:17:20Z INFO - [foo.1] -submission succeeded +2014-08-14T20:17:20Z INFO - [foo.1] -(current:submitted)> foo.1 started at 2014-08-14T20:17:19Z +2014-08-14T20:17:20Z CRITICAL - [foo.1] -(current:running)> Task job script received signal ERR at 2014-08-14T20:17:19Z +2014-08-14T20:17:20Z CRITICAL - [foo.1] -(current:running)> foo.1 failed at 2014-08-14T20:17:19Z +2014-08-14T20:17:20Z INFO - [foo.1] -job failed, retrying in P0Y +2014-08-14T20:17:21Z INFO - [foo.1] -triggered off [] +2014-08-14T20:17:22Z INFO - [foo.1] -submit_method_id=22599 +2014-08-14T20:17:22Z INFO - [foo.1] -submission succeeded +2014-08-14T20:17:22Z INFO - [foo.1] -(current:submitted)> foo.1 started at 2014-08-14T20:17:21Z +2014-08-14T20:17:22Z CRITICAL - [foo.1] -(current:running)> Task job script received signal ERR at 2014-08-14T20:17:21Z +2014-08-14T20:17:22Z CRITICAL - [foo.1] -(current:running)> foo.1 failed at 2014-08-14T20:17:21Z +2014-08-14T20:17:22Z INFO - [foo.1] -job failed, retrying in P0Y +2014-08-14T20:17:23Z INFO - [foo.1] -triggered off [] +2014-08-14T20:17:24Z INFO - [foo.1] -submit_method_id=22643 +2014-08-14T20:17:24Z INFO - [foo.1] -submission succeeded +2014-08-14T20:17:24Z INFO - [foo.1] -(current:submitted)> foo.1 started at 2014-08-14T20:17:23Z +2014-08-14T20:17:24Z CRITICAL - [foo.1] -(current:running)> Task job script received signal ERR at 2014-08-14T20:17:23Z +2014-08-14T20:17:24Z CRITICAL - [foo.1] -(current:running)> foo.1 failed at 2014-08-14T20:17:23Z +2014-08-14T20:17:24Z INFO - [foo.1] -job failed, retrying in P0Y +2014-08-14T20:17:25Z INFO - [foo.1] -triggered off [] +2014-08-14T20:17:26Z INFO - [foo.1] -submit_method_id=22686 +2014-08-14T20:17:26Z INFO - [foo.1] -submission succeeded +2014-08-14T20:17:26Z INFO - [foo.1] -(current:submitted)> foo.1 started at 2014-08-14T20:17:25Z +2014-08-14T20:17:26Z CRITICAL - [foo.1] -(current:running)> Task job script received signal ERR at 2014-08-14T20:17:25Z +2014-08-14T20:17:26Z CRITICAL - [foo.1] -(current:running)> foo.1 failed at 2014-08-14T20:17:25Z +2014-08-14T20:17:26Z INFO - [foo.1] -job failed, retrying in P0Y +2014-08-14T20:17:27Z INFO - [foo.1] -triggered off [] +2014-08-14T20:17:28Z INFO - [foo.1] -submit_method_id=22729 +2014-08-14T20:17:28Z INFO - [foo.1] -submission succeeded +2014-08-14T20:17:28Z INFO - [foo.1] -(current:submitted)> foo.1 started at 2014-08-14T20:17:27Z +2014-08-14T20:17:28Z CRITICAL - [foo.1] -(current:running)> Task job script received signal ERR at 2014-08-14T20:17:27Z +2014-08-14T20:17:28Z CRITICAL - [foo.1] -(current:running)> foo.1 failed at 2014-08-14T20:17:27Z +2014-08-14T20:17:28Z INFO - [foo.1] -job failed, retrying in P0Y +2014-08-14T20:17:29Z INFO - [foo.1] -triggered off [] +2014-08-14T20:17:30Z INFO - [foo.1] -submit_method_id=22772 +2014-08-14T20:17:30Z INFO - [foo.1] -submission succeeded +2014-08-14T20:17:30Z INFO - [foo.1] -(current:submitted)> foo.1 started at 2014-08-14T20:17:29Z +2014-08-14T20:17:30Z CRITICAL - [foo.1] -(current:running)> Task job script received signal ERR at 2014-08-14T20:17:30Z +2014-08-14T20:17:30Z CRITICAL - [foo.1] -(current:running)> foo.1 failed at 2014-08-14T20:17:30Z +2014-08-14T20:17:30Z INFO - [foo.1] -job failed, retrying in P0Y +2014-08-14T20:17:31Z INFO - [foo.1] -triggered off [] +2014-08-14T20:17:32Z INFO - [foo.1] -submit_method_id=22815 +2014-08-14T20:17:32Z INFO - [foo.1] -submission succeeded +2014-08-14T20:17:33Z INFO - [foo.1] -(current:submitted)> foo.1 started at 2014-08-14T20:17:32Z +2014-08-14T20:17:33Z CRITICAL - [foo.1] -(current:running)> Task job script received signal ERR at 2014-08-14T20:17:32Z +2014-08-14T20:17:33Z CRITICAL - [foo.1] -(current:running)> foo.1 failed at 2014-08-14T20:17:32Z +2014-08-14T20:17:33Z INFO - [foo.1] -job failed, retrying in P0Y +2014-08-14T20:17:34Z INFO - [foo.1] -triggered off [] +2014-08-14T20:17:35Z INFO - [foo.1] -submit_method_id=22858 +2014-08-14T20:17:35Z INFO - [foo.1] -submission succeeded +2014-08-14T20:17:35Z INFO - [foo.1] -(current:submitted)> foo.1 started at 2014-08-14T20:17:34Z +2014-08-14T20:17:35Z CRITICAL - [foo.1] -(current:running)> Task job script received signal ERR at 2014-08-14T20:17:34Z +2014-08-14T20:17:35Z CRITICAL - [foo.1] -(current:running)> foo.1 failed at 2014-08-14T20:17:34Z +2014-08-14T20:17:35Z INFO - [foo.1] -job failed, retrying in P0Y +2014-08-14T20:17:36Z INFO - [foo.1] -triggered off [] +2014-08-14T20:17:37Z INFO - [foo.1] -submit_method_id=22901 +2014-08-14T20:17:37Z INFO - [foo.1] -submission succeeded +2014-08-14T20:17:37Z INFO - [foo.1] -(current:submitted)> foo.1 started at 2014-08-14T20:17:36Z +2014-08-14T20:17:37Z CRITICAL - [foo.1] -(current:running)> Task job script received signal ERR at 2014-08-14T20:17:36Z +2014-08-14T20:17:37Z CRITICAL - [foo.1] -(current:running)> foo.1 failed at 2014-08-14T20:17:36Z +2014-08-14T20:17:37Z INFO - [foo.1] -job failed, retrying in P0Y +2014-08-14T20:17:38Z INFO - [foo.1] -triggered off [] +2014-08-14T20:17:39Z INFO - [foo.1] -submit_method_id=22944 +2014-08-14T20:17:39Z INFO - [foo.1] -submission succeeded +2014-08-14T20:17:39Z INFO - [foo.1] -(current:submitted)> foo.1 started at 2014-08-14T20:17:38Z +2014-08-14T20:17:39Z CRITICAL - [foo.1] -(current:running)> Task job script received signal ERR at 2014-08-14T20:17:38Z +2014-08-14T20:17:39Z CRITICAL - [foo.1] -(current:running)> foo.1 failed at 2014-08-14T20:17:38Z +2014-08-14T20:17:39Z INFO - [foo.1] -job failed, retrying in P0Y +2014-08-14T20:17:40Z INFO - [foo.1] -triggered off [] +2014-08-14T20:17:41Z INFO - [foo.1] -submit_method_id=22987 +2014-08-14T20:17:41Z INFO - [foo.1] -submission succeeded +2014-08-14T20:17:41Z INFO - [foo.1] -(current:submitted)> foo.1 started at 2014-08-14T20:17:40Z +2014-08-14T20:17:41Z CRITICAL - [foo.1] -(current:running)> Task job script received signal ERR at 2014-08-14T20:17:40Z +2014-08-14T20:17:41Z CRITICAL - [foo.1] -(current:running)> foo.1 failed at 2014-08-14T20:17:40Z +2014-08-14T20:17:41Z INFO - [foo.1] -job failed, retrying in P0Y +2014-08-14T20:17:42Z INFO - [foo.1] -triggered off [] +2014-08-14T20:17:43Z INFO - [foo.1] -submit_method_id=23030 +2014-08-14T20:17:43Z INFO - [foo.1] -submission succeeded +2014-08-14T20:17:43Z INFO - [foo.1] -(current:submitted)> foo.1 started at 2014-08-14T20:17:42Z +2014-08-14T20:17:43Z CRITICAL - [foo.1] -(current:running)> Task job script received signal ERR at 2014-08-14T20:17:43Z +2014-08-14T20:17:43Z CRITICAL - [foo.1] -(current:running)> foo.1 failed at 2014-08-14T20:17:43Z +2014-08-14T20:17:43Z INFO - [foo.1] -job failed, retrying in P0Y +2014-08-14T20:17:44Z INFO - [foo.1] -triggered off [] +2014-08-14T20:17:46Z INFO - [foo.1] -submit_method_id=23073 +2014-08-14T20:17:46Z INFO - [foo.1] -submission succeeded +2014-08-14T20:17:46Z INFO - [foo.1] -(current:submitted)> foo.1 started at 2014-08-14T20:17:45Z +2014-08-14T20:17:46Z CRITICAL - [foo.1] -(current:running)> Task job script received signal ERR at 2014-08-14T20:17:45Z +2014-08-14T20:17:46Z CRITICAL - [foo.1] -(current:running)> foo.1 failed at 2014-08-14T20:17:45Z +2014-08-14T20:17:46Z INFO - [foo.1] -job failed, retrying in P0Y +2014-08-14T20:17:47Z INFO - [foo.1] -triggered off [] +2014-08-14T20:17:48Z INFO - [foo.1] -submit_method_id=23117 +2014-08-14T20:17:48Z INFO - [foo.1] -submission succeeded +2014-08-14T20:17:48Z INFO - [foo.1] -(current:submitted)> foo.1 started at 2014-08-14T20:17:47Z +2014-08-14T20:17:48Z CRITICAL - [foo.1] -(current:running)> Task job script received signal ERR at 2014-08-14T20:17:47Z +2014-08-14T20:17:48Z CRITICAL - [foo.1] -(current:running)> foo.1 failed at 2014-08-14T20:17:47Z +2014-08-14T20:17:48Z INFO - [foo.1] -job failed, retrying in P0Y +2014-08-14T20:17:49Z INFO - [foo.1] -triggered off [] +2014-08-14T20:17:50Z INFO - [foo.1] -submit_method_id=23160 +2014-08-14T20:17:50Z INFO - [foo.1] -submission succeeded +2014-08-14T20:17:50Z INFO - [foo.1] -(current:submitted)> foo.1 started at 2014-08-14T20:17:49Z +2014-08-14T20:17:50Z CRITICAL - [foo.1] -(current:running)> Task job script received signal ERR at 2014-08-14T20:17:49Z +2014-08-14T20:17:50Z CRITICAL - [foo.1] -(current:running)> foo.1 failed at 2014-08-14T20:17:49Z +2014-08-14T20:17:50Z INFO - [foo.1] -job failed, retrying in P0Y +2014-08-14T20:17:51Z INFO - [foo.1] -triggered off [] +2014-08-14T20:17:52Z INFO - [foo.1] -submit_method_id=23203 +2014-08-14T20:17:52Z INFO - [foo.1] -submission succeeded +2014-08-14T20:17:52Z INFO - [foo.1] -(current:submitted)> foo.1 started at 2014-08-14T20:17:51Z +2014-08-14T20:17:52Z CRITICAL - [foo.1] -(current:running)> Task job script received signal ERR at 2014-08-14T20:17:51Z +2014-08-14T20:17:52Z CRITICAL - [foo.1] -(current:running)> foo.1 failed at 2014-08-14T20:17:51Z +2014-08-14T20:17:52Z INFO - [foo.1] -job failed, retrying in P0Y +2014-08-14T20:17:53Z INFO - [foo.1] -triggered off [] +2014-08-14T20:17:54Z INFO - [foo.1] -submit_method_id=23246 +2014-08-14T20:17:54Z INFO - [foo.1] -submission succeeded +2014-08-14T20:17:54Z INFO - [foo.1] -(current:submitted)> foo.1 started at 2014-08-14T20:17:53Z +2014-08-14T20:17:54Z CRITICAL - [foo.1] -(current:running)> Task job script received signal ERR at 2014-08-14T20:17:53Z +2014-08-14T20:17:54Z CRITICAL - [foo.1] -(current:running)> foo.1 failed at 2014-08-14T20:17:53Z +2014-08-14T20:17:54Z INFO - [foo.1] -job failed, retrying in P0Y +2014-08-14T20:17:55Z INFO - [foo.1] -triggered off [] +2014-08-14T20:17:56Z INFO - [foo.1] -submit_method_id=23289 +2014-08-14T20:17:56Z INFO - [foo.1] -submission succeeded +2014-08-14T20:17:56Z INFO - [foo.1] -(current:submitted)> foo.1 started at 2014-08-14T20:17:55Z +2014-08-14T20:17:56Z CRITICAL - [foo.1] -(current:running)> Task job script received signal ERR at 2014-08-14T20:17:56Z +2014-08-14T20:17:56Z CRITICAL - [foo.1] -(current:running)> foo.1 failed at 2014-08-14T20:17:56Z +2014-08-14T20:17:56Z INFO - [foo.1] -job failed, retrying in P0Y +2014-08-14T20:17:57Z INFO - [foo.1] -triggered off [] +2014-08-14T20:17:58Z INFO - [foo.1] -submit_method_id=23332 +2014-08-14T20:17:58Z INFO - [foo.1] -submission succeeded +2014-08-14T20:17:58Z INFO - [foo.1] -(current:submitted)> foo.1 started at 2014-08-14T20:17:57Z +2014-08-14T20:17:58Z CRITICAL - [foo.1] -(current:running)> Task job script received signal ERR at 2014-08-14T20:17:58Z +2014-08-14T20:17:58Z CRITICAL - [foo.1] -(current:running)> foo.1 failed at 2014-08-14T20:17:58Z +2014-08-14T20:17:58Z INFO - [foo.1] -job failed, retrying in P0Y +2014-08-14T20:18:00Z INFO - [foo.1] -triggered off [] +2014-08-14T20:18:01Z INFO - [foo.1] -submit_method_id=23375 +2014-08-14T20:18:01Z INFO - [foo.1] -submission succeeded +2014-08-14T20:18:01Z INFO - [foo.1] -(current:submitted)> foo.1 started at 2014-08-14T20:18:00Z +2014-08-14T20:18:01Z CRITICAL - [foo.1] -(current:running)> Task job script received signal ERR at 2014-08-14T20:18:00Z +2014-08-14T20:18:01Z CRITICAL - [foo.1] -(current:running)> foo.1 failed at 2014-08-14T20:18:00Z +2014-08-14T20:18:01Z INFO - [foo.1] -job failed, retrying in P0Y +2014-08-14T20:18:02Z INFO - [foo.1] -triggered off [] +2014-08-14T20:18:03Z INFO - [foo.1] -submit_method_id=23418 +2014-08-14T20:18:03Z INFO - [foo.1] -submission succeeded +2014-08-14T20:18:03Z INFO - [foo.1] -(current:submitted)> foo.1 started at 2014-08-14T20:18:02Z +2014-08-14T20:18:03Z CRITICAL - [foo.1] -(current:running)> Task job script received signal ERR at 2014-08-14T20:18:02Z +2014-08-14T20:18:03Z CRITICAL - [foo.1] -(current:running)> foo.1 failed at 2014-08-14T20:18:02Z +2014-08-14T20:18:03Z INFO - [foo.1] -job failed, retrying in P0Y +2014-08-14T20:18:04Z INFO - [foo.1] -triggered off [] +2014-08-14T20:18:05Z INFO - [foo.1] -submit_method_id=23461 +2014-08-14T20:18:05Z INFO - [foo.1] -submission succeeded +2014-08-14T20:18:05Z INFO - [foo.1] -(current:submitted)> foo.1 started at 2014-08-14T20:18:04Z +2014-08-14T20:18:05Z CRITICAL - [foo.1] -(current:running)> Task job script received signal ERR at 2014-08-14T20:18:04Z +2014-08-14T20:18:05Z CRITICAL - [foo.1] -(current:running)> foo.1 failed at 2014-08-14T20:18:04Z +2014-08-14T20:18:05Z INFO - [foo.1] -job failed, retrying in P0Y +2014-08-14T20:18:06Z INFO - [foo.1] -triggered off [] +2014-08-14T20:18:07Z INFO - [foo.1] -submit_method_id=23504 +2014-08-14T20:18:07Z INFO - [foo.1] -submission succeeded +2014-08-14T20:18:07Z INFO - [foo.1] -(current:submitted)> foo.1 started at 2014-08-14T20:18:06Z +2014-08-14T20:18:07Z CRITICAL - [foo.1] -(current:running)> Task job script received signal ERR at 2014-08-14T20:18:06Z +2014-08-14T20:18:07Z CRITICAL - [foo.1] -(current:running)> foo.1 failed at 2014-08-14T20:18:06Z +2014-08-14T20:18:07Z INFO - [foo.1] -job failed, retrying in P0Y +2014-08-14T20:18:08Z INFO - [foo.1] -triggered off [] +2014-08-14T20:18:09Z INFO - [foo.1] -submit_method_id=23547 +2014-08-14T20:18:09Z INFO - [foo.1] -submission succeeded +2014-08-14T20:18:09Z INFO - [foo.1] -(current:submitted)> foo.1 started at 2014-08-14T20:18:08Z +2014-08-14T20:18:09Z CRITICAL - [foo.1] -(current:running)> Task job script received signal ERR at 2014-08-14T20:18:09Z +2014-08-14T20:18:09Z CRITICAL - [foo.1] -(current:running)> foo.1 failed at 2014-08-14T20:18:09Z +2014-08-14T20:18:09Z INFO - [foo.1] -job failed, retrying in P0Y +2014-08-14T20:18:10Z INFO - [foo.1] -triggered off [] +2014-08-14T20:18:11Z INFO - [foo.1] -submit_method_id=23590 +2014-08-14T20:18:11Z INFO - [foo.1] -submission succeeded +2014-08-14T20:18:12Z INFO - [foo.1] -(current:submitted)> foo.1 started at 2014-08-14T20:18:11Z +2014-08-14T20:18:12Z CRITICAL - [foo.1] -(current:running)> Task job script received signal ERR at 2014-08-14T20:18:11Z +2014-08-14T20:18:12Z CRITICAL - [foo.1] -(current:running)> foo.1 failed at 2014-08-14T20:18:11Z +2014-08-14T20:18:12Z INFO - [foo.1] -job failed, retrying in P0Y +2014-08-14T20:18:13Z INFO - [foo.1] -triggered off [] +2014-08-14T20:18:14Z INFO - [foo.1] -submit_method_id=23633 +2014-08-14T20:18:14Z INFO - [foo.1] -submission succeeded +2014-08-14T20:18:14Z INFO - [foo.1] -(current:submitted)> foo.1 started at 2014-08-14T20:18:13Z +2014-08-14T20:18:14Z CRITICAL - [foo.1] -(current:running)> Task job script received signal ERR at 2014-08-14T20:18:13Z +2014-08-14T20:18:14Z CRITICAL - [foo.1] -(current:running)> foo.1 failed at 2014-08-14T20:18:13Z +2014-08-14T20:18:14Z INFO - [foo.1] -job failed, retrying in P0Y +2014-08-14T20:18:15Z INFO - [foo.1] -triggered off [] +2014-08-14T20:18:16Z INFO - [foo.1] -submit_method_id=23676 +2014-08-14T20:18:16Z INFO - [foo.1] -submission succeeded +2014-08-14T20:18:16Z INFO - [foo.1] -(current:submitted)> foo.1 started at 2014-08-14T20:18:15Z +2014-08-14T20:18:16Z CRITICAL - [foo.1] -(current:running)> Task job script received signal ERR at 2014-08-14T20:18:15Z +2014-08-14T20:18:16Z CRITICAL - [foo.1] -(current:running)> foo.1 failed at 2014-08-14T20:18:15Z +2014-08-14T20:18:16Z INFO - [foo.1] -job failed, retrying in P0Y +2014-08-14T20:18:17Z INFO - [foo.1] -triggered off [] +2014-08-14T20:18:18Z INFO - [foo.1] -submit_method_id=23719 +2014-08-14T20:18:18Z INFO - [foo.1] -submission succeeded +2014-08-14T20:18:18Z INFO - [foo.1] -(current:submitted)> foo.1 started at 2014-08-14T20:18:17Z +2014-08-14T20:18:18Z CRITICAL - [foo.1] -(current:running)> Task job script received signal ERR at 2014-08-14T20:18:17Z +2014-08-14T20:18:18Z CRITICAL - [foo.1] -(current:running)> foo.1 failed at 2014-08-14T20:18:17Z +2014-08-14T20:18:18Z INFO - [foo.1] -job failed, retrying in P0Y +2014-08-14T20:18:19Z INFO - [foo.1] -triggered off [] +2014-08-14T20:18:20Z INFO - [foo.1] -submit_method_id=23762 +2014-08-14T20:18:20Z INFO - [foo.1] -submission succeeded +2014-08-14T20:18:20Z INFO - [foo.1] -(current:submitted)> foo.1 started at 2014-08-14T20:18:19Z +2014-08-14T20:18:20Z CRITICAL - [foo.1] -(current:running)> Task job script received signal ERR at 2014-08-14T20:18:19Z +2014-08-14T20:18:20Z CRITICAL - [foo.1] -(current:running)> foo.1 failed at 2014-08-14T20:18:19Z +2014-08-14T20:18:20Z INFO - [foo.1] -job failed, retrying in P0Y +2014-08-14T20:18:21Z INFO - [foo.1] -triggered off [] +2014-08-14T20:18:22Z INFO - [foo.1] -submit_method_id=23805 +2014-08-14T20:18:22Z INFO - [foo.1] -submission succeeded +2014-08-14T20:18:22Z INFO - [foo.1] -(current:submitted)> foo.1 started at 2014-08-14T20:18:21Z +2014-08-14T20:18:22Z CRITICAL - [foo.1] -(current:running)> Task job script received signal ERR at 2014-08-14T20:18:22Z +2014-08-14T20:18:22Z CRITICAL - [foo.1] -(current:running)> foo.1 failed at 2014-08-14T20:18:22Z +2014-08-14T20:18:22Z INFO - [foo.1] -job failed, retrying in P0Y +2014-08-14T20:18:23Z INFO - [foo.1] -triggered off [] +2014-08-14T20:18:24Z INFO - [foo.1] -submit_method_id=23848 +2014-08-14T20:18:24Z INFO - [foo.1] -submission succeeded +2014-08-14T20:18:25Z INFO - [foo.1] -(current:submitted)> foo.1 started at 2014-08-14T20:18:24Z +2014-08-14T20:18:25Z CRITICAL - [foo.1] -(current:running)> Task job script received signal ERR at 2014-08-14T20:18:24Z +2014-08-14T20:18:25Z CRITICAL - [foo.1] -(current:running)> foo.1 failed at 2014-08-14T20:18:24Z +2014-08-14T20:18:25Z INFO - [foo.1] -job failed, retrying in P0Y +2014-08-14T20:18:26Z INFO - [foo.1] -triggered off [] +2014-08-14T20:18:27Z INFO - [foo.1] -submit_method_id=23891 +2014-08-14T20:18:27Z INFO - [foo.1] -submission succeeded +2014-08-14T20:18:27Z INFO - [foo.1] -(current:submitted)> foo.1 started at 2014-08-14T20:18:26Z +2014-08-14T20:18:27Z CRITICAL - [foo.1] -(current:running)> Task job script received signal ERR at 2014-08-14T20:18:26Z +2014-08-14T20:18:27Z CRITICAL - [foo.1] -(current:running)> foo.1 failed at 2014-08-14T20:18:26Z +2014-08-14T20:18:27Z INFO - [foo.1] -job failed, retrying in P0Y +2014-08-14T20:18:28Z INFO - [foo.1] -triggered off [] +2014-08-14T20:18:29Z INFO - [foo.1] -submit_method_id=23934 +2014-08-14T20:18:29Z INFO - [foo.1] -submission succeeded +2014-08-14T20:18:29Z INFO - [foo.1] -(current:submitted)> foo.1 started at 2014-08-14T20:18:28Z +2014-08-14T20:18:29Z CRITICAL - [foo.1] -(current:running)> Task job script received signal ERR at 2014-08-14T20:18:28Z +2014-08-14T20:18:29Z CRITICAL - [foo.1] -(current:running)> foo.1 failed at 2014-08-14T20:18:28Z +2014-08-14T20:18:29Z INFO - [foo.1] -job failed, retrying in P0Y +2014-08-14T20:18:30Z INFO - [foo.1] -triggered off [] +2014-08-14T20:18:31Z INFO - [foo.1] -submit_method_id=23977 +2014-08-14T20:18:31Z INFO - [foo.1] -submission succeeded +2014-08-14T20:18:31Z INFO - [foo.1] -(current:submitted)> foo.1 started at 2014-08-14T20:18:30Z +2014-08-14T20:18:31Z CRITICAL - [foo.1] -(current:running)> Task job script received signal ERR at 2014-08-14T20:18:30Z +2014-08-14T20:18:31Z CRITICAL - [foo.1] -(current:running)> foo.1 failed at 2014-08-14T20:18:30Z +2014-08-14T20:18:31Z INFO - [foo.1] -job failed, retrying in P0Y +2014-08-14T20:18:32Z INFO - [foo.1] -triggered off [] +2014-08-14T20:18:33Z INFO - [foo.1] -submit_method_id=24020 +2014-08-14T20:18:33Z INFO - [foo.1] -submission succeeded +2014-08-14T20:18:33Z INFO - [foo.1] -(current:submitted)> foo.1 started at 2014-08-14T20:18:32Z +2014-08-14T20:18:33Z CRITICAL - [foo.1] -(current:running)> Task job script received signal ERR at 2014-08-14T20:18:32Z +2014-08-14T20:18:33Z CRITICAL - [foo.1] -(current:running)> foo.1 failed at 2014-08-14T20:18:32Z +2014-08-14T20:18:33Z INFO - [foo.1] -job failed, retrying in P0Y +2014-08-14T20:18:34Z INFO - [foo.1] -triggered off [] +2014-08-14T20:18:35Z INFO - [foo.1] -submit_method_id=24063 +2014-08-14T20:18:35Z INFO - [foo.1] -submission succeeded +2014-08-14T20:18:35Z INFO - [foo.1] -(current:submitted)> foo.1 started at 2014-08-14T20:18:34Z +2014-08-14T20:18:35Z CRITICAL - [foo.1] -(current:running)> Task job script received signal ERR at 2014-08-14T20:18:35Z +2014-08-14T20:18:35Z CRITICAL - [foo.1] -(current:running)> foo.1 failed at 2014-08-14T20:18:35Z +2014-08-14T20:18:35Z INFO - [foo.1] -job failed, retrying in P0Y +2014-08-14T20:18:36Z INFO - [foo.1] -triggered off [] +2014-08-14T20:18:37Z INFO - [foo.1] -submit_method_id=24106 +2014-08-14T20:18:37Z INFO - [foo.1] -submission succeeded +2014-08-14T20:18:37Z INFO - [foo.1] -(current:submitted)> foo.1 started at 2014-08-14T20:18:36Z +2014-08-14T20:18:37Z CRITICAL - [foo.1] -(current:running)> Task job script received signal ERR at 2014-08-14T20:18:37Z +2014-08-14T20:18:37Z CRITICAL - [foo.1] -(current:running)> foo.1 failed at 2014-08-14T20:18:37Z +2014-08-14T20:18:37Z INFO - [foo.1] -job failed, retrying in P0Y +2014-08-14T20:18:38Z INFO - [foo.1] -triggered off [] +2014-08-14T20:18:39Z INFO - [foo.1] -(current:ready)> foo.1 started at 2014-08-14T20:18:39Z +2014-08-14T20:18:40Z INFO - [foo.1] -submit_method_id=24149 +2014-08-14T20:18:40Z INFO - [foo.1] -submission succeeded +2014-08-14T20:18:40Z CRITICAL - [foo.1] -(current:running)> Task job script received signal ERR at 2014-08-14T20:18:39Z +2014-08-14T20:18:40Z CRITICAL - [foo.1] -(current:running)> foo.1 failed at 2014-08-14T20:18:39Z +2014-08-14T20:18:40Z INFO - [foo.1] -job failed, retrying in P0Y +2014-08-14T20:18:41Z INFO - [foo.1] -triggered off [] +2014-08-14T20:18:42Z INFO - [foo.1] -submit_method_id=24192 +2014-08-14T20:18:42Z INFO - [foo.1] -submission succeeded +2014-08-14T20:18:42Z INFO - [foo.1] -(current:submitted)> foo.1 started at 2014-08-14T20:18:41Z +2014-08-14T20:18:42Z CRITICAL - [foo.1] -(current:running)> Task job script received signal ERR at 2014-08-14T20:18:41Z +2014-08-14T20:18:42Z CRITICAL - [foo.1] -(current:running)> foo.1 failed at 2014-08-14T20:18:41Z +2014-08-14T20:18:42Z INFO - [foo.1] -job failed, retrying in P0Y +2014-08-14T20:18:43Z INFO - [foo.1] -triggered off [] +2014-08-14T20:18:44Z INFO - [foo.1] -submit_method_id=24245 +2014-08-14T20:18:44Z INFO - [foo.1] -submission succeeded +2014-08-14T20:18:44Z INFO - [foo.1] -(current:submitted)> foo.1 started at 2014-08-14T20:18:43Z +2014-08-14T20:18:44Z CRITICAL - [foo.1] -(current:running)> Task job script received signal ERR at 2014-08-14T20:18:43Z +2014-08-14T20:18:44Z CRITICAL - [foo.1] -(current:running)> foo.1 failed at 2014-08-14T20:18:43Z +2014-08-14T20:18:44Z INFO - [foo.1] -job failed, retrying in P0Y +2014-08-14T20:18:45Z INFO - [foo.1] -triggered off [] +2014-08-14T20:18:46Z INFO - [foo.1] -submit_method_id=24288 +2014-08-14T20:18:46Z INFO - [foo.1] -submission succeeded +2014-08-14T20:18:46Z INFO - [foo.1] -(current:submitted)> foo.1 started at 2014-08-14T20:18:45Z +2014-08-14T20:18:46Z CRITICAL - [foo.1] -(current:running)> Task job script received signal ERR at 2014-08-14T20:18:46Z +2014-08-14T20:18:46Z CRITICAL - [foo.1] -(current:running)> foo.1 failed at 2014-08-14T20:18:46Z +2014-08-14T20:18:46Z INFO - [foo.1] -job failed, retrying in P0Y +2014-08-14T20:18:47Z INFO - [foo.1] -triggered off [] +2014-08-14T20:18:48Z INFO - [foo.1] -submit_method_id=24332 +2014-08-14T20:18:48Z INFO - [foo.1] -submission succeeded +2014-08-14T20:18:49Z INFO - [foo.1] -(current:submitted)> foo.1 started at 2014-08-14T20:18:48Z +2014-08-14T20:18:49Z CRITICAL - [foo.1] -(current:running)> Task job script received signal ERR at 2014-08-14T20:18:48Z +2014-08-14T20:18:49Z CRITICAL - [foo.1] -(current:running)> foo.1 failed at 2014-08-14T20:18:48Z +2014-08-14T20:18:49Z INFO - [foo.1] -job failed, retrying in P0Y +2014-08-14T20:18:50Z INFO - [foo.1] -triggered off [] +2014-08-14T20:18:51Z INFO - [foo.1] -submit_method_id=24375 +2014-08-14T20:18:51Z INFO - [foo.1] -submission succeeded +2014-08-14T20:18:51Z INFO - [foo.1] -(current:submitted)> foo.1 started at 2014-08-14T20:18:50Z +2014-08-14T20:18:51Z CRITICAL - [foo.1] -(current:running)> Task job script received signal ERR at 2014-08-14T20:18:50Z +2014-08-14T20:18:51Z CRITICAL - [foo.1] -(current:running)> foo.1 failed at 2014-08-14T20:18:50Z +2014-08-14T20:18:51Z INFO - [foo.1] -job failed, retrying in P0Y +2014-08-14T20:18:52Z INFO - [foo.1] -triggered off [] +2014-08-14T20:18:53Z INFO - [foo.1] -submit_method_id=24418 +2014-08-14T20:18:53Z INFO - [foo.1] -submission succeeded +2014-08-14T20:18:53Z INFO - [foo.1] -(current:submitted)> foo.1 started at 2014-08-14T20:18:52Z +2014-08-14T20:18:53Z CRITICAL - [foo.1] -(current:running)> Task job script received signal ERR at 2014-08-14T20:18:52Z +2014-08-14T20:18:53Z CRITICAL - [foo.1] -(current:running)> foo.1 failed at 2014-08-14T20:18:52Z +2014-08-14T20:18:53Z INFO - [foo.1] -job failed, retrying in P0Y +2014-08-14T20:18:54Z INFO - [foo.1] -triggered off [] +2014-08-14T20:18:55Z INFO - [foo.1] -submit_method_id=24461 +2014-08-14T20:18:55Z INFO - [foo.1] -submission succeeded +2014-08-14T20:18:55Z INFO - [foo.1] -(current:submitted)> foo.1 started at 2014-08-14T20:18:54Z +2014-08-14T20:18:55Z CRITICAL - [foo.1] -(current:running)> Task job script received signal ERR at 2014-08-14T20:18:54Z +2014-08-14T20:18:55Z CRITICAL - [foo.1] -(current:running)> foo.1 failed at 2014-08-14T20:18:54Z +2014-08-14T20:18:55Z INFO - [foo.1] -job failed, retrying in P0Y +2014-08-14T20:18:56Z INFO - [foo.1] -triggered off [] +2014-08-14T20:18:57Z INFO - [foo.1] -submit_method_id=24504 +2014-08-14T20:18:57Z INFO - [foo.1] -submission succeeded +2014-08-14T20:18:57Z INFO - [foo.1] -(current:submitted)> foo.1 started at 2014-08-14T20:18:56Z +2014-08-14T20:18:57Z INFO - [foo.1] -(current:running)> foo.1 succeeded at 2014-08-14T20:18:56Z +2014-08-14T20:18:58Z INFO - [bar.1] -triggered off ['foo.1'] +2014-08-14T20:18:59Z INFO - [bar.1] -submit_method_id=24547 +2014-08-14T20:18:59Z INFO - [bar.1] -submission succeeded +2014-08-14T20:18:59Z INFO - [bar.1] -(current:submitted)> bar.1 started at 2014-08-14T20:18:58Z +2014-08-14T20:18:59Z INFO - [bar.1] -(current:running)> bar.1 succeeded at 2014-08-14T20:18:59Z +2014-08-14T20:19:00Z INFO - Suite shutting down at 2014-08-14T20:19:00Z diff --git a/tests/job-submission/01-job-nn-localhost/suite.rc b/tests/job-submission/01-job-nn-localhost/suite.rc new file mode 100644 index 00000000000..e8de571c259 --- /dev/null +++ b/tests/job-submission/01-job-nn-localhost/suite.rc @@ -0,0 +1,28 @@ +#!jinja2 +[cylc] + UTC mode = True # Ignore DST + [[reference test]] + live mode suite timeout = PT10M + +[scheduling] + [[dependencies]] + graph="""foo => bar""" + +[runtime] +{% if "CYLC_TEST_HOST" in environ %} + [[root]] + [[[remote]]] + host = {{environ["CYLC_TEST_HOST"]}} +{% endif %} + [[foo]] + command scripting=""" +JOB_LOG_DIR=$(dirname $(dirname "$CYLC_TASK_LOG_ROOT")) +readlink "$JOB_LOG_DIR/NN" >>what-is-nn.txt +echo "$JOB_LOG_DIR/NN" +((CYLC_TASK_TRY_NUMBER == 100)) +""" + retry delays=99*PT0S + [[bar]] + command scripting=""" +printf "%02d\n" {1..100} | cmp -s ../foo/what-is-nn.txt - +""" diff --git a/tests/job-submission/02-job-nn-remote-host b/tests/job-submission/02-job-nn-remote-host new file mode 120000 index 00000000000..6808ad34c93 --- /dev/null +++ b/tests/job-submission/02-job-nn-remote-host @@ -0,0 +1 @@ +01-job-nn-localhost \ No newline at end of file diff --git a/tests/job-submission/02-job-nn-remote-host.t b/tests/job-submission/02-job-nn-remote-host.t new file mode 100755 index 00000000000..930d9da4978 --- /dev/null +++ b/tests/job-submission/02-job-nn-remote-host.t @@ -0,0 +1,48 @@ +#!/bin/bash +#C: THIS FILE IS PART OF THE CYLC SUITE ENGINE. +#C: Copyright (C) 2008-2014 Hilary Oliver, NIWA +#C: +#C: This program is free software: you can redistribute it and/or modify +#C: it under the terms of the GNU General Public License as published by +#C: the Free Software Foundation, either version 3 of the License, or +#C: (at your option) any later version. +#C: +#C: This program is distributed in the hope that it will be useful, +#C: but WITHOUT ANY WARRANTY; without even the implied warranty of +#C: MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +#C: GNU General Public License for more details. +#C: +#C: You should have received a copy of the GNU General Public License +#C: along with this program. If not, see . +#------------------------------------------------------------------------------- +# Test remote host job log NN link correctness. +. $(dirname $0)/test_header +#------------------------------------------------------------------------------- +export CYLC_TEST_HOST=$( \ + cylc get-global-config -i '[test battery]remote host') +if [[ -z "$CYLC_TEST_HOST" ]]; then + skip_all '[test battery]remote host: not defined' +fi +set_test_number 2 +#------------------------------------------------------------------------------- +install_suite "$TEST_NAME_BASE" "$TEST_NAME_BASE" +set -eu +if [[ $CYLC_TEST_HOST != 'localhost' ]]; then + SSH='ssh -oBatchMode=yes -oConnectTimeout=5' + $SSH "$CYLC_TEST_HOST" \ + "mkdir -p .cylc/$SUITE_NAME/ && cat >.cylc/$SUITE_NAME/passphrase" \ + <"$TEST_DIR/$SUITE_NAME/passphrase" +fi +set +eu +#------------------------------------------------------------------------------- +TEST_NAME="$TEST_NAME_BASE-validate" +run_ok "$TEST_NAME" cylc validate "$SUITE_NAME" +#------------------------------------------------------------------------------- +TEST_NAME="$TEST_NAME_BASE-run" +suite_run_ok "$TEST_NAME" cylc run --reference-test --debug "$SUITE_NAME" +#------------------------------------------------------------------------------- +if [[ "$CYLC_TEST_HOST" != 'localhost' ]]; then + $SSH "$CYLC_TEST_HOST" "rm -rf '.cylc/$SUITE_NAME' 'cylc-run/$SUITE_NAME'" +fi +purge_suite "$SUITE_NAME" +exit diff --git a/tests/job-submission/03-job-nn-remote-host-with-shared-fs b/tests/job-submission/03-job-nn-remote-host-with-shared-fs new file mode 120000 index 00000000000..6808ad34c93 --- /dev/null +++ b/tests/job-submission/03-job-nn-remote-host-with-shared-fs @@ -0,0 +1 @@ +01-job-nn-localhost \ No newline at end of file diff --git a/tests/job-submission/03-job-nn-remote-host-with-shared-fs.t b/tests/job-submission/03-job-nn-remote-host-with-shared-fs.t new file mode 100755 index 00000000000..1b9c8d299ad --- /dev/null +++ b/tests/job-submission/03-job-nn-remote-host-with-shared-fs.t @@ -0,0 +1,48 @@ +#!/bin/bash +#C: THIS FILE IS PART OF THE CYLC SUITE ENGINE. +#C: Copyright (C) 2008-2014 Hilary Oliver, NIWA +#C: +#C: This program is free software: you can redistribute it and/or modify +#C: it under the terms of the GNU General Public License as published by +#C: the Free Software Foundation, either version 3 of the License, or +#C: (at your option) any later version. +#C: +#C: This program is distributed in the hope that it will be useful, +#C: but WITHOUT ANY WARRANTY; without even the implied warranty of +#C: MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +#C: GNU General Public License for more details. +#C: +#C: You should have received a copy of the GNU General Public License +#C: along with this program. If not, see . +#------------------------------------------------------------------------------- +# Test remote host (with shared fs) job log NN link correctness. +. $(dirname $0)/test_header +#------------------------------------------------------------------------------- +export CYLC_TEST_HOST=$( \ + cylc get-global-config -i '[test battery]remote host with shared fs') +if [[ -z "$CYLC_TEST_HOST" ]]; then + skip_all '[test battery]remote host with shared fs: not defined' +fi +set_test_number 2 +#------------------------------------------------------------------------------- +install_suite "$TEST_NAME_BASE" "$TEST_NAME_BASE" +set -eu +if [[ $CYLC_TEST_HOST != 'localhost' ]]; then + SSH='ssh -oBatchMode=yes -oConnectTimeout=5' + $SSH "$CYLC_TEST_HOST" \ + "mkdir -p .cylc/$SUITE_NAME/ && cat >.cylc/$SUITE_NAME/passphrase" \ + <"$TEST_DIR/$SUITE_NAME/passphrase" +fi +set +eu +#------------------------------------------------------------------------------- +TEST_NAME="$TEST_NAME_BASE-validate" +run_ok "$TEST_NAME" cylc validate "$SUITE_NAME" +#------------------------------------------------------------------------------- +TEST_NAME="$TEST_NAME_BASE-run" +suite_run_ok "$TEST_NAME" cylc run --reference-test --debug "$SUITE_NAME" +#------------------------------------------------------------------------------- +if [[ "$CYLC_TEST_HOST" != 'localhost' ]]; then + $SSH "$CYLC_TEST_HOST" "rm -rf '.cylc/$SUITE_NAME' 'cylc-run/$SUITE_NAME'" +fi +purge_suite "$SUITE_NAME" +exit