Optionally spawn to max active cycle points.

lib/cylc/cfgspec/suite.py

@@ -274,6 +274,8 @@ def _coerce_final_cycletime(value, keys, _):
             options=(Calendar.MODES.keys() + ["integer"])),
         'runahead limit': vdr(vtype='cycleinterval'),
         'max active cycle points': vdr(vtype='integer', default=3),
+        'spawn to max active cycle points': vdr(
+            vtype='boolean', default=False),
         'queues': {
             'default': {
                 'limit': vdr(vtype='integer', default=0),

lib/cylc/config.py

@@ -2228,7 +2228,9 @@ def get_taskdef(self, name):
         # initial cycle via restart (accidentally or otherwise).
 
         # Get the taskdef object for generating the task proxy class
-        taskd = TaskDef(name, rtcfg, self.run_mode, self.start_point)
+        taskd = TaskDef(
+            name, rtcfg, self.run_mode, self.start_point,
+            self.cfg['scheduling']['spawn to max active cycle points'])
 
         # TODO - put all taskd.foo items in a single config dict
         # Set cold-start task indicators.

lib/cylc/task_proxy.py

@@ -240,7 +240,12 @@ def __init__(
                 self.point, self.tdef.intercycle_offsets)
             self.identity = TaskID.get(self.tdef.name, self.point)
 
-        self.has_spawned = has_spawned
+        if self.tdef.is_coldstart:
+            # Cylc-5 cold-start tasks have no successor.
+            self.has_spawned = True
+        else:
+            self.has_spawned = has_spawned
+
         self.point_as_seconds = None
 
         # Manually inserted tasks may have a final cycle point set.
@@ -1489,17 +1494,24 @@ def spawn(self, state):
             return None
 
     def ready_to_spawn(self):
-        """Spawn successor on any state beyond submit (except submit-failed, to
-        prevent multi-spawning a task with bad job submission config).
-
-        Allows successive instances to run in parallel, but not out of order.
-
+        """Return True if ready to spawn my next-cycle successor.
+
+        A task proxy is never ready to spawn if:
+           * it has spawned already
+           * its state is submit-failed (avoid running multiple instances
+             of a task with bad job submission config).
+        Otherwise a task proxy is ready to spawn if either:
+           * self.tdef.spawn ahead is True (results in spawning out to max
+             active cycle points), OR
+           * its state is >= submitted (allows successive instances
+             to run concurrently, but not out of order).
         """
-        if self.tdef.is_coldstart:
-            self.has_spawned = True
-        return (not self.has_spawned and
-                self.state.is_greater_than(TASK_STATUS_READY) and
-                self.state.status != TASK_STATUS_SUBMIT_FAILED)
+        if (self.has_spawned or
+                self.state.status == TASK_STATUS_SUBMIT_FAILED):
+            return False
+        else:
+            return (self.tdef.spawn_ahead or
+                    self.state.is_greater_than(TASK_STATUS_READY))
 
     def get_state_summary(self):
         """Return a dict containing the state summary of this task proxy."""

lib/cylc/taskdef.py

@@ -44,15 +44,16 @@ class TaskDef(object):
                  "clocktrigger_offset", "expiration_offset",
                  "namespace_hierarchy", "triggers", "outputs",
                  "external_triggers", "name", "elapsed_times",
-                 "mean_total_elapsed_time"]
+                 "mean_total_elapsed_time", "spawn_ahead"]
 
-    def __init__(self, name, rtcfg, run_mode, start_point):
+    def __init__(self, name, rtcfg, run_mode, start_point, spawn_ahead):
         if not TaskID.is_valid_name(name):
             raise TaskDefError("Illegal task name: %s" % name)
 
         self.run_mode = run_mode
         self.rtconfig = rtcfg
         self.start_point = start_point
+        self.spawn_ahead = spawn_ahead
 
         self.sequences = []
         self.implicit_sequences = []  # Implicit sequences are deprecated.

tests/cylc-get-config/00-simple/section1.stdout

@@ -2,6 +2,7 @@ runahead limit =
 hold after point = 
 max active cycle points = 3
 final cycle point constraints = 
+spawn to max active cycle points = False
 cycling mode = integer
 initial cycle point = 1
 initial cycle point constraints = 

tests/spawn-max/00-basic.t

@@ -0,0 +1,33 @@
+#!/bin/bash
+# THIS FILE IS PART OF THE CYLC SUITE ENGINE.
+# Copyright (C) 2008-2016 NIWA
+# 
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+#-------------------------------------------------------------------------------
+# Test that spawning out to max active cycle points to allows tasks to run out
+# of order - see GitHub #1538, #1904.
+
+. $(dirname $0)/test_header
+set_test_number 2
+
+install_suite $TEST_NAME_BASE $TEST_NAME_BASE
+
+TEST_NAME=$TEST_NAME_BASE-validate
+run_ok $TEST_NAME cylc validate $SUITE_NAME
+
+TEST_NAME=$TEST_NAME_BASE-run
+suite_run_ok $TEST_NAME cylc run --debug --reference-test $SUITE_NAME
+
+purge_suite $SUITE_NAME

tests/spawn-max/00-basic/reference.log

@@ -0,0 +1,11 @@
+2016-08-22T15:03:14+12 INFO - Initial point: 2010
+2016-08-22T15:03:14+12 INFO - Final point: 2013
+2016-08-22T15:03:14+12 INFO - [get_obs.2010] -triggered off []
+2016-08-22T15:03:16+12 INFO - [get_obs.2011] -triggered off []
+2016-08-22T15:03:17+12 INFO - [get_obs.2012] -triggered off []
+2016-08-22T15:03:18+12 INFO - [proc_obs.2011] -triggered off ['get_obs.2011']
+2016-08-22T15:03:18+12 INFO - [get_obs.2013] -triggered off []
+2016-08-22T15:03:20+12 INFO - [proc_obs.2012] -triggered off ['get_obs.2012']
+2016-08-22T15:03:21+12 INFO - [proc_obs.2013] -triggered off ['get_obs.2013']
+2016-08-22T15:03:34+12 INFO - [cleanup.2013] -triggered off ['proc_obs.2013']
+2016-08-22T15:03:36+12 INFO - [proc_obs.2010] -triggered off ['get_obs.2010']

tests/spawn-max/00-basic/suite.rc

@@ -0,0 +1,21 @@
+[cylc]
+    cycle point format = %Y
+    [[reference test]]
+        live mode suite timeout = PT1M
+        expected task failures = get_obs.2010
+[scheduling]
+    initial cycle point = 2010
+    final cycle point = 2013
+    max active cycle points = 4
+    spawn to max active cycle points = True
+    [[dependencies]]
+        [[[P1Y]]]
+            graph = get_obs => proc_obs
+        [[[R1/P0Y]]]
+            graph = proc_obs => cleanup
+[runtime]
+    [[get_obs]]
+        script = """
+((CYLC_TASK_CYCLE_POINT == CYLC_SUITE_INITIAL_CYCLE_POINT)) && false"""
+    [[cleanup]]
+        script = cylc reset -s succeeded $CYLC_SUITE_NAME get_obs:failed

tests/spawn-max/test_header

@@ -0,0 +1 @@
+../lib/bash/test_header