diff --git a/bin/cylc-test-battery b/bin/cylc-test-battery index ee5d47ad0b7..fb54b893d92 100755 --- a/bin/cylc-test-battery +++ b/bin/cylc-test-battery @@ -46,6 +46,10 @@ machines this may result in some test failures due to timeouts intended to catch problems that can prevent a suite from shutting down normally. Use the "-j N" option to change the amount of concurrency. +To output stderr from failed tests to the terminal, export +CYLC_TEST_DEBUG=true before running this command. To perform comparison +tests using `xxdiff -D`, export CYLC_TEST_DEBUG_CMP=true. + For more information see "Reference Tests" in the User Guide. Options: diff --git a/dev/suites/busy/suite.rc b/dev/suites/busy/suite.rc new file mode 100644 index 00000000000..66e6af0831e --- /dev/null +++ b/dev/suites/busy/suite.rc @@ -0,0 +1,49 @@ +#!jinja2 +title = Suite-1000-x3 +description = A suite of 1000 tasks per cycle +[cylc] + UTC mode = True # Ignore DST +[scheduling] + initial cycle point = 20130101T00 + final cycle point = 20130103T00 + runahead limit = PT12H + [[dependencies]] + [[[T00]]] + graph=""" + {% for i0 in range(10) -%} + {% for i1 in range(10) -%} + {% for i2 in range(1) -%} + t{{i0}}{{i1}}{{i2}}9[-P1D] => t{{i0}}{{i1}}{{i2}}0 + t{{i0}}{{i1}}{{i2}}0 => t{{i0}}{{i1}}{{i2}}1 => \ + t{{i0}}{{i1}}{{i2}}2 => t{{i0}}{{i1}}{{i2}}3 + t{{i0}}{{i1}}{{i2}}3 => t{{i0}}{{i1}}{{i2}}4 => \ + t{{i0}}{{i1}}{{i2}}5 => t{{i0}}{{i1}}{{i2}}6 + t{{i0}}{{i1}}{{i2}}6 => t{{i0}}{{i1}}{{i2}}7 => \ + t{{i0}}{{i1}}{{i2}}8 => t{{i0}}{{i1}}{{i2}}9 + {% endfor -%} + {% endfor -%} + {% endfor -%} + """ +[runtime] + [[root]] + command scripting = sleep 1 + [[[job submission]]] + method = at + [[[event hooks]]] + succeeded handler = true + failed handler = true + retry handler = true + submission failed handler = true + submission timeout handler = true + execution timeout handler = true + execution timeout = PT6M + submission timeout = PT1M +{% for i0 in range(10) -%} +{% for i1 in range(10) -%} +{% for i2 in range(1) -%} +{% for i3 in range(10) -%} + [[t{{i0}}{{i1}}{{i2}}{{i3}}]] +{% endfor -%} +{% endfor -%} +{% endfor -%} +{% endfor -%} diff --git a/dev/suites/integer/one/suite.rc b/dev/suites/integer/one/suite.rc index 87c80c264a0..c0592d306a9 100644 --- a/dev/suites/integer/one/suite.rc +++ b/dev/suites/integer/one/suite.rc @@ -2,7 +2,7 @@ [scheduling] initial cycle point = 1 final cycle point = 16 - runahead factor = 4 + runahead limit = 12 cycling mode = integer [[special tasks]] sequential = seq diff --git a/doc/suiterc.tex b/doc/suiterc.tex index 98f7421a81f..9d4569bce92 100644 --- a/doc/suiterc.tex +++ b/doc/suiterc.tex @@ -506,19 +506,55 @@ \subsection{[scheduling]} \end{myitemize} -\subsubsection[runahead limit]{[scheduling] $\rightarrow$ runahead factor} +\subsubsection[runahead limit]{[scheduling] $\rightarrow$ runahead limit} +\label{runahead limit} -The suite runahead limit prevents the fastest tasks in a suite from +Runahead limiting prevents the fastest tasks in a suite from getting too far ahead of the slowest ones, as documented in Section~\ref{RunaheadLimit}. Tasks exceeding the limit are put into a special runahead held state until slower tasks have caught up -sufficiently. The limit computed as this factor of the suite's minimum -cycling interval. +sufficiently. + +The default behaviour is to limit the number of tasks +by only allowing up to \lstinline=N= (default 3) consecutive cycle +points to be active at any time, up-adjusted if necessary for any +future triggering. + +\lstinline=runahead limit= is deprecated in favour of specifying this +maximum number of active cycle points (\ref{max active cycle points}). + \begin{myitemize} - \item {\em type:} integer multiple of the suite's minimum cycling interval - \item {\em default:} $2$ + \item {\em type:} Cycle interval string e.g. \lstinline=PT12H= + for a 12 hour limit under ISO 8601 cycling. + \item {\em default:} (none) \end{myitemize} + +\subsubsection[max active cycle points]{[scheduling] $\rightarrow$ + max active cycle points} +\label{max active cycle points} + +Runahead limiting prevents the fastest tasks in a suite from +getting too far ahead of the slowest ones, as documented in +Section~\ref{RunaheadLimit}. Tasks exceeding the limit are put into +a special runahead held state until slower tasks have caught up +sufficiently. + +The default behaviour is to limit the number of tasks +by only allowing up to \lstinline=N= (default 3) consecutive cycle +points to be active at any time, up-adjusted if necessary for any +future triggering. + +This input allows you to change \lstinline=N=, the maximum +number of consecutive cycle points that can be active at once. This +supersedes \ref{runahead limit}. + +\begin{myitemize} + \item {\em type:} integer + \item {\em default:} 3 +\end{myitemize} + + \subsubsection[{[[}queues{]]}]{[scheduling] $\rightarrow$ [[queues]]} Configuration of internal queues, by which the number of simultaneously diff --git a/examples/future-trigger/suite.rc b/examples/future-trigger/suite.rc index c2ba000e495..c16e66d5d50 100644 --- a/examples/future-trigger/suite.rc +++ b/examples/future-trigger/suite.rc @@ -1,7 +1,7 @@ [scheduling] - initial cycle point = 2010080800 - runahead factor = 4 + initial cycle point = 20100808T00 + runahead limit = P1D [[special tasks]] cold-start = cold [[dependencies]] diff --git a/lib/cylc/cfgspec/suite.py b/lib/cylc/cfgspec/suite.py index 963e39d2171..ea27b2a3c51 100644 --- a/lib/cylc/cfgspec/suite.py +++ b/lib/cylc/cfgspec/suite.py @@ -30,11 +30,22 @@ from isodatetime.data import Calendar, TimePoint from isodatetime.parsers import TimePointParser, TimeIntervalParser - "Define all legal items and values for cylc suite definition files." interval_parser = TimeIntervalParser() +def _coerce_cycleinterval( value, keys, args ): + """Coerce value to a cycle interval.""" + value = _strip_and_unquote( keys, value ) + if value.isdigit(): + # Old runahead limit format. + return value + parser = TimeIntervalParser() + try: + parser.parse(value) + except ValueError: + raise IllegalValueError("interval", keys, value) + return value def _coerce_cycletime( value, keys, args ): """Coerce value to a cycle point.""" @@ -140,6 +151,7 @@ def _coerce_interval_list( value, keys, args, back_comp_unit_factor=1 ): coercers['cycletime'] = _coerce_cycletime coercers['cycletime_format'] = _coerce_cycletime_format coercers['cycletime_time_zone'] = _coerce_cycletime_time_zone +coercers['cycleinterval'] = _coerce_cycleinterval coercers['interval'] = _coerce_interval coercers['interval_minutes'] = lambda *a: _coerce_interval( *a, back_comp_unit_factor=60) @@ -149,6 +161,7 @@ def _coerce_interval_list( value, keys, args, back_comp_unit_factor=1 ): *a, back_comp_unit_factor=60) coercers['interval_seconds_list'] = _coerce_interval_list + SPEC = { 'title' : vdr( vtype='string', default="" ), 'description' : vdr( vtype='string', default="" ), @@ -200,7 +213,8 @@ def _coerce_interval_list( value, keys, args, back_comp_unit_factor=1 ): 'initial cycle point' : vdr(vtype='cycletime'), 'final cycle point' : vdr(vtype='cycletime'), 'cycling mode' : vdr(vtype='string', default=Calendar.MODE_GREGORIAN, options=Calendar.MODES.keys() + ["integer"] ), - 'runahead factor' : vdr(vtype='integer', default=2 ), + 'runahead limit' : vdr(vtype='cycleinterval' ), + 'max active cycle points' : vdr(vtype='integer', default=3), 'queues' : { 'default' : { 'limit' : vdr( vtype='integer', default=0), @@ -352,8 +366,6 @@ def upg( cfg, descr ): ['visualization', 'final cycle time'], ['visualization', 'final cycle point'], converter( lambda x: x, 'changed naming to reflect non-date-time cycling' ) ) - u.deprecate( '6.0.0', ['scheduling', 'runahead limit'], ['scheduling', 'runahead factor'], - converter( lambda x:'2', 'using default runahead factor' )) u.obsolete('6.0.0', ['scheduling', 'dependencies', '__MANY__', 'daemon']) u.obsolete('6.0.0', ['cylc', 'job submission']) u.obsolete('6.0.0', ['cylc', 'event handler submission']) diff --git a/lib/cylc/config.py b/lib/cylc/config.py index 470ced177ef..7d495bbebec 100644 --- a/lib/cylc/config.py +++ b/lib/cylc/config.py @@ -19,9 +19,11 @@ import re, os, sys import taskdef from cylc.cfgspec.suite import get_suitecfg -from cylc.cycling.loader import (get_point, get_interval_cls, +from cylc.cycling.loader import (get_point, + get_interval, get_interval_cls, get_sequence, get_sequence_cls, init_cyclers, INTEGER_CYCLING_TYPE, + ISO8601_CYCLING_TYPE, get_backwards_compatibility_mode) from envvar import check_varnames, expandvars from copy import deepcopy, copy @@ -42,7 +44,9 @@ checking, then construct task proxy objects and graph structures. """ +AUTO_RUNAHEAD_FACTOR = 2 # Factor to apply to the minimum cycling interval. CLOCK_OFFSET_RE = re.compile('(\w+)\s*\(\s*([-+]*\s*[\d.]+)\s*\)') +NUM_RUNAHEAD_SEQ_POINTS = 5 # Number of cycle points to look at per sequence. TRIGGER_TYPES = [ 'submit', 'submit-fail', 'start', 'succeed', 'fail', 'finish' ] try: @@ -113,7 +117,8 @@ def __init__( self, suite, fpath, template_vars=[], self.sequences = [] self.actual_first_ctime = None - self.runahead_limit = None + self.custom_runahead_limit = None + self.max_num_active_cycle_points = None # runtime hierarchy dicts keyed by namespace name: self.runtime = { @@ -296,7 +301,7 @@ def __init__( self, suite, fpath, template_vars=[], if not self.graph_found: raise SuiteConfigError, 'No suite dependency graph defined.' - self.compute_runahead_limit() + self.compute_runahead_limits() self.configure_queues() @@ -319,8 +324,10 @@ def __init__( self, suite, fpath, template_vars=[], self.cfg['visualization']['initial cycle point'] = vict vict_rh = None - if vict and self.runahead_limit: - vict_rh = str( get_point( vict ) + self.runahead_limit ) + v_runahead_limit = ( + self.custom_runahead_limit or self.minimum_runahead_limit) + if vict and v_runahead_limit: + vict_rh = str( get_point( vict ) + v_runahead_limit ) vfct = self.cfg['visualization']['final cycle point'] or vict_rh or vict self.cfg['visualization']['final cycle point'] = vfct @@ -559,43 +566,55 @@ def print_inheritance(self): for item, val in self.runtime[foo].items(): print ' ', ' ', item, val - def compute_runahead_limit( self ): - rfactor = self.cfg['scheduling']['runahead factor'] - if not rfactor: - # no runahead limit! - return - try: - rfactor = int( rfactor ) - except ValueError: - raise SuiteConfigError, "ERROR, illegal runahead limit: " + str(rfactor) + def compute_runahead_limits( self ): + """Extract the custom and the minimum runahead limits.""" - rlim = None - intervals = [] - offsets = [] - for seq in self.sequences: - i = seq.get_interval() - if i: - intervals.append( i ) - offsets.append( seq.get_offset() ) + self.max_num_active_cycle_points = self.cfg['scheduling'][ + 'max active cycle points'] + + limit = self.cfg['scheduling']['runahead limit'] + if (limit is not None and limit.isdigit() and + get_interval_cls().get_null().TYPE == ISO8601_CYCLING_TYPE): + # Backwards-compatibility for raw number of hours. + limit = "PT%sH" % limit + + # The custom runahead limit is None if not user-configured. + self.custom_runahead_limit = get_interval(limit) + + # Find the minimum runahead limit necessary for any future triggers. + self.minimum_runahead_limit = None + + offsets = set() + for name, taskdef in self.taskdefs.items(): + if taskdef.min_intercycle_offset: + offsets.add(taskdef.min_intercycle_offset) - if intervals: - rlim = min( intervals ) * rfactor if offsets: - min_offset = min( offsets ) - if min_offset < get_interval_cls().get_null(): - # future triggers... - if abs(min_offset) >= rlim: - #... that extend past the default rl - # set to offsets plus one minimum interval - rlim = abs(min_offset) + rlim - - self.runahead_limit = rlim - if flags.verbose: - print "Runahead limit:", self.runahead_limit + min_offset = min(offsets) + if min_offset < get_interval_cls().get_null(): + # A negative offset comes from future triggering. + self.minimum_runahead_limit = abs(min_offset) + if (self.custom_runahead_limit is not None and + self.custom_runahead_limit < + self.minimum_runahead_limit): + print >> sys.stderr, ( + ' WARNING, custom runahead limit of %s is less than ' + 'future triggering offset %s: suite may stall.' % + (self.custom_runahead_limit, + self.minimum_runahead_limit) + ) + + def get_custom_runahead_limit( self ): + """Return the custom runahead limit (may be None).""" + return self.custom_runahead_limit + + def get_max_num_active_cycle_points( self ): + """Return the maximum allowed number of pool cycle points.""" + return self.max_num_active_cycle_points - def get_runahead_limit( self ): - # may be None (no cycling tasks) - return self.runahead_limit + def get_minimum_runahead_limit( self ): + """Return the minimum runahead limit to apply.""" + return self.minimum_runahead_limit def get_config( self, args, sparse=False ): return self.pcfg.get( args, sparse ) @@ -1295,6 +1314,8 @@ def generate_taskdefs( self, line, left_nodes, right, ttype, section, seq, offset_seq_map[str(offset)] = seq_offset self.taskdefs[name].add_sequence( seq_offset, is_implicit=True) + if seq_offset not in self.sequences: + self.sequences.append(seq_offset) # We don't handle implicit cycling in new-style cycling. else: self.taskdefs[ name ].add_sequence(seq) @@ -1326,20 +1347,23 @@ def generate_triggers( self, lexpression, left_nodes, right, seq, suicide ): # (GraphNodeError checked above) cycle_point = None lnode = graphnode(left, base_interval=base_interval) + ltaskdef = self.taskdefs[lnode.name] + if lnode.intercycle: - self.taskdefs[lnode.name].intercycle = True - if (self.taskdefs[lnode.name].intercycle_offset is None or ( - lnode.offset is not None and - lnode.offset > - self.taskdefs[lnode.name].intercycle_offset)): - self.taskdefs[lnode.name].intercycle_offset = lnode.offset + ltaskdef.intercycle = True + if (ltaskdef.max_intercycle_offset is None or + lnode.offset > ltaskdef.max_intercycle_offset): + ltaskdef.max_intercycle_offset = lnode.offset + if (ltaskdef.min_intercycle_offset is None or + lnode.offset < ltaskdef.min_intercycle_offset): + ltaskdef.min_intercycle_offset = lnode.offset + if lnode.offset_is_from_ict: last_point = seq.get_stop_point() - first_point = self.taskdefs[lnode.name].ict - lnode.offset + first_point = ltaskdef.ict - lnode.offset if first_point and last_point is not None: - self.taskdefs[lnode.name].intercycle_offset = (last_point - first_point) - else: - self.taskdefs[lnode.name].intercycle_offset = None + offset = (last_point - first_point) + ltaskdef.max_intercycle_offset = offset cycle_point = first_point trigger = self.set_trigger( lnode.name, right, lnode.output, lnode.offset, cycle_point, suicide, seq.get_interval() ) if not trigger: diff --git a/lib/cylc/cycling/__init__.py b/lib/cylc/cycling/__init__.py index f2b57823283..0a99cba0a31 100644 --- a/lib/cylc/cycling/__init__.py +++ b/lib/cylc/cycling/__init__.py @@ -127,7 +127,6 @@ def __abs__( self ): raise NotImplementedError() def __mul__( self, m ): - # the suite runahead limit is a multiple of the smallest sequence interval raise NotImplementedError() def __nonzero__(self): diff --git a/lib/cylc/cycling/integer.py b/lib/cylc/cycling/integer.py index 2b4ff9c6027..c06cfb71ebd 100755 --- a/lib/cylc/cycling/integer.py +++ b/lib/cylc/cycling/integer.py @@ -44,8 +44,7 @@ # absolute, or relative to some context, so a special character 'c' # is used to signify that context is required. '?' can be used for # the period in one-off (no-repeat) expressions, otherwise an arbitrary -# given value will be ignored (an arbitrary interval is not stored as -# it may affect the default runahead limit calculation). +# given value will be ignored. # # 1) REPEAT/START/PERIOD: R[n]/[c]i/Pi # missing n means repeat indefinitely @@ -139,7 +138,6 @@ def __int__(self): return int(self.value.replace("P", "")) def __mul__(self, m): - # the suite runahead limit is a multiple of the smallest sequence interval return IntegerInterval(int(self) * m) def __nonzero__(self): diff --git a/lib/cylc/cycling/iso8601.py b/lib/cylc/cycling/iso8601.py index 97b1287735e..b92616a6a40 100755 --- a/lib/cylc/cycling/iso8601.py +++ b/lib/cylc/cycling/iso8601.py @@ -116,6 +116,9 @@ def sub(self, other): return ISO8601Point( self._iso_point_sub_interval(self.value, other.value)) + def __hash__(self): + return hash(self.value) + @staticmethod @memoize def _iso_point_add(point_string, interval_string): @@ -192,7 +195,6 @@ def __abs__(self): self._iso_interval_abs(self.value, self.NULL_INTERVAL_STRING)) def __mul__(self, m): - # the suite runahead limit is a multiple of the smallest sequence interval return ISO8601Interval(self._iso_interval_mul(self.value, m)) def __nonzero__(self): @@ -384,7 +386,7 @@ def get_next_point_on_sequence(self, point): return result def get_first_point( self, point): - """Return the first point >= to poing, or None if out of bounds.""" + """Return the first point >= to point, or None if out of bounds.""" try: return ISO8601Point(self._cached_first_point_values[point.value]) except KeyError: diff --git a/lib/cylc/scheduler.py b/lib/cylc/scheduler.py index 7d5ed232b7e..7b6490c6d88 100644 --- a/lib/cylc/scheduler.py +++ b/lib/cylc/scheduler.py @@ -944,7 +944,7 @@ def update_state_summary(self): self.paused(), self.will_pause_at(), (self.shut_down_cleanly or self.shut_down_quickly), - self.will_stop_at(), self.pool.runahead_limit, + self.will_stop_at(), self.pool.custom_runahead_limit, self.config.ns_defn_order) def process_resolved(self, tasks): diff --git a/lib/cylc/task_pool.py b/lib/cylc/task_pool.py index 9b151f020a1..485c00af8ff 100644 --- a/lib/cylc/task_pool.py +++ b/lib/cylc/task_pool.py @@ -69,11 +69,17 @@ def __init__( self, suite, db, stop_point, config, pyro, log, run_mode, proc_poo self.reconfiguring = False self.db = db - self.runahead_limit = config.get_runahead_limit() + self.custom_runahead_limit = config.get_custom_runahead_limit() + self.minimum_runahead_limit = config.get_minimum_runahead_limit() + self.max_num_active_cycle_points = ( + config.get_max_num_active_cycle_points()) + self._prev_runahead_base_point = None + self._prev_runahead_sequence_points = None + self.config = config + self.pool = {} self.proc_pool = proc_pool - self.runahead_pool = {} self.myq = {} self.queues = {} @@ -140,7 +146,8 @@ def add_to_runahead_pool( self, itask ): itask.reset_state_held() # add to the runahead pool - self.runahead_pool[itask.id] = itask + self.runahead_pool.setdefault(itask.c_time, {}) + self.runahead_pool[itask.c_time][itask.id] = itask self.rhpool_changed = True return True @@ -158,42 +165,101 @@ def release_runahead_tasks( self ): if not self.runahead_pool: return - runahead_base = None - for itask in self.get_tasks(all=True): - if itask.state.is_currently('failed', 'succeeded'): + limit = self.max_num_active_cycle_points + + points = [] + for point, itasks in sorted( + self.get_tasks_by_point(all=True).items()): + has_unfinished_itasks = False + for itask in itasks: + if not itask.state.is_currently('failed', 'succeeded'): + has_unfinished_itasks = True + break + if not points and not has_unfinished_itasks: + # We need to begin with an unfinished cycle point. continue - if not runahead_base or itask.c_time < runahead_base: - runahead_base = itask.c_time - - # release tasks below the limit - if runahead_base: - for itask in self.runahead_pool.values(): - if not self.runahead_limit or \ - itask.c_time - self.runahead_limit <= runahead_base: - # release task to the appropriate queue - # (no runahead limit implies R1 tasks only) - queue = self.myq[itask.name] - if queue not in self.queues: - self.queues[queue] = {} - self.queues[queue][itask.id] = itask - self.pool_changed = True - flags.pflag = True - itask.log('DEBUG', "released to the task pool" ) - del self.runahead_pool[itask.id] - self.rhpool_changed = True - try: - self.pyro.connect( itask.message_queue, itask.id ) - except Exception, x: - if flags.debug: - raise - print >> sys.stderr, x - self.log.warning( itask.id + ' cannot be added (use --debug and see stderr)' ) - return False + points.append(point) + + if not points: + return + + # Get the earliest point with unfinished tasks. + runahead_base_point = min(points) + + # Get all cycling points possible after the runahead base point. + if (self._prev_runahead_base_point is not None and + runahead_base_point == self._prev_runahead_base_point): + # Cache for speed. + sequence_points = self._prev_runahead_sequence_points + else: + sequence_points = [] + for sequence in self.config.sequences: + point = runahead_base_point + for i in range(limit): + point = sequence.get_next_point(point) + if point is None: + break + sequence_points.append(point) + sequence_points = set(sequence_points) + self._prev_runahead_sequence_points = sequence_points + self._prev_runahead_base_point = runahead_base_point + + points = set(points).union(sequence_points) + + if self.custom_runahead_limit is None: + # Calculate which tasks to release based on a maximum number of + # active cycle points (active meaning non-finished tasks). + latest_allowed_point = sorted(points)[:limit][-1] + if self.minimum_runahead_limit is not None: + latest_allowed_point = max([ + latest_allowed_point, + runahead_base_point + self.minimum_runahead_limit + ]) + else: + # Calculate which tasks to release based on a maximum duration + # measured from the oldest non-finished task. + latest_allowed_point = ( + runahead_base_point + self.custom_runahead_limit) + + for point, itask_id_map in self.runahead_pool.items(): + if point <= latest_allowed_point: + for itask in itask_id_map.values(): + self.release_runahead_task(itask) + + def release_runahead_task(self, itask): + """Release itask to the appropriate queue in the active pool.""" + queue = self.myq[itask.name] + if queue not in self.queues: + self.queues[queue] = {} + self.queues[queue][itask.id] = itask + self.pool.setdefault(itask.c_time, {}) + self.pool[itask.c_time][itask.id] = itask + self.pool_changed = True + flags.pflag = True + itask.log('DEBUG', "released to the task pool" ) + del self.runahead_pool[itask.c_time][itask.id] + if not self.runahead_pool[itask.c_time]: + del self.runahead_pool[itask.c_time] + self.rhpool_changed = True + try: + self.pyro.connect( itask.message_queue, itask.id ) + except Exception, x: + if flags.debug: + raise + print >> sys.stderr, x + self.log.warning( + '%s cannot be added (use --debug and see stderr)' % itask.id) + return False def remove( self, itask, reason=None ): - if itask.id in self.runahead_pool: - del self.runahead_pool[itask.id] + try: + del self.runahead_pool[itask.c_time][itask.id] + except KeyError: + pass + else: + if not self.runahead_pool[itask.c_time]: + del self.runahead_pool[itask.c_time] self.rhpool_changed = True return @@ -210,6 +276,9 @@ def remove( self, itask, reason=None ): # remove from queue queue = self.myq[itask.name] del self.queues[queue][itask.id] + del self.pool[itask.c_time][itask.id] + if not self.pool[itask.c_time]: + del self.pool[itask.c_time] self.pool_changed = True msg = "task proxy removed" if reason: @@ -236,17 +305,33 @@ def get_tasks( self, all=False ): if all: if self.rhpool_changed: self.rhpool_changed = False - self.rhpool_list = self.runahead_pool.values() + self.rhpool_list = [] + for itask_id_maps in self.runahead_pool.values(): + self.rhpool_list.extend(itask_id_maps.values()) return self.rhpool_list + self.pool_list else: return self.pool_list + def get_tasks_by_point( self, all=False ): + """Return a map of task proxies by cycle point.""" + point_itasks = {} + for point, itask_id_map in self.pool.items(): + point_itasks[point] = itask_id_map.values() + + if not all: + return point_itasks + + for point, itask_id_map in self.runahead_pool.items(): + point_itasks.setdefault(point, []) + point_itasks[point].extend(itask_id_map.values()) + return point_itasks def id_exists( self, id ): """Check if task id is in the runahead_pool or pool""" - if id in self.runahead_pool: - return True + for point, itask_ids in self.runahead_pool.items(): + if id in itask_ids: + return True for queue in self.queues: if id in self.queues[queue]: return True @@ -353,16 +438,16 @@ def set_runahead( self, interval=None ): if interval is None: # No limit self.log.warning( "setting NO runahead limit" ) - self.runahead_limit = None + self.custom_runahead_limit = None else: self.log.info( "setting runahead limit to " + str(interval) ) - self.runahead_limit = interval + self.custom_runahead_limit = interval self.release_runahead_tasks() def get_min_ctime( self ): """Return the minimum cycle currently in the pool.""" - cycles = [ t.c_time for t in self.get_tasks() ] + cycles = self.pool.keys() minc = None if cycles: minc = min(cycles) @@ -371,7 +456,7 @@ def get_min_ctime( self ): def get_max_ctime( self ): """Return the maximum cycle currently in the pool.""" - cycles = [ t.c_time for t in self.get_tasks() ] + cycles = self.pool.keys() maxc = None if cycles: maxc = max(cycles) @@ -382,7 +467,10 @@ def reconfigure( self, config, stop_point ): self.reconfiguring = True - self.runahead_limit = config.get_runahead_limit() + self.custom_runahead_limit = config.get_custom_runahead_limit() + self.minimum_runahead_limit = config.get_minimum_runahead_limit() + self.max_num_active_cycle_points = ( + config.get_max_num_active_cycle_points()) self.config = config self.stop_point = stop_point # TODO: Any point in using set_stop_point? diff --git a/lib/cylc/taskdef.py b/lib/cylc/taskdef.py index 87c5b868b84..e33e31d700d 100644 --- a/lib/cylc/taskdef.py +++ b/lib/cylc/taskdef.py @@ -63,7 +63,8 @@ def __init__( self, name, rtcfg, run_mode, ict ): # some defaults self.intercycle = False - self.intercycle_offset = get_interval_cls().get_null() + self.max_intercycle_offset = get_interval_cls().get_null() + self.min_intercycle_offset = get_interval_cls().get_null() self.sequential = False self.cycling = False self.modifiers = [] @@ -270,7 +271,7 @@ def tclass_init( sself, start_point, initial_state, stop_c_time=None, sself.startup = startup sself.submit_num = submit_num sself.exists=exists - sself.intercycle_offset = self.intercycle_offset + sself.max_intercycle_offset = self.max_intercycle_offset if self.cycling and startup: # adjust up to the first on-sequence cycle point @@ -282,10 +283,11 @@ def tclass_init( sself, start_point, initial_state, stop_c_time=None, adjusted.append( adj ) if adjusted: sself.tag = min( adjusted ) - if sself.intercycle_offset is None: + if sself.max_intercycle_offset is None: sself.cleanup_cutoff = None else: - sself.cleanup_cutoff = sself.tag + sself.intercycle_offset + sself.cleanup_cutoff = ( + sself.tag + sself.max_intercycle_offset) sself.id = TaskID.get( sself.name, str(sself.tag) ) else: sself.tag = None @@ -294,10 +296,11 @@ def tclass_init( sself, start_point, initial_state, stop_c_time=None, return else: sself.tag = start_point - if sself.intercycle_offset is None: + if sself.max_intercycle_offset is None: sself.cleanup_cutoff = None else: - sself.cleanup_cutoff = sself.tag + sself.intercycle_offset + sself.cleanup_cutoff = ( + sself.tag + sself.max_intercycle_offset) sself.id = TaskID.get( sself.name, str(sself.tag) ) sself.c_time = sself.tag diff --git a/tests/cyclers/integer1/suite.rc b/tests/cyclers/integer1/suite.rc index eddef1d24f3..390ce67cf59 100644 --- a/tests/cyclers/integer1/suite.rc +++ b/tests/cyclers/integer1/suite.rc @@ -4,7 +4,7 @@ [scheduling] initial cycle point = 1 final cycle point = 16 - runahead factor = 4 + runahead limit = 12 cycling mode = integer [[special tasks]] sequential = seq diff --git a/tests/cylc-get-config/00-simple.t b/tests/cylc-get-config/00-simple.t index 9f4c461dceb..6b84b4179a9 100644 --- a/tests/cylc-get-config/00-simple.t +++ b/tests/cylc-get-config/00-simple.t @@ -91,9 +91,12 @@ TEST_NAME=$TEST_NAME_BASE-section1 run_ok $TEST_NAME cylc get-config --item=[scheduling] $SUITE_NAME cmp_ok $TEST_NAME.stdout - <<__OUT__ cycling mode = integer -runahead factor = 2 +max active cycle points = 3 initial cycle point = 1 final cycle point = 1 +runahead limit = +[[dependencies]] + graph = OPS:finish-all => VAR [[queues]] [[[default]]] limit = 0 @@ -105,8 +108,6 @@ final cycle point = 1 sequential = clock-triggered = exclude at start-up = -[[dependencies]] - graph = OPS:finish-all => VAR __OUT__ cmp_ok $TEST_NAME.stderr - /dev/tty + echo "$TEST_NAME_BASE $TEST_NAME" >/dev/tty + cat "$TEST_NAME.stderr" >/dev/tty + fi } function run_ok() { @@ -157,21 +166,37 @@ function cmp_ok() { local FILE_TEST=$1 local FILE_CONTROL=${2:--} local TEST_NAME=$(basename $FILE_TEST)-cmp-ok - local CMP_COMMAND="cmp" - if [[ -n ${TEST_DEBUG_CMP:-} ]]; then + local CMP_COMMAND="diff" + if [[ -n ${CYLC_TEST_DEBUG_CMP:-} ]]; then CMP_COMMAND="xxdiff -D" fi if $CMP_COMMAND "$FILE_TEST" "$FILE_CONTROL" \ - 1>$TEST_NAME.stdout 2>$TEST_NAME.stderr; then + 1>$TEST_NAME.stderr 2>&1; then ok $TEST_NAME return fi - diff "$FILE_TEST" "$FILE_CONTROL" 2>>$TEST_NAME.stderr mkdir -p $TEST_LOG_DIR cp $TEST_NAME.stderr $TEST_LOG_DIR/$TEST_NAME.stderr fail $TEST_NAME } +function contains_ok() { + local FILE_TEST=$1 + local FILE_CONTROL=${2:--} + local TEST_NAME=$(basename $FILE_TEST)-contains-ok + comm -13 <(sort "$FILE_TEST") <(sort "$FILE_CONTROL") \ + 1>$TEST_NAME.stdout 2>$TEST_NAME.stderr + if [[ -s $TEST_NAME.stdout ]]; then + mkdir -p $TEST_LOG_DIR + echo "Missing lines:" >>$TEST_NAME.stderr + cat $TEST_NAME.stdout >>$TEST_NAME.stderr + cp $TEST_NAME.stderr $TEST_LOG_DIR/$TEST_NAME.stderr + fail $TEST_NAME + return + fi + ok $TEST_NAME +} + function grep_ok() { local BRE=$1 local FILE=$2 @@ -281,7 +306,8 @@ TEST_NAME_BASE=$(basename $0 .t) TEST_SOURCE_DIR=$(cd $(dirname $0) && pwd) TEST_DIR=$(mktemp -d) cd $TEST_DIR -TEST_LOG_DIR=${TMPDIR:-/tmp}/cylc-tests-"$LOGNAME"/$(basename $TEST_SOURCE_DIR) +TEST_LOG_DIR_BASE=${TMPDIR:-/tmp}/cylc-tests-"$LOGNAME"/$(basename $TEST_SOURCE_DIR) +TEST_LOG_DIR=$TEST_LOG_DIR_BASE/$TEST_NAME_BASE-$(date -u +%Y%m%dT%H%M%SZ) SUITE_RUN_FAILS=false # these variables should be moved to site/user config: diff --git a/tests/periodicals/Monthly-reorder/suite.rc b/tests/periodicals/Monthly-reorder/suite.rc index fadc3204497..adf8e115f5d 100644 --- a/tests/periodicals/Monthly-reorder/suite.rc +++ b/tests/periodicals/Monthly-reorder/suite.rc @@ -6,7 +6,7 @@ [scheduling] initial cycle time = 20130130T00 final cycle time = 20130202T00 - runahead factor = 1 # enforce sequential running + runahead limit = PT12H # Enforces sequential behaviour [[dependencies]] # (this triggers a monthly task off the last daily task each month) [[[T00]]] diff --git a/tests/queues/qsize/suite.rc b/tests/queues/qsize/suite.rc index 2bcc4acaac5..81282bb1a03 100644 --- a/tests/queues/qsize/suite.rc +++ b/tests/queues/qsize/suite.rc @@ -30,7 +30,8 @@ fi sleep 1 done - if ((TASKS > 3)); then + if ((TASKS > {{q_size}})); then + echo "[FAIL] more than {{q_size}} tasks running: $TASKS" >&2 false else true diff --git a/tests/reload/runahead/suite.rc b/tests/reload/runahead/suite.rc index 0897d23b459..959a0beb5a8 100644 --- a/tests/reload/runahead/suite.rc +++ b/tests/reload/runahead/suite.rc @@ -4,9 +4,9 @@ timeout = 0.2 abort on timeout = True [scheduling] - runahead factor = 2 # marker initial cycle time = 2010-01-01 final cycle time = 2010-01-05 + runahead limit = PT12H # marker [[dependencies]] [[[T00]]] graph = "foo:fail => reloader" @@ -20,7 +20,7 @@ [[reloader]] command scripting = """ # change the dependencies section name to garbage: -perl -pi -e 's/(runahead factor = )2( # marker)/\1 3\2/' $CYLC_SUITE_DEF_PATH/suite.rc +perl -pi -e 's/(runahead limit = )PT12H( # marker)/\1 PT18H\2/' $CYLC_SUITE_DEF_PATH/suite.rc # reload cylc reload -f $CYLC_SUITE_NAME """ diff --git a/tests/restart/01-broadcast.t b/tests/restart/01-broadcast.t index 9d3556ca342..01168509833 100644 --- a/tests/restart/01-broadcast.t +++ b/tests/restart/01-broadcast.t @@ -91,14 +91,11 @@ tidy.2013092300 : status=waiting, spawned=false __STATE__ grep_ok "broadcast_task|2013092300|0|1|waiting" $TEST_DIR/states-db-pre-restart-2013092300 grep_ok "send_a_broadcast_task|2013092300|1|1|succeeded" $TEST_DIR/states-db-pre-restart-2013092300 -cmp_ok $TEST_DIR/states-db-post-restart-2013092300 <<'__DB_DUMP__' +contains_ok $TEST_DIR/states-db-post-restart-2013092300 <<'__DB_DUMP__' broadcast_task|2013092300|0|1|waiting force_restart|2013092300|1|1|succeeded -force_restart|2013092306|0|1|waiting output_states|2013092300|1|1|running -output_states|2013092306|0|1|waiting send_a_broadcast_task|2013092300|1|1|succeeded -send_a_broadcast_task|2013092306|0|1|waiting tidy|2013092300|0|1|waiting __DB_DUMP__ cmp_ok $TEST_DIR/state-pre-restart-2013092306 <<'__STATE__' @@ -143,33 +140,28 @@ send_a_broadcast_task.2013092312 : status=held, spawned=false tidy.2013092300 : status=succeeded, spawned=true tidy.2013092306 : status=waiting, spawned=false __STATE__ -cmp_ok $TEST_DIR/states-db-pre-restart-2013092306 <<'__DB_DUMP__' +contains_ok $TEST_DIR/states-db-pre-restart-2013092306 <<'__DB_DUMP__' broadcast_task|2013092300|1|1|succeeded broadcast_task|2013092306|0|1|waiting force_restart|2013092300|1|1|succeeded force_restart|2013092306|1|1|running -force_restart|2013092312|0|1|held output_states|2013092300|1|1|succeeded output_states|2013092306|0|1|waiting send_a_broadcast_task|2013092300|1|1|succeeded send_a_broadcast_task|2013092306|1|1|succeeded -send_a_broadcast_task|2013092312|0|1|held tidy|2013092300|1|1|succeeded tidy|2013092306|0|1|waiting __DB_DUMP__ -cmp_ok $TEST_DIR/states-db-post-restart-2013092306 <<'__DB_DUMP__' +contains_ok $TEST_DIR/states-db-post-restart-2013092306 <<'__DB_DUMP__' broadcast_task|2013092300|1|1|succeeded broadcast_task|2013092306|0|1|waiting force_restart|2013092300|1|1|succeeded force_restart|2013092306|1|1|succeeded -force_restart|2013092312|0|1|held output_states|2013092300|1|1|succeeded output_states|2013092306|1|1|running -output_states|2013092312|0|1|held send_a_broadcast_task|2013092300|1|1|succeeded send_a_broadcast_task|2013092306|1|1|succeeded -send_a_broadcast_task|2013092312|0|1|held tidy|2013092300|1|1|succeeded tidy|2013092306|0|1|waiting __DB_DUMP__ @@ -204,19 +196,15 @@ sqlite3 $(cylc get-global-config --print-run-dir)/$SUITE_NAME/cylc-suite.db \ "select name, cycle, submit_num, try_num, status from task_states order by name, cycle;" > $TEST_DIR/states-db -cmp_ok $TEST_DIR/states-db <<'__DB_DUMP__' +contains_ok $TEST_DIR/states-db <<'__DB_DUMP__' broadcast_task|2013092300|1|1|succeeded broadcast_task|2013092306|1|1|succeeded -broadcast_task|2013092312|0|1|held force_restart|2013092300|1|1|succeeded force_restart|2013092306|1|1|succeeded -force_restart|2013092312|0|1|held output_states|2013092300|1|1|succeeded output_states|2013092306|1|1|succeeded -output_states|2013092312|0|1|held send_a_broadcast_task|2013092300|1|1|succeeded send_a_broadcast_task|2013092306|1|1|succeeded -send_a_broadcast_task|2013092312|0|1|held tidy|2013092300|1|1|succeeded tidy|2013092306|1|1|succeeded __DB_DUMP__ diff --git a/tests/restart/02-failed.t b/tests/restart/02-failed.t index 50e5e0a6de7..203f69a026c 100644 --- a/tests/restart/02-failed.t +++ b/tests/restart/02-failed.t @@ -35,7 +35,7 @@ fi export TEST_DIR #------------------------------------------------------------------------------- TEST_NAME=$TEST_NAME_BASE-validate -run_ok $TEST_NAME cylc validate $SUITE_NAME +run_ok $TEST_NAME cylc validate -v $SUITE_NAME cmp_ok "$TEST_NAME.stderr" $TEST_DIR/states-db -cmp_ok $TEST_DIR/states-db <<'__DB_DUMP__' +contains_ok $TEST_DIR/states-db <<'__DB_DUMP__' failed_task|2013092300|1|1|failed failed_task|2013092306|1|1|failed -failed_task|2013092312|0|1|held force_restart|2013092300|1|1|succeeded force_restart|2013092306|1|1|succeeded -force_restart|2013092312|0|1|held output_states|2013092300|1|1|succeeded output_states|2013092306|1|1|succeeded -output_states|2013092312|0|1|held tidy|2013092300|1|1|succeeded tidy|2013092306|1|1|succeeded -tidy|2013092312|0|1|held __DB_DUMP__ #------------------------------------------------------------------------------- purge_suite $SUITE_NAME diff --git a/tests/restart/03-retrying.t b/tests/restart/03-retrying.t index b4c50b31bf3..79d20f71c24 100644 --- a/tests/restart/03-retrying.t +++ b/tests/restart/03-retrying.t @@ -76,24 +76,17 @@ retrying_task.2013092306 : status=waiting, spawned=false tidy.2013092300 : status=waiting, spawned=false __STATE__ grep_ok "retrying_task|2013092300|1|2|retrying" $TEST_DIR/states-db-pre-restart-2013092300 -cmp_ok $TEST_DIR/states-db-post-restart-2013092300 <<'__DB_DUMP__' +contains_ok $TEST_DIR/states-db-post-restart-2013092300 <<'__DB_DUMP__' force_restart|2013092300|1|1|succeeded -force_restart|2013092306|0|1|waiting output_states|2013092300|1|1|running -output_states|2013092306|0|1|waiting retrying_task|2013092300|2|2|retrying -retrying_task|2013092306|0|1|waiting tidy|2013092300|0|1|waiting __DB_DUMP__ -cmp_ok $TEST_DIR/states-db-tidy-2013092300 <<'__DB_DUMP__' +contains_ok $TEST_DIR/states-db-tidy-2013092300 <<'__DB_DUMP__' force_restart|2013092300|1|1|succeeded -force_restart|2013092306|0|1|waiting output_states|2013092300|1|1|succeeded -output_states|2013092306|0|1|waiting retrying_task|2013092300|4|3|succeeded -retrying_task|2013092306|0|1|waiting tidy|2013092300|1|1|running -tidy|2013092306|0|1|waiting __DB_DUMP__ cmp_ok $TEST_DIR/state-pre-restart-2013092306 <<'__STATE__' run mode : live @@ -110,28 +103,23 @@ retrying_task.2013092312 : status=held, spawned=false tidy.2013092300 : status=succeeded, spawned=true tidy.2013092306 : status=waiting, spawned=false __STATE__ -cmp_ok $TEST_DIR/states-db-pre-restart-2013092306 <<'__DB_DUMP__' +contains_ok $TEST_DIR/states-db-pre-restart-2013092306 <<'__DB_DUMP__' force_restart|2013092300|1|1|succeeded force_restart|2013092306|1|1|running -force_restart|2013092312|0|1|held output_states|2013092300|1|1|succeeded output_states|2013092306|0|1|waiting retrying_task|2013092300|4|3|succeeded retrying_task|2013092306|1|2|retrying -retrying_task|2013092312|0|1|held tidy|2013092300|1|1|succeeded tidy|2013092306|0|1|waiting __DB_DUMP__ -cmp_ok $TEST_DIR/states-db-post-restart-2013092306 <<'__DB_DUMP__' +contains_ok $TEST_DIR/states-db-post-restart-2013092306 <<'__DB_DUMP__' force_restart|2013092300|1|1|succeeded force_restart|2013092306|1|1|succeeded -force_restart|2013092312|0|1|held output_states|2013092300|1|1|succeeded output_states|2013092306|1|1|running -output_states|2013092312|0|1|held retrying_task|2013092300|4|3|succeeded retrying_task|2013092306|2|2|retrying -retrying_task|2013092312|0|1|held tidy|2013092300|1|1|succeeded tidy|2013092306|0|1|waiting __DB_DUMP__ @@ -152,19 +140,15 @@ sqlite3 $(cylc get-global-config --print-run-dir)/$SUITE_NAME/cylc-suite.db \ "select name, cycle, submit_num, try_num, status from task_states order by name, cycle;" > $TEST_DIR/states-db -cmp_ok $TEST_DIR/states-db <<'__DB_DUMP__' +contains_ok $TEST_DIR/states-db <<'__DB_DUMP__' force_restart|2013092300|1|1|succeeded force_restart|2013092306|1|1|succeeded -force_restart|2013092312|0|1|held output_states|2013092300|1|1|succeeded output_states|2013092306|1|1|succeeded -output_states|2013092312|0|1|held retrying_task|2013092300|4|3|succeeded retrying_task|2013092306|4|3|succeeded -retrying_task|2013092312|0|1|held tidy|2013092300|1|1|succeeded tidy|2013092306|1|1|succeeded -tidy|2013092312|0|1|held __DB_DUMP__ #------------------------------------------------------------------------------- purge_suite $SUITE_NAME diff --git a/tests/restart/04-running.t b/tests/restart/04-running.t index dba6f6d6cda..2509cc8fb19 100644 --- a/tests/restart/04-running.t +++ b/tests/restart/04-running.t @@ -76,13 +76,9 @@ running_task.2013092306 : status=waiting, spawned=false tidy.2013092300 : status=waiting, spawned=false __STATE__ grep_ok "running_task|2013092300|1|1|running" $TEST_DIR/states-db-pre-restart-2013092300 -cmp_ok $TEST_DIR/states-db-post-restart-2013092300 <<'__DB_DUMP__' +contains_ok $TEST_DIR/states-db-post-restart-2013092300 <<'__DB_DUMP__' force_restart|2013092300|1|1|succeeded -force_restart|2013092306|0|1|waiting -output_states|2013092300|1|1|running -output_states|2013092306|0|1|waiting running_task|2013092300|1|1|running -running_task|2013092306|0|1|waiting tidy|2013092300|0|1|waiting __DB_DUMP__ cmp_ok $TEST_DIR/state-pre-restart-2013092306 <<'__STATE__' @@ -100,29 +96,24 @@ running_task.2013092312 : status=held, spawned=false tidy.2013092300 : status=succeeded, spawned=true tidy.2013092306 : status=waiting, spawned=false __STATE__ -cmp_ok $TEST_DIR/states-db-pre-restart-2013092306 <<'__DB_DUMP__' +contains_ok $TEST_DIR/states-db-pre-restart-2013092306 <<'__DB_DUMP__' force_restart|2013092300|1|1|succeeded force_restart|2013092306|1|1|running -force_restart|2013092312|0|1|held output_states|2013092300|1|1|succeeded output_states|2013092306|0|1|waiting running_task|2013092300|1|1|succeeded running_task|2013092306|1|1|running -running_task|2013092312|0|1|held tidy|2013092300|1|1|succeeded tidy|2013092306|0|1|waiting __DB_DUMP__ -cmp_ok $TEST_DIR/states-db-post-restart-2013092306 <<'__DB_DUMP__' +contains_ok $TEST_DIR/states-db-post-restart-2013092306 <<'__DB_DUMP__' force_restart|2013092300|1|1|succeeded force_restart|2013092306|1|1|succeeded -force_restart|2013092312|0|1|held output_states|2013092300|1|1|succeeded output_states|2013092306|2|1|running -output_states|2013092312|0|1|held running_task|2013092300|1|1|succeeded running_task|2013092306|1|1|succeeded -running_task|2013092312|0|1|held tidy|2013092300|1|1|succeeded tidy|2013092306|0|1|waiting __DB_DUMP__ @@ -143,16 +134,13 @@ sqlite3 $(cylc get-global-config --print-run-dir)/$SUITE_NAME/cylc-suite.db \ "select name, cycle, submit_num, try_num, status from task_states order by name, cycle;" > $TEST_DIR/states-db -cmp_ok $TEST_DIR/states-db <<'__DB_DUMP__' +contains_ok $TEST_DIR/states-db <<'__DB_DUMP__' force_restart|2013092300|1|1|succeeded force_restart|2013092306|1|1|succeeded -force_restart|2013092312|0|1|held output_states|2013092300|1|1|succeeded output_states|2013092306|2|1|succeeded -output_states|2013092312|0|1|held running_task|2013092300|1|1|succeeded running_task|2013092306|1|1|succeeded -running_task|2013092312|0|1|held tidy|2013092300|1|1|succeeded tidy|2013092306|1|1|succeeded __DB_DUMP__ diff --git a/tests/restart/05-submit-failed.t b/tests/restart/05-submit-failed.t index 8a00ab6a471..65317935338 100644 --- a/tests/restart/05-submit-failed.t +++ b/tests/restart/05-submit-failed.t @@ -39,7 +39,7 @@ run_ok $TEST_NAME cylc validate $SUITE_NAME cmp_ok "$TEST_NAME.stderr" $TEST_DIR/states-db -cmp_ok $TEST_DIR/states-db <<'__DB_DUMP__' +contains_ok $TEST_DIR/states-db <<'__DB_DUMP__' force_restart|2013092300|1|1|succeeded force_restart|2013092306|1|1|succeeded -force_restart|2013092312|0|1|held output_states|2013092300|1|1|succeeded output_states|2013092306|1|1|succeeded -output_states|2013092312|0|1|held submit_fail_task|2013092300|1|1|submit-failed submit_fail_task|2013092306|1|1|submit-failed tidy|2013092300|1|1|succeeded tidy|2013092306|1|1|succeeded -tidy|2013092312|0|1|held __DB_DUMP__ #------------------------------------------------------------------------------- purge_suite $SUITE_NAME diff --git a/tests/restart/06-succeeded.t b/tests/restart/06-succeeded.t index 9ac86007a4b..2dd99479006 100644 --- a/tests/restart/06-succeeded.t +++ b/tests/restart/06-succeeded.t @@ -76,13 +76,10 @@ succeed_task.2013092306 : status=waiting, spawned=false tidy.2013092300 : status=waiting, spawned=false __STATE__ grep_ok "succeed_task|2013092300|1|1|succeeded" $TEST_DIR/states-db-pre-restart-2013092300 -cmp_ok $TEST_DIR/states-db-post-restart-2013092300 <<'__DB_DUMP__' +contains_ok $TEST_DIR/states-db-post-restart-2013092300 <<'__DB_DUMP__' force_restart|2013092300|1|1|succeeded -force_restart|2013092306|0|1|waiting output_states|2013092300|1|1|running -output_states|2013092306|0|1|waiting succeed_task|2013092300|1|1|succeeded -succeed_task|2013092306|0|1|waiting tidy|2013092300|0|1|waiting __DB_DUMP__ cmp_ok $TEST_DIR/state-pre-restart-2013092306 <<'__STATE__' @@ -100,29 +97,24 @@ succeed_task.2013092312 : status=held, spawned=false tidy.2013092300 : status=succeeded, spawned=true tidy.2013092306 : status=waiting, spawned=false __STATE__ -cmp_ok $TEST_DIR/states-db-pre-restart-2013092306 <<'__DB_DUMP__' +contains_ok $TEST_DIR/states-db-pre-restart-2013092306 <<'__DB_DUMP__' force_restart|2013092300|1|1|succeeded force_restart|2013092306|1|1|running -force_restart|2013092312|0|1|held output_states|2013092300|1|1|succeeded output_states|2013092306|0|1|waiting succeed_task|2013092300|1|1|succeeded succeed_task|2013092306|1|1|succeeded -succeed_task|2013092312|0|1|held tidy|2013092300|1|1|succeeded tidy|2013092306|0|1|waiting __DB_DUMP__ -cmp_ok $TEST_DIR/states-db-post-restart-2013092306 <<'__DB_DUMP__' +contains_ok $TEST_DIR/states-db-post-restart-2013092306 <<'__DB_DUMP__' force_restart|2013092300|1|1|succeeded force_restart|2013092306|1|1|succeeded -force_restart|2013092312|0|1|held output_states|2013092300|1|1|succeeded output_states|2013092306|1|1|running -output_states|2013092312|0|1|held succeed_task|2013092300|1|1|succeeded succeed_task|2013092306|1|1|succeeded -succeed_task|2013092312|0|1|held tidy|2013092300|1|1|succeeded tidy|2013092306|0|1|waiting __DB_DUMP__ @@ -143,19 +135,15 @@ sqlite3 $(cylc get-global-config --print-run-dir)/$SUITE_NAME/cylc-suite.db \ "select name, cycle, submit_num, try_num, status from task_states order by name, cycle;" > $TEST_DIR/states-db -cmp_ok $TEST_DIR/states-db <<'__DB_DUMP__' +contains_ok $TEST_DIR/states-db <<'__DB_DUMP__' force_restart|2013092300|1|1|succeeded force_restart|2013092306|1|1|succeeded -force_restart|2013092312|0|1|held output_states|2013092300|1|1|succeeded output_states|2013092306|1|1|succeeded -output_states|2013092312|0|1|held succeed_task|2013092300|1|1|succeeded succeed_task|2013092306|1|1|succeeded -succeed_task|2013092312|0|1|held tidy|2013092300|1|1|succeeded tidy|2013092306|1|1|succeeded -tidy|2013092312|0|1|held __DB_DUMP__ #------------------------------------------------------------------------------- purge_suite $SUITE_NAME diff --git a/tests/restart/07-waiting.t b/tests/restart/07-waiting.t index 6ef459683b1..3f66109ddb4 100644 --- a/tests/restart/07-waiting.t +++ b/tests/restart/07-waiting.t @@ -75,11 +75,9 @@ tidy.2013092300 : status=waiting, spawned=false waiting_task.2013092300 : status=waiting, spawned=false __STATE__ grep_ok "waiting_task|2013092300|0|1|waiting" $TEST_DIR/states-db-pre-restart-2013092300 -cmp_ok $TEST_DIR/states-db-post-restart-2013092300 <<'__DB_DUMP__' +contains_ok $TEST_DIR/states-db-post-restart-2013092300 <<'__DB_DUMP__' force_restart|2013092300|1|1|succeeded -force_restart|2013092306|0|1|waiting output_states|2013092300|1|1|running -output_states|2013092306|0|1|waiting tidy|2013092300|0|1|waiting waiting_task|2013092300|0|1|waiting __DB_DUMP__ @@ -97,10 +95,9 @@ tidy.2013092300 : status=succeeded, spawned=true tidy.2013092306 : status=waiting, spawned=false waiting_task.2013092306 : status=waiting, spawned=false __STATE__ -cmp_ok $TEST_DIR/states-db-pre-restart-2013092306 <<'__DB_DUMP__' +contains_ok $TEST_DIR/states-db-pre-restart-2013092306 <<'__DB_DUMP__' force_restart|2013092300|1|1|succeeded force_restart|2013092306|1|1|running -force_restart|2013092312|0|1|held output_states|2013092300|1|1|succeeded output_states|2013092306|0|1|waiting tidy|2013092300|1|1|succeeded @@ -108,13 +105,11 @@ tidy|2013092306|0|1|waiting waiting_task|2013092300|1|1|succeeded waiting_task|2013092306|0|1|waiting __DB_DUMP__ -cmp_ok $TEST_DIR/states-db-post-restart-2013092306 <<'__DB_DUMP__' +contains_ok $TEST_DIR/states-db-post-restart-2013092306 <<'__DB_DUMP__' force_restart|2013092300|1|1|succeeded force_restart|2013092306|1|1|succeeded -force_restart|2013092312|0|1|held output_states|2013092300|1|1|succeeded output_states|2013092306|1|1|running -output_states|2013092312|0|1|held tidy|2013092300|1|1|succeeded tidy|2013092306|0|1|waiting waiting_task|2013092300|1|1|succeeded @@ -137,19 +132,15 @@ sqlite3 $(cylc get-global-config --print-run-dir)/$SUITE_NAME/cylc-suite.db \ "select name, cycle, submit_num, try_num, status from task_states order by name, cycle;" > $TEST_DIR/states-db -cmp_ok $TEST_DIR/states-db <<'__DB_DUMP__' +contains_ok $TEST_DIR/states-db <<'__DB_DUMP__' force_restart|2013092300|1|1|succeeded force_restart|2013092306|1|1|succeeded -force_restart|2013092312|0|1|held output_states|2013092300|1|1|succeeded output_states|2013092306|1|1|succeeded -output_states|2013092312|0|1|held tidy|2013092300|1|1|succeeded tidy|2013092306|1|1|succeeded -tidy|2013092312|0|1|held waiting_task|2013092300|1|1|succeeded waiting_task|2013092306|1|1|succeeded -waiting_task|2013092312|0|1|held __DB_DUMP__ #------------------------------------------------------------------------------- purge_suite $SUITE_NAME diff --git a/tests/restart/failed/suite.rc b/tests/restart/failed/suite.rc index c50aa6b1c38..09bd81188c8 100644 --- a/tests/restart/failed/suite.rc +++ b/tests/restart/failed/suite.rc @@ -13,8 +13,8 @@ timeout handler = "touch {{ TEST_DIR }}/suite-stopping && shutdown_this_suite_hook" timeout = 3 [scheduling] - initial cycle time = 2013092300 - final cycle time = 2013092306 + initial cycle point = 2013092300 + final cycle point = 2013092306 runahead limit = 2 [[dependencies]] [[[0,6,12,18]]] diff --git a/tests/restart/pre-init/ref-state b/tests/restart/pre-init/ref-state index 2c8109f6407..bc13cf30d17 100644 --- a/tests/restart/pre-init/ref-state +++ b/tests/restart/pre-init/ref-state @@ -1,4 +1,5 @@ bar|2010080800|succeeded +bar|2010080900|held foo|2010080800|succeeded foo|2010080900|held p1|2010080800|succeeded diff --git a/tests/restart/reload/suite.rc b/tests/restart/reload/suite.rc index b6ea087ae79..a8c94ec9db9 100644 --- a/tests/restart/reload/suite.rc +++ b/tests/restart/reload/suite.rc @@ -12,6 +12,7 @@ which should run to completion on restarting.""" [scheduling] initial cycle time = 2010080800 final cycle time = 2010080900 + runahead limit = 12 [[special tasks]] sequential = foo [[dependencies]] diff --git a/tests/runahead/01-check-default-simple.t b/tests/runahead/01-check-default-simple.t new file mode 100644 index 00000000000..0a76bc5eacd --- /dev/null +++ b/tests/runahead/01-check-default-simple.t @@ -0,0 +1,45 @@ +#!/bin/bash +#C: THIS FILE IS PART OF THE CYLC SUITE ENGINE. +#C: Copyright (C) 2008-2014 Hilary Oliver, NIWA +#C: +#C: This program is free software: you can redistribute it and/or modify +#C: it under the terms of the GNU General Public License as published by +#C: the Free Software Foundation, either version 3 of the License, or +#C: (at your option) any later version. +#C: +#C: This program is distributed in the hope that it will be useful, +#C: but WITHOUT ANY WARRANTY; without even the implied warranty of +#C: MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +#C: GNU General Public License for more details. +#C: +#C: You should have received a copy of the GNU General Public License +#C: along with this program. If not, see . +#------------------------------------------------------------------------------- +#C: Test default runahead limit behaviour is still the same +. $(dirname $0)/test_header +#------------------------------------------------------------------------------- +set_test_number 4 +#------------------------------------------------------------------------------- +install_suite $TEST_NAME_BASE default-simple +#------------------------------------------------------------------------------- +TEST_NAME=$TEST_NAME_BASE-validate +run_ok $TEST_NAME cylc validate -v $SUITE_NAME +#------------------------------------------------------------------------------- +TEST_NAME=$TEST_NAME_BASE-run +run_fail $TEST_NAME cylc run --debug $SUITE_NAME +#------------------------------------------------------------------------------- +TEST_NAME=$TEST_NAME_BASE-check-fail +DB=$(cylc get-global-config --print-run-dir)/$SUITE_NAME/cylc-suite.db +RUNAHEAD=$(sqlite3 $DB "select max(cycle) from task_states") +# manual comparison for the test +if [[ "$RUNAHEAD" == "20100101T1200Z" ]]; then + ok $TEST_NAME +else + fail $TEST_NAME +fi +#------------------------------------------------------------------------------- +TEST_NAME=$TEST_NAME_BASE-check-timeout +LOG=$(cylc get-global-config --print-run-dir)/$SUITE_NAME/log/suite/log +grep_ok 'suite timed out after' $LOG +#------------------------------------------------------------------------------- +purge_suite $SUITE_NAME diff --git a/tests/runahead/02-check-default-complex.t b/tests/runahead/02-check-default-complex.t new file mode 100644 index 00000000000..6bb36a69cef --- /dev/null +++ b/tests/runahead/02-check-default-complex.t @@ -0,0 +1,45 @@ +#!/bin/bash +#C: THIS FILE IS PART OF THE CYLC SUITE ENGINE. +#C: Copyright (C) 2008-2014 Hilary Oliver, NIWA +#C: +#C: This program is free software: you can redistribute it and/or modify +#C: it under the terms of the GNU General Public License as published by +#C: the Free Software Foundation, either version 3 of the License, or +#C: (at your option) any later version. +#C: +#C: This program is distributed in the hope that it will be useful, +#C: but WITHOUT ANY WARRANTY; without even the implied warranty of +#C: MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +#C: GNU General Public License for more details. +#C: +#C: You should have received a copy of the GNU General Public License +#C: along with this program. If not, see . +#------------------------------------------------------------------------------- +#C: Test default runahead limit behaviour is still the same +. $(dirname $0)/test_header +#------------------------------------------------------------------------------- +set_test_number 4 +#------------------------------------------------------------------------------- +install_suite $TEST_NAME_BASE default-complex +#------------------------------------------------------------------------------- +TEST_NAME=$TEST_NAME_BASE-validate +run_ok $TEST_NAME cylc validate -v $SUITE_NAME +#------------------------------------------------------------------------------- +TEST_NAME=$TEST_NAME_BASE-run +run_fail $TEST_NAME cylc run --debug $SUITE_NAME +#------------------------------------------------------------------------------- +TEST_NAME=$TEST_NAME_BASE-check-fail +DB=$(cylc get-global-config --print-run-dir)/$SUITE_NAME/cylc-suite.db +RUNAHEAD=$(sqlite3 $DB "select max(cycle) from task_states") +# manual comparison for the test +if [[ "$RUNAHEAD" == "20100101T0500Z" ]]; then + ok $TEST_NAME +else + fail $TEST_NAME +fi +#------------------------------------------------------------------------------- +TEST_NAME=$TEST_NAME_BASE-check-timeout +LOG=$(cylc get-global-config --print-run-dir)/$SUITE_NAME/log/suite/log +grep_ok 'suite timed out after' $LOG +#------------------------------------------------------------------------------- +purge_suite $SUITE_NAME diff --git a/tests/runahead/01-check-default.t b/tests/runahead/03-check-default-future.t similarity index 95% rename from tests/runahead/01-check-default.t rename to tests/runahead/03-check-default-future.t index bde8354238f..54177996c0a 100644 --- a/tests/runahead/01-check-default.t +++ b/tests/runahead/03-check-default-future.t @@ -20,10 +20,10 @@ #------------------------------------------------------------------------------- set_test_number 4 #------------------------------------------------------------------------------- -install_suite $TEST_NAME_BASE default +install_suite $TEST_NAME_BASE default-future #------------------------------------------------------------------------------- TEST_NAME=$TEST_NAME_BASE-validate -run_ok $TEST_NAME cylc validate $SUITE_NAME +run_ok $TEST_NAME cylc validate -v $SUITE_NAME #------------------------------------------------------------------------------- TEST_NAME=$TEST_NAME_BASE-run run_fail $TEST_NAME cylc run --debug $SUITE_NAME diff --git a/tests/runahead/02-no-final-cycle.t b/tests/runahead/04-no-final-cycle.t similarity index 100% rename from tests/runahead/02-no-final-cycle.t rename to tests/runahead/04-no-final-cycle.t diff --git a/tests/runahead/default-complex/suite.rc b/tests/runahead/default-complex/suite.rc new file mode 100644 index 00000000000..7ae430f9f37 --- /dev/null +++ b/tests/runahead/default-complex/suite.rc @@ -0,0 +1,26 @@ +[cylc] + UTC mode = True + [[event hooks]] + timeout = 0.1 + abort on timeout = True +[scheduling] + initial cycle point = 20100101T00 + final cycle point = 20100105T00 + [[dependencies]] + # T00, T07, T14, ... + [[[PT7H]]] + graph = "foo => bar" + # T00, T12, T18... + [[[T00, T12, T18]]] + graph = "foo" + # T04... + [[[T04]]] + graph = "run_ok" + # T05... + [[[T05]]] + graph = "never_run" +[runtime] + [[foo]] + command scripting = false + [[bar]] + command scripting = true diff --git a/tests/runahead/default-future/suite.rc b/tests/runahead/default-future/suite.rc new file mode 100644 index 00000000000..d10cfa5c413 --- /dev/null +++ b/tests/runahead/default-future/suite.rc @@ -0,0 +1,23 @@ +[cylc] + UTC mode = True + [[event hooks]] + timeout = 0.1 + abort on timeout = True +[scheduling] + initial cycle point = 20100101T00 + final cycle point = 20100105T00 + [[dependencies]] + # T00, T01, T02, ... + [[[PT1H]]] + graph = "foo => bar" + # T04 (depending on T10) + [[[T04/PT6H]]] + graph = """ + baz[+PT6H] => bar + baz + """ +[runtime] + [[foo]] + command scripting = false + [[bar]] + command scripting = true diff --git a/tests/runahead/default/suite.rc b/tests/runahead/default-simple/suite.rc similarity index 59% rename from tests/runahead/default/suite.rc rename to tests/runahead/default-simple/suite.rc index 0b1f05d85eb..f8af7de5c1d 100644 --- a/tests/runahead/default/suite.rc +++ b/tests/runahead/default-simple/suite.rc @@ -4,10 +4,11 @@ timeout = 0.1 abort on timeout = True [scheduling] - initial cycle time = 20100101T00 - final cycle time = 20100105T00 + initial cycle point = 20100101T00 + final cycle point = 20100105T00 [[dependencies]] - [[[PT3H]]] + # Intervals are all 24 hours, but we really have a 6 hour repetition. + [[[T00, T06, T12, T18]]] graph = "foo => bar" [runtime] [[foo]] diff --git a/tests/runahead/no_final/suite.rc b/tests/runahead/no_final/suite.rc index cdd83e98edc..3d1291bce61 100644 --- a/tests/runahead/no_final/suite.rc +++ b/tests/runahead/no_final/suite.rc @@ -4,7 +4,7 @@ timeout = 0.1 abort on timeout = True [scheduling] - runahead factor = 3 + runahead limit = PT18H initial cycle time = 20100101T00 [[dependencies]] [[[PT6H]]] diff --git a/tests/runahead/runahead/suite.rc b/tests/runahead/runahead/suite.rc index 8915d3acee7..ef189d1462f 100644 --- a/tests/runahead/runahead/suite.rc +++ b/tests/runahead/runahead/suite.rc @@ -4,7 +4,7 @@ timeout = 0.1 abort on timeout = True [scheduling] - runahead factor = 3 + runahead limit = PT18H initial cycle time = 20100101T00 final cycle time = 20100105T00 [[dependencies]] diff --git a/tests/special/sequential/suite.rc b/tests/special/sequential/suite.rc index 786af5bc4ba..e5c8dbf1192 100644 --- a/tests/special/sequential/suite.rc +++ b/tests/special/sequential/suite.rc @@ -5,6 +5,7 @@ [scheduling] initial cycle time = 2010010100 final cycle time = 2010010118 + runahead limit = 18 [[special tasks]] sequential = "foo" [[dependencies]] diff --git a/tests/vacation/01-loadleveler.t b/tests/vacation/01-loadleveler.t index 75b054cfb8e..256c1cc084d 100755 --- a/tests/vacation/01-loadleveler.t +++ b/tests/vacation/01-loadleveler.t @@ -46,6 +46,7 @@ run_ok $TEST_NAME cylc validate $SUITE_NAME #------------------------------------------------------------------------------- TEST_NAME=$TEST_NAME_BASE-run run_ok $TEST_NAME cylc run --reference-test --debug $SUITE_NAME +cat $TEST_NAME.stderr >/dev/tty #------------------------------------------------------------------------------- TEST_NAME=$TEST_NAME_BASE-t1.1 T1_JOB_FILE=$SUITE_RUN_DIR/log/job/t1.1.1