Merge pull request #1026 from benfitzpatrick/1008.start-up-back-comp-…

…dep-tweak

#1008: more minimal start-up dependencies

bin/cylc-graph

@@ -94,6 +94,9 @@ The "Save" button generates an image of the current view, of format (e.g.
 png, svg, jpg, eps) determined by the filename extension. If the chosen
 format is not available a dialog box will show those that are available.
 
+If an output filename is specified in the options, the viewer will not
+open and a graph representation will be output to the filename.
+
 GRAPH VIEWER CONTROLS:
     * Center on a node: left-click.
     * Pan view: left-drag.
@@ -132,6 +135,16 @@ parser.add_option( "-f", "--file",
     help="View a specific dot-language graphfile.",
     metavar="FILE", action="store", default=None, dest="filename" )
 
+parser.add_option( "-O", "--output-file",
+    help="Output to a specific file, with a format given by --output-format" +
+         " or extrapolated from the extension.",
+    metavar="FILE", action="store", default=None, dest="output_filename" )
+
+parser.add_option( "--output-format",
+    help="Specify a format for writing out the graph to --output-file " +
+         "e.g. png, svg, jpg, eps, dot.",
+    metavar="FORMAT", action="store", default=None, dest="output_format" )
+
 ( options, args ) = parser.parse_args()
 
 # import modules that require gtk now, so that a display is not needed
@@ -155,6 +168,9 @@ if options.filename:
         from xdot import DotWindow
     except:
         raise SystemExit( "Failed to import the xdot viewer.")
+    if options.output_filename:
+        raise SystemExit( "ERROR: output-file not supported for " +
+                          "dot files. Use 'dot' command instead." )
     window = DotWindow()
     try:
         window.update( file )
@@ -168,6 +184,8 @@ if options.filename:
     gtk.main()
     sys.exit(0)
 
+should_hide_gtk_window = (not options.output_filename)
+
 suite, suiterc, watchers = parser.get_suite()
 
 # parse and plot the suite.rc dependency graph
@@ -176,7 +194,8 @@ if len(args) < 1 or len(args) > 3:
 
 if options.namespaces:
     window = MyDotWindow2( suite, suiterc, options.templatevars,
-                           options.templatevars_file, watchers )
+                           options.templatevars_file, watchers,
+                           should_hide=should_hide_gtk_window )
 else:
     # SUITE DEPENDENCY GRAPH
 
@@ -201,6 +220,10 @@ window.widget.connect( 'clicked', on_url_clicked, window )
 
 window.get_graph()
 
+if options.output_filename:
+    window.graph.draw( options.output_filename, prog="dot" )
+    sys.exit(0)
+
 window.connect( 'destroy', gtk.main_quit)
 
 #if options.updatelive:

lib/cylc/config.py

@@ -1594,7 +1594,7 @@ def load_graph( self ):
             print "Parsing the dependency graph"
 
         start_up_tasks = self.cfg['scheduling']['special tasks']['start-up']
-        initial_tasks = list(start_up_tasks)
+        back_comp_initial_tasks = list(start_up_tasks)
 
         self.graph_found = False
         has_non_async_graphs = False
@@ -1611,28 +1611,30 @@ def load_graph( self ):
             )
             for left, left_output, right in async_dependencies:
                 if left:
-                    initial_tasks.append(left)
+                    back_comp_initial_tasks.append(left)
                 if right:
-                    initial_tasks.append(right)
+                    back_comp_initial_tasks.append(right)
 
         # Create a stack of sections (sequence strings) and graphs.
         items = []
         for item, value in self.cfg['scheduling']['dependencies'].items():
             if item == 'graph':
                 continue
             has_non_async_graphs = True
-            items.append((item, value, initial_tasks, False))
+            items.append((item, value, back_comp_initial_tasks))
 
-        start_up_tasks_graphed = []
+        back_comp_initial_dep_points = {}
+        initial_point = get_point(
+            self.cfg['scheduling']['initial cycle point'])
+        back_comp_initial_tasks_graphed = []
         while items:
-            item, value, tasks_to_prune, is_inserted = items.pop(0)
+            item, value, tasks_to_prune = items.pop(0)
 
             # If the section consists of more than one sequence, split it up.
             if re.search("(?![^(]+\)),", item):
                 new_items = re.split("(?![^(]+\)),", item)
                 for new_item in new_items:
-                    items.append((new_item.strip(), value,
-                                  tasks_to_prune, False))
+                    items.append((new_item.strip(), value, tasks_to_prune))
                 continue
 
             try:
@@ -1643,10 +1645,6 @@ def load_graph( self ):
                 continue
 
             section = item
-            if is_inserted:
-                print "INSERTED DEPENDENCIES REPLACEMENT:"
-                print "[[[" + section + "]]]"
-                print "    " + 'graph = """' + graph + '"""' 
             special_dependencies = self.parse_graph(
                 section, graph, section_seq_map=section_seq_map,
                 tasks_to_prune=tasks_to_prune
@@ -1657,32 +1655,57 @@ def load_graph( self ):
                     self.cfg['scheduling']['initial cycle point'],
                     self.cfg['scheduling']['final cycle point']
                 )
-                first_point = section_seq.get_first_point(
-                    get_point(self.cfg['scheduling']['initial cycle point'])
-                )
-                graph_text = ""
-                for left, left_output, right in special_dependencies:
-                    # Set e.g. (foo, fail, bar) to be foo[^]:fail => bar.
-                    graph_text += left + "[^]"
-                    if left_output:
-                        graph_text += ":" + left_output
-                    graph_text += " => " + right + "\n"
-                    if (left in start_up_tasks and
-                            left not in start_up_tasks_graphed):
-                        # Start-up tasks need their own explicit section.
-                        items.append((get_sequence_cls().get_async_expr(),
-                                     {"graph": left}, [], True))
-                        start_up_tasks_graphed.append(left)
-                graph_text = graph_text.rstrip()
-                section = get_sequence_cls().get_async_expr(first_point)
-                items.append((section, {"graph": graph_text}, [], True))
+                first_point = section_seq.get_first_point(initial_point)
+                for dep in special_dependencies:
+                    # Set e.g. (foo, fail, bar) => foo, foo[^]:fail => bar.
+                    left, left_output, right = dep
+                    if left in back_comp_initial_tasks:
+                        # Start-up/Async tasks now always run at R1.
+                        back_comp_initial_dep_points[(left, None, None)] = [
+                            initial_point]
+                    # Sort out the dependencies on R1 at R1/some-time.
+                    back_comp_initial_dep_points.setdefault(tuple(dep), [])
+                    back_comp_initial_dep_points[tuple(dep)].append(
+                        first_point)
+
+        back_comp_initial_section_graphs = {}
+        for dep in sorted(back_comp_initial_dep_points):           
+            first_common_point = min(back_comp_initial_dep_points[dep])
+            at_initial_point = (first_common_point == initial_point)
+            left, left_output, right = dep
+            graph_text = left
+            if not at_initial_point:
+                # Reference left at the initial point.
+                graph_text += "[^]"
+            if left_output:
+                graph_text += ":" + left_output
+            if right:
+                graph_text += " => " + right
+            if at_initial_point:
+                section = get_sequence_cls().get_async_expr()
+            else:
+                section = get_sequence_cls().get_async_expr(
+                    first_common_point)
+            back_comp_initial_section_graphs.setdefault(section, [])
+            back_comp_initial_section_graphs[section].append(graph_text)
+
+        for section in sorted(back_comp_initial_section_graphs):
+            total_graph_text = "\n".join(
+                back_comp_initial_section_graphs[section])
+            print "INSERTED DEPENDENCIES REPLACEMENT:"
+            print "[[[" + section + "]]]"
+            print "    " + 'graph = """\n' + total_graph_text + '\n"""' 
+            self.parse_graph(
+                section, total_graph_text,
+                section_seq_map=section_seq_map, tasks_to_prune=[]
+            )
         if not flags.back_comp_cycling:
             if async_graph and has_non_async_graphs:
                 raise SuiteConfigError(
                     "Error: mixed async & cycling graphs is not allowed in " +
                     "new-style cycling. Use 'R1...' tasks instead."
                 )
-            if start_up_tasks:
+            if back_comp_initial_tasks:
                 raise SuiteConfigError(
                     "Error: start-up tasks should be 'R1...' tasks in " +
                     "new-style cycling"

lib/cylc/cylc_xdot.py

@@ -72,7 +72,8 @@ class MyDotWindow2( CylcDotViewerCommon ):
     </ui>
     '''
     def __init__(self, suite, suiterc, template_vars,
-            template_vars_file, watch, orientation="TB" ):
+            template_vars_file, watch, orientation="TB",
+            should_hide=False ):
         self.outfile = None
         self.disable_output_image = False
         self.suite = suite
@@ -169,7 +170,8 @@ def __init__(self, suite, suiterc, template_vars,
                     #self.rc_mtimes[rc] = self.rc_last_mtimes[rc]
                     break
 
-        self.show_all()
+        if not should_hide:
+            self.show_all()
         while True:
             if self.load_config():
                 break

tests/cyclers/00-daily.t

@@ -18,12 +18,17 @@
 # Test intercycle dependencies.
 . $(dirname $0)/test_header
 #-------------------------------------------------------------------------------
-set_test_number 2
+set_test_number 3
 #-------------------------------------------------------------------------------
-install_suite $TEST_NAME_BASE $(basename $0 | sed "s/^.*-\(.*\)\.t/\1/g")
+CHOSEN_SUITE=$(basename $0 | sed "s/^.*-\(.*\)\.t/\1/g")
+install_suite $TEST_NAME_BASE $CHOSEN_SUITE
 #-------------------------------------------------------------------------------
 TEST_NAME=$TEST_NAME_BASE-validate
-run_ok $TEST_NAME cylc validate $SUITE_NAME
+run_ok $TEST_NAME cylc validate "$SUITE_NAME"
+#-------------------------------------------------------------------------------
+TEST_NAME=$TEST_NAME_BASE-graph
+graph_suite "$SUITE_NAME" "$SUITE_NAME.graph.plain"
+cmp_ok "$SUITE_NAME.graph.plain" "$TEST_SOURCE_DIR/$CHOSEN_SUITE/graph.plain.ref"
 #-------------------------------------------------------------------------------
 TEST_NAME=$TEST_NAME_BASE-run
 suite_run_ok $TEST_NAME cylc run --reference-test --debug $SUITE_NAME

tests/cyclers/23-multidaily_local.t

@@ -18,9 +18,10 @@
 # Test intercycle dependencies, local time.
 . $(dirname $0)/test_header
 #-------------------------------------------------------------------------------
-set_test_number 2
+set_test_number 3
 #-------------------------------------------------------------------------------
-install_suite $TEST_NAME_BASE $(basename $0 | sed "s/^.*-\(.*\)\.t/\1/g")
+CHOSEN_SUITE=$(basename $0 | sed "s/^.*-\(.*\)\.t/\1/g")
+install_suite $TEST_NAME_BASE $CHOSEN_SUITE
 CURRENT_TZ_UTC_OFFSET=$(date +%z)
 if [[ $CURRENT_TZ_UTC_OFFSET == '+0000' ]]; then
     CURRENT_TZ_UTC_OFFSET="Z"
@@ -32,6 +33,13 @@ sed -i "s/Z/$CURRENT_TZ_UTC_OFFSET/g" reference.log
 TEST_NAME=$TEST_NAME_BASE-validate
 run_ok $TEST_NAME cylc validate $SUITE_NAME
 #-------------------------------------------------------------------------------
+TEST_NAME=$TEST_NAME_BASE-graph
+graph_suite "$SUITE_NAME" "$SUITE_NAME.graph.plain"
+sed "s/Z/$CURRENT_TZ_UTC_OFFSET/g" \
+    "$TEST_SOURCE_DIR/$CHOSEN_SUITE/graph.plain.ref" > graph.plain.local.ref
+cmp_ok "$SUITE_NAME.graph.plain" graph.plain.local.ref
+xxdiff -D "$SUITE_NAME.graph.plain" graph.plain.local.ref
+#-------------------------------------------------------------------------------
 TEST_NAME=$TEST_NAME_BASE-run
 suite_run_ok $TEST_NAME cylc run --reference-test --debug $SUITE_NAME
 #-------------------------------------------------------------------------------

tests/cyclers/360/graph.plain.ref

@@ -0,0 +1,9 @@
+edge "foo.20130228T00" "foo.20130229T00" solid black
+edge "foo.20130229T00" "foo.20130230T00" solid black
+edge "foo.20130230T00" "foo.20130301T00" solid black
+graph 
+node "foo.20130228T00" "foo\n20130228T00" unfilled box black lightgrey
+node "foo.20130229T00" "foo\n20130229T00" unfilled box black lightgrey
+node "foo.20130230T00" "foo\n20130230T00" unfilled box black lightgrey
+node "foo.20130301T00" "foo\n20130301T00" unfilled box black lightgrey
+stop

tests/cyclers/360/suite.rc

@@ -13,3 +13,6 @@
 [runtime]
     [[foo]]
         command scripting = true
+[visualization]
+    initial cycle point = 20130228T00
+    final cycle point = 20130301T00

tests/cyclers/async_integer/graph.plain.ref

@@ -0,0 +1,5 @@
+edge "foo.1" "bar.1" solid black
+graph 
+node "bar.1" "bar\n1" unfilled box black lightgrey
+node "foo.1" "foo\n1" unfilled box black lightgrey
+stop

tests/cyclers/daily/graph.plain.ref

@@ -0,0 +1,13 @@
+edge "foo.20131231T2300Z" "bar.20131231T2300Z" solid black
+edge "foo.20131231T2300Z" "foo.20140101T2300Z" solid black
+edge "foo.20140101T2300Z" "bar.20140101T2300Z" solid black
+edge "foo.20140101T2300Z" "foo.20140102T2300Z" solid black
+edge "foo.20140102T2300Z" "bar.20140102T2300Z" solid black
+graph 
+node "bar.20131231T2300Z" "bar\n20131231T2300Z" unfilled box black lightgrey
+node "bar.20140101T2300Z" "bar\n20140101T2300Z" unfilled box black lightgrey
+node "bar.20140102T2300Z" "bar\n20140102T2300Z" unfilled box black lightgrey
+node "foo.20131231T2300Z" "foo\n20131231T2300Z" unfilled box black lightgrey
+node "foo.20140101T2300Z" "foo\n20140101T2300Z" unfilled box black lightgrey
+node "foo.20140102T2300Z" "foo\n20140102T2300Z" unfilled box black lightgrey
+stop

tests/cyclers/daily/suite.rc

@@ -9,3 +9,6 @@
 [runtime]
     [[root]]
         command scripting = true
+[visualization]
+    initial cycle point = 20131231T2300
+    final cycle point = 20140103T0000

tests/cyclers/hourly/graph.plain.ref

@@ -0,0 +1,21 @@
+edge "foo.20000229T2000Z" "bar.20000229T2000Z" solid black
+edge "foo.20000229T2000Z" "foo.20000229T2100Z" solid black
+edge "foo.20000229T2100Z" "bar.20000229T2100Z" solid black
+edge "foo.20000229T2100Z" "foo.20000229T2200Z" solid black
+edge "foo.20000229T2200Z" "bar.20000229T2200Z" solid black
+edge "foo.20000229T2200Z" "foo.20000229T2300Z" solid black
+edge "foo.20000229T2300Z" "bar.20000229T2300Z" solid black
+edge "foo.20000229T2300Z" "foo.20000301T0000Z" solid black
+edge "foo.20000301T0000Z" "bar.20000301T0000Z" solid black
+graph 
+node "bar.20000229T2000Z" "bar\n20000229T2000Z" unfilled box black lightgrey
+node "bar.20000229T2100Z" "bar\n20000229T2100Z" unfilled box black lightgrey
+node "bar.20000229T2200Z" "bar\n20000229T2200Z" unfilled box black lightgrey
+node "bar.20000229T2300Z" "bar\n20000229T2300Z" unfilled box black lightgrey
+node "bar.20000301T0000Z" "bar\n20000301T0000Z" unfilled box black lightgrey
+node "foo.20000229T2000Z" "foo\n20000229T2000Z" unfilled box black lightgrey
+node "foo.20000229T2100Z" "foo\n20000229T2100Z" unfilled box black lightgrey
+node "foo.20000229T2200Z" "foo\n20000229T2200Z" unfilled box black lightgrey
+node "foo.20000229T2300Z" "foo\n20000229T2300Z" unfilled box black lightgrey
+node "foo.20000301T0000Z" "foo\n20000301T0000Z" unfilled box black lightgrey
+stop

tests/cyclers/hourly/suite.rc

@@ -9,3 +9,6 @@
 [runtime]
     [[root]]
         command scripting = true
+[visualization]
+    initial cycle point = 20000229T2000
+    final cycle point = 20000301