From 3cb988d3f63a6a663f26fab1b5214c7693d92866 Mon Sep 17 00:00:00 2001
From: Andy Davidoff <andy+github@disruptek.com>
Date: Sat, 22 Feb 2020 22:40:24 -0500
Subject: [PATCH 1/5] fix creepy logic

---
 tests/hello/hello_multiple.test | 2 ++
 testutils/spec.nim              | 7 +++----
 2 files changed, 5 insertions(+), 4 deletions(-)

diff --git a/tests/hello/hello_multiple.test b/tests/hello/hello_multiple.test
index 8a4ca1d..5e513ec 100644
--- a/tests/hello/hello_multiple.test
+++ b/tests/hello/hello_multiple.test
@@ -5,3 +5,5 @@ args = "steve"
 stdout = "hello steve\n"
 args = "jeff"
 stdout = "hello jeff\n"
+args = "larry"
+stdout = "hello larry\n"
diff --git a/testutils/spec.nim b/testutils/spec.nim
index 7941936..50b3529 100644
--- a/testutils/spec.nim
+++ b/testutils/spec.nim
@@ -106,12 +106,11 @@ proc parseTestFile*(filePath: string; config: TestConfig): TestSpec =
             if e.key.cmpIgnoreStyle("args") == 0:
               # if this is the first args statement in the test,
               # then we'll just use it.  otherwise, we'll clone
-              # ourselves and link to the test behind us.
-              if result.args.len == 0:
-                result.args = e.value
-              else:
+              # ourselves and link to the test behind us, first.
+              if result.args.len != 0:
                 # create our parent; the eternal chain
                 result = result.clone
+              result.args = e.value
             else:
               # guard against stupidly pointing at redundant outputs
               result.outputs = result.outputs.filterIt it.name != e.key

From 1b194e5ab6749b382db2920a91789fc73427e0a6 Mon Sep 17 00:00:00 2001
From: Andy Davidoff <andy+github@disruptek.com>
Date: Sun, 23 Feb 2020 11:57:19 -0500
Subject: [PATCH 2/5] correct README

---
 README.md | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/README.md b/README.md
index c1ae452..419e71c 100644
--- a/README.md
+++ b/README.md
@@ -71,15 +71,15 @@ Specify command-line arguments as an escaped string in the following syntax:
 args="arg1 arg2 arg3"
 ```
 
-Each new `args` specification instantiates a new test execution with the most
-recently-specified outputs.
+Each new `args` specification instantiates a new test execution with the most-recently-specified outputs.
 
 ```
-output="6"
-args="2 * 3"
-args="3 * 2"
-output="7"
+stdout="6"
+args="2 \* 3"  # shell escapes are necessary
+args="3 \* 2"  # implicitly reuse existing output definitions
+
 args="3 + 4"
+stdout="7"
 ```
 
 ## License

From 64d0136b0eaa512b7d16f0a50c6677629f9a33c4 Mon Sep 17 00:00:00 2001
From: Andy Davidoff <andy+github@disruptek.com>
Date: Sun, 23 Feb 2020 13:16:55 -0500
Subject: [PATCH 3/5] refactor prior to adding rewrite

---
 testrunner.nim                  | 45 ++++++++++++++++++++-------------
 tests/hello/hello_multiple.test | 14 +++++++---
 testutils/spec.nim              | 34 +++++++++++++++++++------
 3 files changed, 63 insertions(+), 30 deletions(-)

diff --git a/testrunner.nim b/testrunner.nim
index a524449..f94ad2e 100644
--- a/testrunner.nim
+++ b/testrunner.nim
@@ -1,3 +1,4 @@
+import std/strtabs
 import std/os
 import std/osproc
 import std/strutils
@@ -110,33 +111,39 @@ template time(duration, body): untyped =
     body
   duration =  epochTime() - t0
 
-proc cmpOutputs(test: TestSpec, stdout: string): TestStatus =
-  result = OK
-  for output in test.outputs:
-    var testOutput: string
-    if output.name == "stdout":
-      testOutput = stdout
+proc composeOutputs(test: TestSpec, stdout: string): TestOutputs =
+  result = newTestOutputs()
+  for name, expected in test.outputs.pairs:
+    if name == "stdout":
+      result[name] = stdout
     else:
-      if not existsFile(output.name):
-        logFailure(test, OutputFileNotFound, output.name)
-        result = FAILED
+      if not existsFile(name):
         continue
+      result[name] = readFile(name)
+      removeFile(name)
+
+proc cmpOutputs(test: TestSpec, outputs: TestOutputs): TestStatus =
+  result = OK
+  for name, expected in test.outputs.pairs:
+    if name notin outputs:
+      logFailure(test, OutputFileNotFound, name)
+      result = FAILED
+      continue
 
-      testOutput = readFile(output.name)
+    let
+      testOutput = outputs[name]
 
     # Would be nice to do a real diff here instead of simple compare
     if test.timestampPeg.len > 0:
-      if not cmpIgnorePegs(testOutput, output.expectedOutput, peg(test.timestampPeg), pegXid):
-        logFailure(test, OutputsDiffer, output.name, output.expectedOutput, testOutput)
+      if not cmpIgnorePegs(testOutput, expected,
+                           peg(test.timestampPeg), pegXid):
+        logFailure(test, OutputsDiffer, name, expected, testOutput)
         result = FAILED
     else:
-      if not cmpIgnoreDefaultTimestamps(testOutput, output.expectedOutput):
-        logFailure(test, OutputsDiffer, output.name, output.expectedOutput, testOutput)
+      if not cmpIgnoreDefaultTimestamps(testOutput, expected):
+        logFailure(test, OutputsDiffer, name, expected, testOutput)
         result = FAILED
 
-    if output.name != "stdout":
-      removeFile(output.name)
-
 proc compile(test: TestSpec): TestStatus =
   let
     source = test.config.path / test.program.addFileExt(".nim")
@@ -200,7 +207,9 @@ proc execute(test: TestSpec): TestStatus =
           logFailure(test, RuntimeError, output)
           result = FAILED
         else:
-          result = test.cmpOutputs(output)
+          let
+            outputs = test.composeOutputs(output)
+          result = test.cmpOutputs(outputs)
     if test.child != nil:
       result = test.child.execute
 
diff --git a/tests/hello/hello_multiple.test b/tests/hello/hello_multiple.test
index 5e513ec..b4091fd 100644
--- a/tests/hello/hello_multiple.test
+++ b/tests/hello/hello_multiple.test
@@ -1,9 +1,15 @@
 program = "hello"
 
+# my aim is true
 [Output]
-args = "steve"
-stdout = "hello steve\n"
-args = "jeff"
-stdout = "hello jeff\n"
+stdout = "hello world\n"
+
+# option 2
+[Output_larry_is_a_good_boy]
 args = "larry"
 stdout = "hello larry\n"
+
+# option 47
+[Output_stevie_is_a_good_boy]
+args = "stephen"
+stdout = "hello stephen\n"
diff --git a/testutils/spec.nim b/testutils/spec.nim
index 50b3529..4abf81e 100644
--- a/testutils/spec.nim
+++ b/testutils/spec.nim
@@ -2,7 +2,7 @@ import std/os
 import std/parsecfg
 import std/strutils
 import std/streams
-import std/sequtils
+import std/strtabs
 
 import testutils/config
 
@@ -10,9 +10,9 @@ const
   DefaultOses = @["linux", "macosx", "windows"]
 
 type
-  TestOutput = tuple[name: string, expectedOutput: string]
-  TestOutputs = seq[TestOutput]
+  TestOutputs* = StringTableRef
   TestSpec* = ref object
+    section*: string
     args*: string
     config*: TestConfig
     path*: string
@@ -20,6 +20,7 @@ type
     skip*: bool
     program*: string
     flags*: string
+    preamble*: seq[tuple[key: string; value: string]]
     outputs*: TestOutputs
     timestampPeg*: string
     errorMsg*: string
@@ -31,10 +32,13 @@ type
     os*: seq[string]
     child*: TestSpec
 
+proc newTestOutputs*(): StringTableRef =
+  result = newStringTable(mode = modeStyleInsensitive)
+
 proc clone*(spec: TestSpec): TestSpec =
   result = new(TestSpec)
   result[] = spec[]
-  result.outputs = newSeqOfCap[TestOutput](1)
+  result.outputs = newTestOutputs()
   result.args = ""
   result.child = spec
 
@@ -45,9 +49,13 @@ proc binary*(spec: TestSpec): string =
 proc defaults(spec: var TestSpec) =
   ## assert some default values for a given spec
   spec.os = DefaultOses
+  spec.outputs = newTestOutputs()
 
 proc consumeConfigEvent(spec: var TestSpec; event: CfgEvent) =
   ## parse a specification supplied prior to any sections
+
+  # save the key/value pair in case we need to write out the test file
+  spec.preamble.add (key: event.key, value: event.value)
   case event.key
   of "program":
     spec.program = event.value
@@ -70,6 +78,16 @@ proc consumeConfigEvent(spec: var TestSpec; event: CfgEvent) =
       flag = "--define:$#:$#" % [event.key, event.value]
     spec.flags.add flag.quoteShell & " "
 
+proc rewriteTestFile*(spec: TestSpec) =
+  ## rewrite a test file with updated outputs after having run the tests
+  var
+    test = loadConfig(spec.path)
+  if spec.args != "":
+    test.setSectionKey(spec.section, "args", spec.args)
+  for name, expected in spec.outputs.pairs:
+    test.setSectionKey(spec.section, name, expected)
+  test.writeConfig(spec.path)
+
 proc parseTestFile*(filePath: string; config: TestConfig): TestSpec =
   ## parse a test input file into a spec
   result = new(TestSpec)
@@ -100,7 +118,9 @@ proc parseTestFile*(filePath: string; config: TestConfig): TestSpec =
           echo "Parsing warning:" & e.msg
         of cfgSectionStart:
           # starts with Output
-          outputSection = e.section.cmpIgnoreCase("Output") == 0
+          if e.section.cmpIgnoreCase("Output") == 0:
+            outputSection = true
+            result.section = e.section
         of cfgKeyValuePair:
           if outputSection:
             if e.key.cmpIgnoreStyle("args") == 0:
@@ -112,9 +132,7 @@ proc parseTestFile*(filePath: string; config: TestConfig): TestSpec =
                 result = result.clone
               result.args = e.value
             else:
-              # guard against stupidly pointing at redundant outputs
-              result.outputs = result.outputs.filterIt it.name != e.key
-              result.outputs.add((e.key, e.value))
+              result.outputs[e.key] = e.value
           else:
             result.consumeConfigEvent(e)
         of cfgOption:

From 439717e4937c743696617dbcecf84f0b05e05cb7 Mon Sep 17 00:00:00 2001
From: Andy Davidoff <andy+github@disruptek.com>
Date: Sun, 23 Feb 2020 14:19:53 -0500
Subject: [PATCH 4/5] add --update and rethink multiple invocations

---
 README.md            | 33 ++++++++++++++++++++++++---------
 testrunner.nim       | 43 ++++++++++++++++++++++++-------------------
 testutils.nimble     |  2 +-
 testutils/config.nim |  4 ++++
 testutils/spec.nim   | 21 +++++++++++++--------
 5 files changed, 66 insertions(+), 37 deletions(-)

diff --git a/README.md b/README.md
index 419e71c..36c383c 100644
--- a/README.md
+++ b/README.md
@@ -10,6 +10,7 @@ Options:
 --targets:"c c++ js objc" [Not implemented] Run tests for specified targets
 --include:"test1 test2"   Run only listed tests (space/comma seperated)
 --exclude:"test1 test2"   Skip listed tests (space/comma seperated)
+--update                  Rewrite failed tests with new output
 --help                    Display this help and exit
 ```
 
@@ -55,32 +56,46 @@ added in a test file:
 
 ### Outputs
 For outputs to be compared, the output string should be set to the output
-name (stdout or filename) from within the "Output" section, e.g.:
+name (stdout or filename) from within an _Output_ section:
+
 ```
 [Output]
 stdout="""expected stdout output"""
 file.log="""expected file output"""
 ```
+
 Triple quotes can be used for multiple lines.
 
 ### Supplying Command-line Arguments
 
-Specify command-line arguments as an escaped string in the following syntax:
+Optionally specify command-line arguments as an escaped string in the following
+syntax inside any _Output_ section:
 
 ```
-args="arg1 arg2 arg3"
+[Output]
+args = "--title \"useful title\""
 ```
 
-Each new `args` specification instantiates a new test execution with the most-recently-specified outputs.
+### Multiple Invocations
+
+Multiple _Output_ sections denote multiple test program invocations. Any
+failure of the test program to match its expected outputs will short-circuit
+and fail the test.
 
 ```
-stdout="6"
-args="2 \* 3"  # shell escapes are necessary
-args="3 \* 2"  # implicitly reuse existing output definitions
+[Output]
+stdout = ""
+args = "--no-output"
 
-args="3 + 4"
-stdout="7"
+[Output_newlines]
+stdout = "\n\n"
+args = "--newlines"
 ```
 
+### Updating Expected Outputs
+
+Pass the `--update` argument to `testrunner` to rewrite any failing test with
+the new outputs of the test.
+
 ## License
 Apache2 or MIT
diff --git a/testrunner.nim b/testrunner.nim
index f94ad2e..f62b3f6 100644
--- a/testrunner.nim
+++ b/testrunner.nim
@@ -191,27 +191,32 @@ proc compile(test: TestSpec): TestStatus =
 proc execute(test: TestSpec): TestStatus =
   if test.child != nil:
     result = test.child.execute
+  if result notin {OK, SKIPPED}:
+    return
+  var
+    cmd = test.binary
+  if not existsFile(cmd):
+    result = FAILED
+    logFailure(test, ExeFileNotFound)
   else:
-    var
-      cmd = test.binary
-    if not existsFile(cmd):
-      result = FAILED
-      logFailure(test, ExeFileNotFound)
-    else:
-      withinDir parentDir(cmd):
-        cmd = cmd.quoteShell & " " & test.args
+    withinDir parentDir(cmd):
+      cmd = cmd.quoteShell & " " & test.args
+      let
+        (output, exitCode) = execCmdEx(cmd)
+      if exitCode != 0:
+        # parseExecuteOutput() # Need to parse the run time failures?
+        logFailure(test, RuntimeError, output)
+        result = FAILED
+      else:
         let
-          (output, exitCode) = execCmdEx(cmd)
-        if exitCode != 0:
-          # parseExecuteOutput() # Need to parse the run time failures?
-          logFailure(test, RuntimeError, output)
-          result = FAILED
-        else:
-          let
-            outputs = test.composeOutputs(output)
-          result = test.cmpOutputs(outputs)
-    if test.child != nil:
-      result = test.child.execute
+          outputs = test.composeOutputs(output)
+        result = test.cmpOutputs(outputs)
+        # perform an update of the testfile if requested and required
+        if test.config.update and result == FAILED:
+          test.rewriteTestFile(outputs)
+          # we'll call this a `skip` because it's not strictly a failure
+          # and we want any dependent testing to proceed as usual.
+          result = SKIPPED
 
 proc scanTestPath(path: string): seq[string] =
   if fileExists(path):
diff --git a/testutils.nimble b/testutils.nimble
index f308796..29af482 100644
--- a/testutils.nimble
+++ b/testutils.nimble
@@ -18,7 +18,7 @@ proc execCmd(cmd: string) =
 proc execTest(test: string) =
   let
     test = "testrunner " & test
-  when true:
+  when false:
     execCmd "nim c           -f -r " & test
     execCmd "nim c   -d:release -r " & test
     execCmd "nim c   -d:danger  -r " & test
diff --git a/testutils/config.nim b/testutils/config.nim
index 952229b..f1f7d40 100644
--- a/testutils/config.nim
+++ b/testutils/config.nim
@@ -12,6 +12,7 @@ Options:
   --targets:"c c++ js objc" [Not implemented] Run tests for specified targets
   --include:"test1 test2"   Run only listed tests (space/comma seperated)
   --exclude:"test1 test2"   Skip listed tests (space/comma seperated)
+  --update                  Rewrite failed tests with new output
   --help                    Display this help and exit
 
   """.unindent.strip
@@ -19,6 +20,7 @@ Options:
 type
   TestConfig* = object
     path*: string
+    update*: bool
     includedTests*: seq[string]
     excludedTests*: seq[string]
     releaseBuild*: bool
@@ -45,6 +47,8 @@ proc processArguments*(): TestConfig =
         result.noThreads = true
       of "targets", "t":
         discard # not implemented
+      of "update":
+        result.update = true
       of "include":
         result.includedTests.add value.split(Whitespace + {','})
       of "exclude":
diff --git a/testutils/spec.nim b/testutils/spec.nim
index 4abf81e..e475886 100644
--- a/testutils/spec.nim
+++ b/testutils/spec.nim
@@ -36,6 +36,7 @@ proc newTestOutputs*(): StringTableRef =
   result = newStringTable(mode = modeStyleInsensitive)
 
 proc clone*(spec: TestSpec): TestSpec =
+  ## create the parent of this test and set the child reference appropriately
   result = new(TestSpec)
   result[] = spec[]
   result.outputs = newTestOutputs()
@@ -78,13 +79,20 @@ proc consumeConfigEvent(spec: var TestSpec; event: CfgEvent) =
       flag = "--define:$#:$#" % [event.key, event.value]
     spec.flags.add flag.quoteShell & " "
 
-proc rewriteTestFile*(spec: TestSpec) =
+proc rewriteTestFile*(spec: TestSpec; outputs: TestOutputs) =
   ## rewrite a test file with updated outputs after having run the tests
   var
     test = loadConfig(spec.path)
+  # take the opportunity to update an args statement if necessary
   if spec.args != "":
     test.setSectionKey(spec.section, "args", spec.args)
+  else:
+    test.delSectionKey(spec.section, "args")
+  # delete the old test outputs for completeness
   for name, expected in spec.outputs.pairs:
+    test.delSectionKey(spec.section, name)
+  # add the new test outputs
+  for name, expected in outputs.pairs:
     test.setSectionKey(spec.section, name, expected)
   test.writeConfig(spec.path)
 
@@ -118,18 +126,15 @@ proc parseTestFile*(filePath: string; config: TestConfig): TestSpec =
           echo "Parsing warning:" & e.msg
         of cfgSectionStart:
           # starts with Output
-          if e.section.cmpIgnoreCase("Output") == 0:
+          if e.section[0..len"Output"-1].cmpIgnoreCase("Output") == 0:
+            if outputSection:
+              # create our parent; the eternal chain
+              result = result.clone
             outputSection = true
             result.section = e.section
         of cfgKeyValuePair:
           if outputSection:
             if e.key.cmpIgnoreStyle("args") == 0:
-              # if this is the first args statement in the test,
-              # then we'll just use it.  otherwise, we'll clone
-              # ourselves and link to the test behind us, first.
-              if result.args.len != 0:
-                # create our parent; the eternal chain
-                result = result.clone
               result.args = e.value
             else:
               result.outputs[e.key] = e.value

From 0744ed8bc751fba003f1ac66e9765721057e5267 Mon Sep 17 00:00:00 2001
From: Andy Davidoff <andy+github@disruptek.com>
Date: Sun, 23 Feb 2020 14:20:54 -0500
Subject: [PATCH 5/5] re-enable full tests

---
 testutils.nimble | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/testutils.nimble b/testutils.nimble
index 29af482..f308796 100644
--- a/testutils.nimble
+++ b/testutils.nimble
@@ -18,7 +18,7 @@ proc execCmd(cmd: string) =
 proc execTest(test: string) =
   let
     test = "testrunner " & test
-  when false:
+  when true:
     execCmd "nim c           -f -r " & test
     execCmd "nim c   -d:release -r " & test
     execCmd "nim c   -d:danger  -r " & test