Merge pull request #4 from disruptek/reset-io

add --update and structure multi-command invocation

README.md

@@ -10,6 +10,7 @@ Options:
 --targets:"c c++ js objc" [Not implemented] Run tests for specified targets
 --include:"test1 test2"   Run only listed tests (space/comma seperated)
 --exclude:"test1 test2"   Skip listed tests (space/comma seperated)
+--update                  Rewrite failed tests with new output
 --help                    Display this help and exit
 ```
 
@@ -55,32 +56,46 @@ added in a test file:
 
 ### Outputs
 For outputs to be compared, the output string should be set to the output
-name (stdout or filename) from within the "Output" section, e.g.:
+name (stdout or filename) from within an _Output_ section:
+
 ```
 [Output]
 stdout="""expected stdout output"""
 file.log="""expected file output"""
 ```
+
 Triple quotes can be used for multiple lines.
 
 ### Supplying Command-line Arguments
 
-Specify command-line arguments as an escaped string in the following syntax:
+Optionally specify command-line arguments as an escaped string in the following
+syntax inside any _Output_ section:
 
 ```
-args="arg1 arg2 arg3"
+[Output]
+args = "--title \"useful title\""
 ```
 
-Each new `args` specification instantiates a new test execution with the most-recently-specified outputs.
+### Multiple Invocations
+
+Multiple _Output_ sections denote multiple test program invocations. Any
+failure of the test program to match its expected outputs will short-circuit
+and fail the test.
 
 ```
-stdout="6"
-args="2 * 3"
-args="3 * 2"
+[Output]
+stdout = ""
+args = "--no-output"
 
-stdout="7"
-args="3 + 4"
+[Output_newlines]
+stdout = "\n\n"
+args = "--newlines"
 ```
 
+### Updating Expected Outputs
+
+Pass the `--update` argument to `testrunner` to rewrite any failing test with
+the new outputs of the test.
+
 ## License
 Apache2 or MIT

testrunner.nim

@@ -1,3 +1,4 @@
+import std/strtabs
 import std/os
 import std/osproc
 import std/strutils
@@ -110,33 +111,39 @@ template time(duration, body): untyped =
     body
   duration =  epochTime() - t0
 
-proc cmpOutputs(test: TestSpec, stdout: string): TestStatus =
-  result = OK
-  for output in test.outputs:
-    var testOutput: string
-    if output.name == "stdout":
-      testOutput = stdout
+proc composeOutputs(test: TestSpec, stdout: string): TestOutputs =
+  result = newTestOutputs()
+  for name, expected in test.outputs.pairs:
+    if name == "stdout":
+      result[name] = stdout
     else:
-      if not existsFile(output.name):
-        logFailure(test, OutputFileNotFound, output.name)
-        result = FAILED
+      if not existsFile(name):
         continue
+      result[name] = readFile(name)
+      removeFile(name)
+
+proc cmpOutputs(test: TestSpec, outputs: TestOutputs): TestStatus =
+  result = OK
+  for name, expected in test.outputs.pairs:
+    if name notin outputs:
+      logFailure(test, OutputFileNotFound, name)
+      result = FAILED
+      continue
 
-      testOutput = readFile(output.name)
+    let
+      testOutput = outputs[name]
 
     # Would be nice to do a real diff here instead of simple compare
     if test.timestampPeg.len > 0:
-      if not cmpIgnorePegs(testOutput, output.expectedOutput, peg(test.timestampPeg), pegXid):
-        logFailure(test, OutputsDiffer, output.name, output.expectedOutput, testOutput)
+      if not cmpIgnorePegs(testOutput, expected,
+                           peg(test.timestampPeg), pegXid):
+        logFailure(test, OutputsDiffer, name, expected, testOutput)
         result = FAILED
     else:
-      if not cmpIgnoreDefaultTimestamps(testOutput, output.expectedOutput):
-        logFailure(test, OutputsDiffer, output.name, output.expectedOutput, testOutput)
+      if not cmpIgnoreDefaultTimestamps(testOutput, expected):
+        logFailure(test, OutputsDiffer, name, expected, testOutput)
         result = FAILED
 
-    if output.name != "stdout":
-      removeFile(output.name)
-
 proc compile(test: TestSpec): TestStatus =
   let
     source = test.config.path / test.program.addFileExt(".nim")
@@ -184,25 +191,32 @@ proc compile(test: TestSpec): TestStatus =
 proc execute(test: TestSpec): TestStatus =
   if test.child != nil:
     result = test.child.execute
+  if result notin {OK, SKIPPED}:
+    return
+  var
+    cmd = test.binary
+  if not existsFile(cmd):
+    result = FAILED
+    logFailure(test, ExeFileNotFound)
   else:
-    var
-      cmd = test.binary
-    if not existsFile(cmd):
-      result = FAILED
-      logFailure(test, ExeFileNotFound)
-    else:
-      withinDir parentDir(cmd):
-        cmd = cmd.quoteShell & " " & test.args
+    withinDir parentDir(cmd):
+      cmd = cmd.quoteShell & " " & test.args
+      let
+        (output, exitCode) = execCmdEx(cmd)
+      if exitCode != 0:
+        # parseExecuteOutput() # Need to parse the run time failures?
+        logFailure(test, RuntimeError, output)
+        result = FAILED
+      else:
         let
-          (output, exitCode) = execCmdEx(cmd)
-        if exitCode != 0:
-          # parseExecuteOutput() # Need to parse the run time failures?
-          logFailure(test, RuntimeError, output)
-          result = FAILED
-        else:
-          result = test.cmpOutputs(output)
-    if test.child != nil:
-      result = test.child.execute
+          outputs = test.composeOutputs(output)
+        result = test.cmpOutputs(outputs)
+        # perform an update of the testfile if requested and required
+        if test.config.update and result == FAILED:
+          test.rewriteTestFile(outputs)
+          # we'll call this a `skip` because it's not strictly a failure
+          # and we want any dependent testing to proceed as usual.
+          result = SKIPPED
 
 proc scanTestPath(path: string): seq[string] =
   if fileExists(path):

tests/hello/hello_multiple.test

@@ -1,7 +1,15 @@
 program = "hello"
 
+# my aim is true
 [Output]
-args = "steve"
-stdout = "hello steve\n"
-args = "jeff"
-stdout = "hello jeff\n"
+stdout = "hello world\n"
+
+# option 2
+[Output_larry_is_a_good_boy]
+args = "larry"
+stdout = "hello larry\n"
+
+# option 47
+[Output_stevie_is_a_good_boy]
+args = "stephen"
+stdout = "hello stephen\n"

testutils/config.nim

@@ -12,13 +12,15 @@ Options:
   --targets:"c c++ js objc" [Not implemented] Run tests for specified targets
   --include:"test1 test2"   Run only listed tests (space/comma seperated)
   --exclude:"test1 test2"   Skip listed tests (space/comma seperated)
+  --update                  Rewrite failed tests with new output
   --help                    Display this help and exit
 
   """.unindent.strip
 
 type
   TestConfig* = object
     path*: string
+    update*: bool
     includedTests*: seq[string]
     excludedTests*: seq[string]
     releaseBuild*: bool
@@ -45,6 +47,8 @@ proc processArguments*(): TestConfig =
         result.noThreads = true
       of "targets", "t":
         discard # not implemented
+      of "update":
+        result.update = true
       of "include":
         result.includedTests.add value.split(Whitespace + {','})
       of "exclude":

testutils/spec.nim

@@ -2,24 +2,25 @@ import std/os
 import std/parsecfg
 import std/strutils
 import std/streams
-import std/sequtils
+import std/strtabs
 
 import testutils/config
 
 const
   DefaultOses = @["linux", "macosx", "windows"]
 
 type
-  TestOutput = tuple[name: string, expectedOutput: string]
-  TestOutputs = seq[TestOutput]
+  TestOutputs* = StringTableRef
   TestSpec* = ref object
+    section*: string
     args*: string
     config*: TestConfig
     path*: string
     name*: string
     skip*: bool
     program*: string
     flags*: string
+    preamble*: seq[tuple[key: string; value: string]]
     outputs*: TestOutputs
     timestampPeg*: string
     errorMsg*: string
@@ -31,10 +32,14 @@ type
     os*: seq[string]
     child*: TestSpec
 
+proc newTestOutputs*(): StringTableRef =
+  result = newStringTable(mode = modeStyleInsensitive)
+
 proc clone*(spec: TestSpec): TestSpec =
+  ## create the parent of this test and set the child reference appropriately
   result = new(TestSpec)
   result[] = spec[]
-  result.outputs = newSeqOfCap[TestOutput](1)
+  result.outputs = newTestOutputs()
   result.args = ""
   result.child = spec
 
@@ -45,9 +50,13 @@ proc binary*(spec: TestSpec): string =
 proc defaults(spec: var TestSpec) =
   ## assert some default values for a given spec
   spec.os = DefaultOses
+  spec.outputs = newTestOutputs()
 
 proc consumeConfigEvent(spec: var TestSpec; event: CfgEvent) =
   ## parse a specification supplied prior to any sections
+
+  # save the key/value pair in case we need to write out the test file
+  spec.preamble.add (key: event.key, value: event.value)
   case event.key
   of "program":
     spec.program = event.value
@@ -70,6 +79,23 @@ proc consumeConfigEvent(spec: var TestSpec; event: CfgEvent) =
       flag = "--define:$#:$#" % [event.key, event.value]
     spec.flags.add flag.quoteShell & " "
 
+proc rewriteTestFile*(spec: TestSpec; outputs: TestOutputs) =
+  ## rewrite a test file with updated outputs after having run the tests
+  var
+    test = loadConfig(spec.path)
+  # take the opportunity to update an args statement if necessary
+  if spec.args != "":
+    test.setSectionKey(spec.section, "args", spec.args)
+  else:
+    test.delSectionKey(spec.section, "args")
+  # delete the old test outputs for completeness
+  for name, expected in spec.outputs.pairs:
+    test.delSectionKey(spec.section, name)
+  # add the new test outputs
+  for name, expected in outputs.pairs:
+    test.setSectionKey(spec.section, name, expected)
+  test.writeConfig(spec.path)
+
 proc parseTestFile*(filePath: string; config: TestConfig): TestSpec =
   ## parse a test input file into a spec
   result = new(TestSpec)
@@ -100,22 +126,18 @@ proc parseTestFile*(filePath: string; config: TestConfig): TestSpec =
           echo "Parsing warning:" & e.msg
         of cfgSectionStart:
           # starts with Output
-          outputSection = e.section.cmpIgnoreCase("Output") == 0
+          if e.section[0..len"Output"-1].cmpIgnoreCase("Output") == 0:
+            if outputSection:
+              # create our parent; the eternal chain
+              result = result.clone
+            outputSection = true
+            result.section = e.section
         of cfgKeyValuePair:
           if outputSection:
             if e.key.cmpIgnoreStyle("args") == 0:
-              # if this is the first args statement in the test,
-              # then we'll just use it.  otherwise, we'll clone
-              # ourselves and link to the test behind us.
-              if result.args.len == 0:
-                result.args = e.value
-              else:
-                # create our parent; the eternal chain
-                result = result.clone
+              result.args = e.value
             else:
-              # guard against stupidly pointing at redundant outputs
-              result.outputs = result.outputs.filterIt it.name != e.key
-              result.outputs.add((e.key, e.value))
+              result.outputs[e.key] = e.value
           else:
             result.consumeConfigEvent(e)
         of cfgOption: