diff --git a/README.md b/README.md index 4acb170..36c383c 100644 --- a/README.md +++ b/README.md @@ -10,6 +10,7 @@ Options: --targets:"c c++ js objc" [Not implemented] Run tests for specified targets --include:"test1 test2" Run only listed tests (space/comma seperated) --exclude:"test1 test2" Skip listed tests (space/comma seperated) +--update Rewrite failed tests with new output --help Display this help and exit ``` @@ -55,32 +56,46 @@ added in a test file: ### Outputs For outputs to be compared, the output string should be set to the output -name (stdout or filename) from within the "Output" section, e.g.: +name (stdout or filename) from within an _Output_ section: + ``` [Output] stdout="""expected stdout output""" file.log="""expected file output""" ``` + Triple quotes can be used for multiple lines. ### Supplying Command-line Arguments -Specify command-line arguments as an escaped string in the following syntax: +Optionally specify command-line arguments as an escaped string in the following +syntax inside any _Output_ section: ``` -args="arg1 arg2 arg3" +[Output] +args = "--title \"useful title\"" ``` -Each new `args` specification instantiates a new test execution with the most-recently-specified outputs. +### Multiple Invocations + +Multiple _Output_ sections denote multiple test program invocations. Any +failure of the test program to match its expected outputs will short-circuit +and fail the test. ``` -stdout="6" -args="2 * 3" -args="3 * 2" +[Output] +stdout = "" +args = "--no-output" -stdout="7" -args="3 + 4" +[Output_newlines] +stdout = "\n\n" +args = "--newlines" ``` +### Updating Expected Outputs + +Pass the `--update` argument to `testrunner` to rewrite any failing test with +the new outputs of the test. + ## License Apache2 or MIT diff --git a/testrunner.nim b/testrunner.nim index a524449..f62b3f6 100644 --- a/testrunner.nim +++ b/testrunner.nim @@ -1,3 +1,4 @@ +import std/strtabs import std/os import std/osproc import std/strutils @@ -110,33 +111,39 @@ template time(duration, body): untyped = body duration = epochTime() - t0 -proc cmpOutputs(test: TestSpec, stdout: string): TestStatus = - result = OK - for output in test.outputs: - var testOutput: string - if output.name == "stdout": - testOutput = stdout +proc composeOutputs(test: TestSpec, stdout: string): TestOutputs = + result = newTestOutputs() + for name, expected in test.outputs.pairs: + if name == "stdout": + result[name] = stdout else: - if not existsFile(output.name): - logFailure(test, OutputFileNotFound, output.name) - result = FAILED + if not existsFile(name): continue + result[name] = readFile(name) + removeFile(name) + +proc cmpOutputs(test: TestSpec, outputs: TestOutputs): TestStatus = + result = OK + for name, expected in test.outputs.pairs: + if name notin outputs: + logFailure(test, OutputFileNotFound, name) + result = FAILED + continue - testOutput = readFile(output.name) + let + testOutput = outputs[name] # Would be nice to do a real diff here instead of simple compare if test.timestampPeg.len > 0: - if not cmpIgnorePegs(testOutput, output.expectedOutput, peg(test.timestampPeg), pegXid): - logFailure(test, OutputsDiffer, output.name, output.expectedOutput, testOutput) + if not cmpIgnorePegs(testOutput, expected, + peg(test.timestampPeg), pegXid): + logFailure(test, OutputsDiffer, name, expected, testOutput) result = FAILED else: - if not cmpIgnoreDefaultTimestamps(testOutput, output.expectedOutput): - logFailure(test, OutputsDiffer, output.name, output.expectedOutput, testOutput) + if not cmpIgnoreDefaultTimestamps(testOutput, expected): + logFailure(test, OutputsDiffer, name, expected, testOutput) result = FAILED - if output.name != "stdout": - removeFile(output.name) - proc compile(test: TestSpec): TestStatus = let source = test.config.path / test.program.addFileExt(".nim") @@ -184,25 +191,32 @@ proc compile(test: TestSpec): TestStatus = proc execute(test: TestSpec): TestStatus = if test.child != nil: result = test.child.execute + if result notin {OK, SKIPPED}: + return + var + cmd = test.binary + if not existsFile(cmd): + result = FAILED + logFailure(test, ExeFileNotFound) else: - var - cmd = test.binary - if not existsFile(cmd): - result = FAILED - logFailure(test, ExeFileNotFound) - else: - withinDir parentDir(cmd): - cmd = cmd.quoteShell & " " & test.args + withinDir parentDir(cmd): + cmd = cmd.quoteShell & " " & test.args + let + (output, exitCode) = execCmdEx(cmd) + if exitCode != 0: + # parseExecuteOutput() # Need to parse the run time failures? + logFailure(test, RuntimeError, output) + result = FAILED + else: let - (output, exitCode) = execCmdEx(cmd) - if exitCode != 0: - # parseExecuteOutput() # Need to parse the run time failures? - logFailure(test, RuntimeError, output) - result = FAILED - else: - result = test.cmpOutputs(output) - if test.child != nil: - result = test.child.execute + outputs = test.composeOutputs(output) + result = test.cmpOutputs(outputs) + # perform an update of the testfile if requested and required + if test.config.update and result == FAILED: + test.rewriteTestFile(outputs) + # we'll call this a `skip` because it's not strictly a failure + # and we want any dependent testing to proceed as usual. + result = SKIPPED proc scanTestPath(path: string): seq[string] = if fileExists(path): diff --git a/tests/hello/hello_multiple.test b/tests/hello/hello_multiple.test index 8a4ca1d..b4091fd 100644 --- a/tests/hello/hello_multiple.test +++ b/tests/hello/hello_multiple.test @@ -1,7 +1,15 @@ program = "hello" +# my aim is true [Output] -args = "steve" -stdout = "hello steve\n" -args = "jeff" -stdout = "hello jeff\n" +stdout = "hello world\n" + +# option 2 +[Output_larry_is_a_good_boy] +args = "larry" +stdout = "hello larry\n" + +# option 47 +[Output_stevie_is_a_good_boy] +args = "stephen" +stdout = "hello stephen\n" diff --git a/testutils/config.nim b/testutils/config.nim index 952229b..f1f7d40 100644 --- a/testutils/config.nim +++ b/testutils/config.nim @@ -12,6 +12,7 @@ Options: --targets:"c c++ js objc" [Not implemented] Run tests for specified targets --include:"test1 test2" Run only listed tests (space/comma seperated) --exclude:"test1 test2" Skip listed tests (space/comma seperated) + --update Rewrite failed tests with new output --help Display this help and exit """.unindent.strip @@ -19,6 +20,7 @@ Options: type TestConfig* = object path*: string + update*: bool includedTests*: seq[string] excludedTests*: seq[string] releaseBuild*: bool @@ -45,6 +47,8 @@ proc processArguments*(): TestConfig = result.noThreads = true of "targets", "t": discard # not implemented + of "update": + result.update = true of "include": result.includedTests.add value.split(Whitespace + {','}) of "exclude": diff --git a/testutils/spec.nim b/testutils/spec.nim index 7941936..e475886 100644 --- a/testutils/spec.nim +++ b/testutils/spec.nim @@ -2,7 +2,7 @@ import std/os import std/parsecfg import std/strutils import std/streams -import std/sequtils +import std/strtabs import testutils/config @@ -10,9 +10,9 @@ const DefaultOses = @["linux", "macosx", "windows"] type - TestOutput = tuple[name: string, expectedOutput: string] - TestOutputs = seq[TestOutput] + TestOutputs* = StringTableRef TestSpec* = ref object + section*: string args*: string config*: TestConfig path*: string @@ -20,6 +20,7 @@ type skip*: bool program*: string flags*: string + preamble*: seq[tuple[key: string; value: string]] outputs*: TestOutputs timestampPeg*: string errorMsg*: string @@ -31,10 +32,14 @@ type os*: seq[string] child*: TestSpec +proc newTestOutputs*(): StringTableRef = + result = newStringTable(mode = modeStyleInsensitive) + proc clone*(spec: TestSpec): TestSpec = + ## create the parent of this test and set the child reference appropriately result = new(TestSpec) result[] = spec[] - result.outputs = newSeqOfCap[TestOutput](1) + result.outputs = newTestOutputs() result.args = "" result.child = spec @@ -45,9 +50,13 @@ proc binary*(spec: TestSpec): string = proc defaults(spec: var TestSpec) = ## assert some default values for a given spec spec.os = DefaultOses + spec.outputs = newTestOutputs() proc consumeConfigEvent(spec: var TestSpec; event: CfgEvent) = ## parse a specification supplied prior to any sections + + # save the key/value pair in case we need to write out the test file + spec.preamble.add (key: event.key, value: event.value) case event.key of "program": spec.program = event.value @@ -70,6 +79,23 @@ proc consumeConfigEvent(spec: var TestSpec; event: CfgEvent) = flag = "--define:$#:$#" % [event.key, event.value] spec.flags.add flag.quoteShell & " " +proc rewriteTestFile*(spec: TestSpec; outputs: TestOutputs) = + ## rewrite a test file with updated outputs after having run the tests + var + test = loadConfig(spec.path) + # take the opportunity to update an args statement if necessary + if spec.args != "": + test.setSectionKey(spec.section, "args", spec.args) + else: + test.delSectionKey(spec.section, "args") + # delete the old test outputs for completeness + for name, expected in spec.outputs.pairs: + test.delSectionKey(spec.section, name) + # add the new test outputs + for name, expected in outputs.pairs: + test.setSectionKey(spec.section, name, expected) + test.writeConfig(spec.path) + proc parseTestFile*(filePath: string; config: TestConfig): TestSpec = ## parse a test input file into a spec result = new(TestSpec) @@ -100,22 +126,18 @@ proc parseTestFile*(filePath: string; config: TestConfig): TestSpec = echo "Parsing warning:" & e.msg of cfgSectionStart: # starts with Output - outputSection = e.section.cmpIgnoreCase("Output") == 0 + if e.section[0..len"Output"-1].cmpIgnoreCase("Output") == 0: + if outputSection: + # create our parent; the eternal chain + result = result.clone + outputSection = true + result.section = e.section of cfgKeyValuePair: if outputSection: if e.key.cmpIgnoreStyle("args") == 0: - # if this is the first args statement in the test, - # then we'll just use it. otherwise, we'll clone - # ourselves and link to the test behind us. - if result.args.len == 0: - result.args = e.value - else: - # create our parent; the eternal chain - result = result.clone + result.args = e.value else: - # guard against stupidly pointing at redundant outputs - result.outputs = result.outputs.filterIt it.name != e.key - result.outputs.add((e.key, e.value)) + result.outputs[e.key] = e.value else: result.consumeConfigEvent(e) of cfgOption: