diff --git a/README.md b/README.md index 16d60bb..807c4e2 100644 --- a/README.md +++ b/README.md @@ -6,6 +6,8 @@ This is a test runner for running integration/regression tests for It uses the same format for identifying, selecting and running tests as [dotnet-bunny](https://github.com/redhat-developer/dotnet-bunny/). +It produces results in various forms, including a junit-compatible xml file. + # Building Use the following command to build the `turkey` program and place it in the diff --git a/Turkey.Tests/TestOutputFormatTest.cs b/Turkey.Tests/TestOutputFormatTest.cs new file mode 100644 index 0000000..c5d965c --- /dev/null +++ b/Turkey.Tests/TestOutputFormatTest.cs @@ -0,0 +1,77 @@ +using System; +using System.IO; +using System.Threading.Tasks; +using Xunit; + +namespace Turkey.Tests +{ + public class JUnitOutputTest + { + [Fact] + public async Task EmptyResultsProduceBasicXml() + { + var resultsFile = new FileInfo(Path.GetTempFileName()); + + var j = new TestOutputFormats.JUnitOutput(resultsFile); + await j.AfterRunningAllTestsAsync(null); + var xml = File.ReadAllText(resultsFile.FullName); + + var expectedXml = @" +"; + + Assert.Equal(expectedXml, xml); + + resultsFile.Delete(); + } + + [Fact] + public async Task SingleTestWithPassingResultProducesValidXml() + { + var resultsFile = new FileInfo(Path.GetTempFileName()); + + var j = new TestOutputFormats.JUnitOutput(resultsFile); + var result = new TestResult(TestStatus.Passed, "", ""); + await j.AfterRunningTestAsync("foo", result, new TimeSpan(0)); + await j.AfterRunningAllTestsAsync(null); + var xml = File.ReadAllText(resultsFile.FullName); + + var expectedXml = @" + + + + + +"; + + Assert.Equal(expectedXml, xml); + + resultsFile.Delete(); + } + + [Fact] + public async Task ControlCharactersInTestOutputAreNotPresentInXml() + { + var resultsFile = new FileInfo(Path.GetTempFileName()); + + var j = new TestOutputFormats.JUnitOutput(resultsFile); + var result = new TestResult(TestStatus.Passed, + standardOutput: "\u0001\u0002\u0003\u0004\u0005aaa\u0006\u0007\u0008", + standardError: "\u001A\u001B\u001Cbbb\u001d"); + await j.AfterRunningTestAsync("foo", result, new TimeSpan(0)); + await j.AfterRunningAllTestsAsync(null); + var xml = File.ReadAllText(resultsFile.FullName); + + var expectedXml = @" + + + aaa + bbb + +"; + + Assert.Equal(expectedXml, xml); + + resultsFile.Delete(); + } + } +} diff --git a/Turkey/IEnumerableExtensions.cs b/Turkey/IEnumerableExtensions.cs new file mode 100644 index 0000000..023805e --- /dev/null +++ b/Turkey/IEnumerableExtensions.cs @@ -0,0 +1,17 @@ +using System; +using System.Collections.Generic; +using System.Threading.Tasks; + +namespace Turkey +{ + static class IEnumerableExtensions + { + public static async Task ForEachAsync(this IEnumerable items, Func task) + { + foreach (T item in items) + { + await task(item); + } + } + } +} diff --git a/Turkey/Program.cs b/Turkey/Program.cs index daa1b79..551fc9e 100644 --- a/Turkey/Program.cs +++ b/Turkey/Program.cs @@ -99,12 +99,19 @@ public static async Task Run(string testRoot, DotNet dotnet = new DotNet(); - TestOutput outputFormat = new TestOutputFormats.NewOutput(logWriter); + TestOutput defaultOutput = new TestOutputFormats.NewOutput(logWriter); if (compatible) { - outputFormat = new TestOutputFormats.DotNetBunnyOutput(logWriter); + defaultOutput = new TestOutputFormats.DotNetBunnyOutput(logWriter); } + TestOutput junitOutput = new TestOutputFormats.JUnitOutput(logDir); + + var testOutputs = new List() { + defaultOutput, + junitOutput, + }; + List platformIds = new PlatformId().CurrentIds; Console.WriteLine($"Current platform is: {string.Join(", ", platformIds)}"); @@ -125,7 +132,8 @@ public static async Task Run(string testRoot, nuGetConfig: nuGetConfig); var cancellationTokenSource = new Func(() => new CancellationTokenSource(timeoutForEachTest)); - var results = await runner.ScanAndRunAsync(outputFormat, cancellationTokenSource); + + var results = await runner.ScanAndRunAsync(testOutputs, cancellationTokenSource); int exitCode = (results.Failed == 0) ? 0 : 1; return exitCode; diff --git a/Turkey/TestOutputFormat.cs b/Turkey/TestOutputFormat.cs index 0147ef3..21b3279 100644 --- a/Turkey/TestOutputFormat.cs +++ b/Turkey/TestOutputFormat.cs @@ -1,6 +1,9 @@ using System; -using System.Runtime; +using System.Collections.Generic; +using System.IO; +using System.Text.RegularExpressions; using System.Threading.Tasks; +using System.Xml; namespace Turkey { @@ -107,5 +110,114 @@ public async override Task AfterRunningAllTestsAsync(TestResults results) Console.WriteLine($"Total: {results.Total} Passed: {results.Passed} Failed: {results.Failed}"); } } + + public class JUnitOutput : TestOutput + { + private struct TestCase { + public string Name; + public string ClassName; + public bool Failed; + public bool Skipped; + public string Message; + public string StandardOutput; + public string StandardError; + } + + private List _testCases = new List(); + private FileInfo _resultsFile; + + public JUnitOutput(DirectoryInfo logDirectory) + : this(new FileInfo(Path.Combine(logDirectory.FullName, "results.xml"))) + { + } + + public JUnitOutput(FileInfo resultsFile) + { + _resultsFile = resultsFile; + } + + public async override Task AfterParsingTestAsync(string name, bool enabled) + { + + } + + public async override Task AfterRunningTestAsync(string name, TestResult result, TimeSpan testTime) + { + var testCase = new TestCase(); + testCase.Name = name; + testCase.ClassName = "TestSuite"; + testCase.Failed = (result.Status == TestStatus.Failed); + testCase.Skipped = (result.Status == TestStatus.Skipped); + testCase.Message = "see stdout/stderr"; + testCase.StandardOutput = result.StandardOutput; + testCase.StandardError = result.StandardError; + + _testCases.Add(testCase); + } + + public async override Task AfterRunningAllTestsAsync(TestResults results) + { + var settings = new XmlWriterSettings(); + settings.Indent = true; + + using (var writer = XmlWriter.Create(_resultsFile.FullName, settings)) + { + writer.WriteStartDocument(); + + writer.WriteStartElement("testsuite"); + + foreach (var testCase in _testCases) + { + writer.WriteStartElement("testcase"); + writer.WriteAttributeString("name", testCase.Name); + writer.WriteAttributeString("classname", testCase.ClassName); + + if (testCase.Skipped) + { + writer.WriteStartElement("skipped"); + writer.WriteAttributeString("message", testCase.Message); + writer.WriteEndElement(); + } + + if (testCase.Failed) + { + writer.WriteStartElement("failure"); + writer.WriteAttributeString("message", testCase.Message); + writer.WriteAttributeString("type", "AssertionError"); + writer.WriteEndElement(); + } + + if (testCase.StandardOutput != null) + { + writer.WriteStartElement("system-out"); + string standardOutput = RemoveInvalidXmlCharacters(testCase.StandardOutput); + writer.WriteString(standardOutput); + writer.WriteEndElement(); + } + + if (testCase.StandardError != null) + { + writer.WriteStartElement("system-err"); + string standardError = RemoveInvalidXmlCharacters(testCase.StandardError); + writer.WriteString(standardError); + writer.WriteEndElement(); + } + + writer.WriteEndElement(); + + } + + writer.WriteEndElement(); + + writer.WriteEndDocument(); + writer.Close(); + } + } + + private string RemoveInvalidXmlCharacters(string input) + { + return Regex.Replace(input, @"[\u0000-\u0008,\u000B,\u000C,\u000E-\u001F]", ""); + } + } } } diff --git a/Turkey/TestRunner.cs b/Turkey/TestRunner.cs index 7756ce8..942c514 100644 --- a/Turkey/TestRunner.cs +++ b/Turkey/TestRunner.cs @@ -3,7 +3,6 @@ using System.Diagnostics; using System.IO; using System.Linq; -using System.Runtime; using System.Threading; using System.Threading.Tasks; @@ -57,9 +56,9 @@ public TestRunner(SystemUnderTest system, DirectoryInfo root, bool verboseOutput this.nuGetConfig = nuGetConfig; } - public async Task ScanAndRunAsync(TestOutput output, Func GetNewCancellationToken) + public async Task ScanAndRunAsync(List outputs, Func GetNewCancellationToken) { - await output.AtStartupAsync(); + await outputs.ForEachAsync(output => output.AtStartupAsync()); TestResults results = new TestResults(); @@ -94,7 +93,7 @@ public async Task ScanAndRunAsync(TestOutput output, Func output.AfterParsingTestAsync(test.Descriptor.Name, !test.Skip)); if (test.Descriptor.Cleanup) { @@ -113,10 +112,10 @@ public async Task ScanAndRunAsync(TestOutput output, Func output.AfterRunningTestAsync(test.Descriptor.Name, result, testTimeWatch.Elapsed)); } - await output.AfterRunningAllTestsAsync(results); + await outputs.ForEachAsync(output => output.AfterRunningAllTestsAsync(results)); return results; }