From 1c5b2b3fd0fb78d52066352e8c56c2ff7391457e Mon Sep 17 00:00:00 2001 From: Tom Deseyn Date: Sat, 10 Jun 2023 14:07:48 +0200 Subject: [PATCH] Write the test log file as the test is running. --- Turkey.Tests/ProcessExtensionsTest.cs | 2 +- Turkey.Tests/TestOutputFormatTest.cs | 16 ++-- Turkey/BashTest.cs | 34 ++------ Turkey/DotNet.cs | 43 ++-------- Turkey/LogWriter.cs | 36 --------- Turkey/ProcessExtensions.cs | 49 +++++++---- Turkey/Program.cs | 18 +---- Turkey/Test.cs | 63 +++------------ Turkey/TestOutputFormat.cs | 112 +++++--------------------- Turkey/TestRunner.cs | 47 ++++++++--- Turkey/XUnitTest.cs | 48 ++--------- 11 files changed, 128 insertions(+), 340 deletions(-) delete mode 100644 Turkey/LogWriter.cs diff --git a/Turkey.Tests/ProcessExtensionsTest.cs b/Turkey.Tests/ProcessExtensionsTest.cs index f6e870d..6ef9d3c 100644 --- a/Turkey.Tests/ProcessExtensionsTest.cs +++ b/Turkey.Tests/ProcessExtensionsTest.cs @@ -40,7 +40,7 @@ public async Task WaitForExitAsync_DoesNotHangForOrphanedGrandChildren() using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(WaitTimeoutSeconds)); // The WaitForExit completes by cancellation. - await Assert.ThrowsAsync(() => process.WaitForExitAsync(cts.Token, new StringWriter(), new StringWriter())); + await Assert.ThrowsAsync(() => process.WaitForExitAsync(logger: msg => { }, cts.Token)); } finally { diff --git a/Turkey.Tests/TestOutputFormatTest.cs b/Turkey.Tests/TestOutputFormatTest.cs index 0004354..9c94e19 100644 --- a/Turkey.Tests/TestOutputFormatTest.cs +++ b/Turkey.Tests/TestOutputFormatTest.cs @@ -1,5 +1,6 @@ using System; using System.IO; +using System.Text; using System.Threading.Tasks; using Xunit; @@ -30,8 +31,8 @@ public async Task SingleTestWithPassingResultProducesValidXml() var resultsFile = new FileInfo(Path.GetTempFileName()); var j = new TestOutputFormats.JUnitOutput(resultsFile); - var result = new TestResult(TestStatus.Passed, "", ""); - await j.AfterRunningTestAsync("foo", result, new TimeSpan(0)); + var result = TestResult.Passed; + await j.AfterRunningTestAsync("foo", result, new StringBuilder(), new TimeSpan(0)); await j.AfterRunningAllTestsAsync(null); var xml = File.ReadAllText(resultsFile.FullName); @@ -39,7 +40,6 @@ public async Task SingleTestWithPassingResultProducesValidXml() - "; @@ -54,18 +54,16 @@ public async Task ControlCharactersInTestOutputAreNotPresentInXml() var resultsFile = new FileInfo(Path.GetTempFileName()); var j = new TestOutputFormats.JUnitOutput(resultsFile); - var result = new TestResult(TestStatus.Passed, - standardOutput: "\u0001\u0002\u0003\u0004\u0005aaa\u0006\u0007\u0008", - standardError: "\u001A\u001B\u001Cbbb\u001d"); - await j.AfterRunningTestAsync("foo", result, new TimeSpan(0)); + var result = TestResult.Passed; + var testLog = new StringBuilder("\u0001\u0002\u0003\u0004\u0005aaa\u0006\u0007\u0008\u001A\u001B\u001Cbbb\u001d"); + await j.AfterRunningTestAsync("foo", result, testLog, new TimeSpan(0)); await j.AfterRunningAllTestsAsync(null); var xml = File.ReadAllText(resultsFile.FullName); var expectedXml = @" - aaa - bbb + aaabbb "; diff --git a/Turkey/BashTest.cs b/Turkey/BashTest.cs index 50cfda1..3dd09ba 100644 --- a/Turkey/BashTest.cs +++ b/Turkey/BashTest.cs @@ -14,19 +14,13 @@ public BashTest(DirectoryInfo directory, SystemUnderTest system, string nuGetCon { } - protected override async Task InternalRunAsync(CancellationToken cancellationToken) + protected override async Task InternalRunAsync(Action logger, CancellationToken cancellationToken) { - var standardOutputWriter = new StringWriter(); - var standardErrorWriter = new StringWriter(); - FileInfo testFile = new FileInfo(Path.Combine(Directory.FullName, "test.sh")); if (!testFile.Exists) { - standardErrorWriter.WriteLine($"Unable to find 'test.sh' in {Directory.FullName}"); - return new TestResult( - status: TestStatus.Failed, - standardOutput: standardOutputWriter.ToString(), - standardError: standardErrorWriter.ToString()); + logger($"Unable to find 'test.sh' in {Directory.FullName}"); + return TestResult.Failed; } ProcessStartInfo startInfo = new ProcessStartInfo() @@ -44,27 +38,9 @@ protected override async Task InternalRunAsync(CancellationToken can startInfo.EnvironmentVariables.Add(key, value); } - standardOutputWriter.WriteLine($"Executing {startInfo.FileName} with arguments {startInfo.Arguments} in working directory {startInfo.WorkingDirectory}"); - using (Process p = Process.Start(startInfo)) - { - var status = TestStatus.Failed; - try - { - await p.WaitForExitAsync(cancellationToken, standardOutputWriter, standardErrorWriter); - status = (p.ExitCode == 0) ? TestStatus.Passed: TestStatus.Failed; - standardOutputWriter.WriteLine($"Exit Code: {p.ExitCode}"); - } - catch (OperationCanceledException) - { - standardOutputWriter.WriteLine("[[TIMEOUT]]"); - standardErrorWriter.WriteLine("[[TIMEOUT]]"); - } + int exitCode = await ProcessRunner.RunAsync(startInfo, logger, cancellationToken); - return new TestResult( - status: status, - standardOutput: standardOutputWriter.ToString(), - standardError: standardErrorWriter.ToString()); - } + return exitCode == 0 ? TestResult.Passed : TestResult.Failed; } } } diff --git a/Turkey/DotNet.cs b/Turkey/DotNet.cs index 2d428db..98a36ca 100644 --- a/Turkey/DotNet.cs +++ b/Turkey/DotNet.cs @@ -79,21 +79,7 @@ public Version LatestSdkVersion } } - public struct ProcessResult - { - public int ExitCode { get; } - public string StandardOutput { get; } - public string StandardError { get; } - - public ProcessResult(int exitCode, string stdout, string stderr) - { - ExitCode = exitCode; - StandardOutput = stdout; - StandardError = stderr; - } - } - - public static async Task BuildAsync(DirectoryInfo workingDirectory, IReadOnlyDictionary environment, CancellationToken token) + public static Task BuildAsync(DirectoryInfo workingDirectory, IReadOnlyDictionary environment, Action logger, CancellationToken token) { var arguments = new string[] { @@ -102,21 +88,16 @@ public static async Task BuildAsync(DirectoryInfo workingDirector "-p:UseSharedCompilation=false", "-m:1", }; - var result = await RunDotNetCommandAsync(workingDirectory, arguments, environment, token); - return result; + return RunDotNetCommandAsync(workingDirectory, arguments, environment, logger, token); } - public static async Task RunAsync(DirectoryInfo workingDirectory, IReadOnlyDictionary environment, CancellationToken token) - { - return await RunDotNetCommandAsync(workingDirectory, new string[] { "run", "--no-restore", "--no-build"} , environment, token); - } + public static Task RunAsync(DirectoryInfo workingDirectory, IReadOnlyDictionary environment, Action logger, CancellationToken token) + => RunDotNetCommandAsync(workingDirectory, new string[] { "run", "--no-restore", "--no-build"} , environment, logger, token); - public static async Task TestAsync(DirectoryInfo workingDirectory, IReadOnlyDictionary environment, CancellationToken token) - { - return await RunDotNetCommandAsync(workingDirectory, new string[] { "test", "--no-restore", "--no-build"} , environment, token); - } + public static Task TestAsync(DirectoryInfo workingDirectory, IReadOnlyDictionary environment, Action logger, CancellationToken token) + => RunDotNetCommandAsync(workingDirectory, new string[] { "test", "--no-restore", "--no-build"} , environment, logger, token); - private static async Task RunDotNetCommandAsync(DirectoryInfo workingDirectory, string[] commands, IReadOnlyDictionary environment, CancellationToken token) + private static async Task RunDotNetCommandAsync(DirectoryInfo workingDirectory, string[] commands, IReadOnlyDictionary environment, Action logger, CancellationToken token) { var arguments = string.Join(" ", commands); ProcessStartInfo startInfo = new ProcessStartInfo() @@ -134,15 +115,7 @@ private static async Task RunDotNetCommandAsync(DirectoryInfo wor startInfo.EnvironmentVariables.Add(key, value); } - using (var process = Process.Start(startInfo)) - { - StringWriter standardOutputWriter = new StringWriter(); - StringWriter standardErrorWriter = new StringWriter(); - await process.WaitForExitAsync(token, standardOutputWriter, standardErrorWriter); - int exitCode = exitCode = process.ExitCode; - - return new ProcessResult(exitCode, standardOutputWriter.ToString(), standardErrorWriter.ToString()); - } + return await ProcessRunner.RunAsync(startInfo, logger, token); } } } diff --git a/Turkey/LogWriter.cs b/Turkey/LogWriter.cs deleted file mode 100644 index 0a07a53..0000000 --- a/Turkey/LogWriter.cs +++ /dev/null @@ -1,36 +0,0 @@ -using System; -using System.IO; -using System.Text; -using System.Threading.Tasks; - -namespace Turkey -{ - public class LogWriter - { - private readonly DirectoryInfo logDirectory; - - public LogWriter(DirectoryInfo logDirectory) - { - this.logDirectory = logDirectory; - } - - public async Task WriteAsync(string testName, string standardOutput, string standardError) - { - var logFileName = $"logfile-{testName}.log"; - var path = Path.Combine(logDirectory.FullName, logFileName); - using (StreamWriter sw = new StreamWriter(path, false, Encoding.UTF8)) - { - await WriteAsync(sw, standardOutput, standardError); - } - } - - protected async Task WriteAsync(StreamWriter writer, string standardOutput, string standardError) - { - await writer.WriteAsync("# Standard Output:" + Environment.NewLine); - await writer.WriteAsync(standardOutput); - await writer.WriteAsync("# Standard Error:" + Environment.NewLine); - await writer.WriteAsync(standardError); - } - } - -} diff --git a/Turkey/ProcessExtensions.cs b/Turkey/ProcessExtensions.cs index ffe5185..d02bb2a 100644 --- a/Turkey/ProcessExtensions.cs +++ b/Turkey/ProcessExtensions.cs @@ -6,38 +6,55 @@ namespace Turkey { + public static class ProcessRunner + { + public static async Task RunAsync(ProcessStartInfo psi, Action logger, CancellationToken token) + { + logger($"Executing {psi.FileName} with arguments {psi.Arguments} in working directory {psi.WorkingDirectory}"); + using var process = Process.Start(psi); + await process.WaitForExitAsync(logger, token); + return process.ExitCode; + } + } + public static class ProcessExtensions { - public static async Task WaitForExitAsync(this Process process, CancellationToken token, TextWriter standardOutput, TextWriter standardError) + public static async Task WaitForExitAsync(this Process process, Action logger, CancellationToken token) { process.EnableRaisingEvents = true; - var outputHandler = new DataReceivedEventHandler( - (sender, e) => - { - if (e.Data != null) - { - standardOutput.WriteLine(e.Data); - } - }); - var errorHandler = new DataReceivedEventHandler( - (sender, e) => + bool captureOutput = true; + DataReceivedEventHandler logToLogger = (sender, e) => + { + if (e.Data != null) { - if (e.Data != null) + lock (logger) { - standardError.WriteLine(e.Data); + if (captureOutput) + { + logger(e.Data); + } } - }); - process.OutputDataReceived += outputHandler; + } + }; + process.OutputDataReceived += logToLogger; process.BeginOutputReadLine(); - process.ErrorDataReceived += errorHandler; + process.ErrorDataReceived += logToLogger; process.BeginErrorReadLine(); try { await process.WaitForExitAsync(token); + + logger($"Process Exit Code: {process.ExitCode}"); } catch (OperationCanceledException ex) { + lock (logger) + { + captureOutput = false; + } + logger($"Process wait for exit cancelled."); + try { process.Kill(entireProcessTree: true); diff --git a/Turkey/Program.cs b/Turkey/Program.cs index 19557c4..c111aa1 100644 --- a/Turkey/Program.cs +++ b/Turkey/Program.cs @@ -18,10 +18,6 @@ public class Program new string[] { "--verbose", "-v" }, "Show verbose output"); - public static readonly Option compatibleOption = new Option( - new string[] { "--compatible", "-c" }, - "Make output compatible with dotnet-bunny"); - public static readonly Option logDirectoryOption = new Option( new string[] { "--log-directory", "-l" }, "Set directory for writing log files"); @@ -43,7 +39,6 @@ public class Program public static async Task Run(string testRoot, bool verbose, - bool compatible, string logDirectory, string additionalFeed, IEnumerable trait, @@ -87,17 +82,11 @@ public static async Task Run(string testRoot, } Console.WriteLine($"Testing everything under {testRootDirectory.FullName}"); - LogWriter logWriter = new LogWriter(logDir); - Cleaner cleaner = new Cleaner(); DotNet dotnet = new DotNet(); - TestOutput defaultOutput = new TestOutputFormats.NewOutput(logWriter); - if (compatible) - { - defaultOutput = new TestOutputFormats.DotNetBunnyOutput(logWriter); - } + TestOutput defaultOutput = new TestOutputFormats.NewOutput(); TestOutput junitOutput = new TestOutputFormats.JUnitOutput(logDir); @@ -138,9 +127,7 @@ public static async Task Run(string testRoot, verboseOutput: verbose, nuGetConfig: nuGetConfig); - var cancellationTokenSource = new Func(() => new CancellationTokenSource(timeoutForEachTest)); - - var results = await runner.ScanAndRunAsync(testOutputs, cancellationTokenSource); + var results = await runner.ScanAndRunAsync(testOutputs, logDir.FullName, timeoutForEachTest); int exitCode = (results.Failed == 0) ? 0 : 1; return exitCode; @@ -257,7 +244,6 @@ static async Task Main(string[] args) testRootArgument.Arity = ArgumentArity.ZeroOrOne; rootCommand.AddArgument(testRootArgument); - rootCommand.AddOption(compatibleOption); rootCommand.AddOption(verboseOption); rootCommand.AddOption(logDirectoryOption); rootCommand.AddOption(additionalFeedOption); diff --git a/Turkey/Test.cs b/Turkey/Test.cs index ecc9254..dc3bc69 100644 --- a/Turkey/Test.cs +++ b/Turkey/Test.cs @@ -22,38 +22,10 @@ public class TestDescriptor } // TODO is this a strongly-typed enum in C#? - public enum TestStatus { + public enum TestResult { Passed, Failed, Skipped, } - public class TestResult - { - public TestStatus Status { get; } - public string StandardOutput { get; } - public string StandardError { get; } - - public TestResult(TestStatus status, string standardOutput, string standardError) - { - Status = status; - StandardOutput = standardOutput; - StandardError = standardError; - } - } - - struct PartialResult - { - public bool Success; - public string StandardOutput; - public string StandardError; - - public PartialResult(bool success, string stdout, string stderr) - { - Success = success; - StandardOutput = stdout; - StandardError = stderr; - } - } - public abstract class Test { public DirectoryInfo Directory { get; } @@ -71,14 +43,11 @@ public Test(DirectoryInfo testDirectory, SystemUnderTest system, string nuGetCon this.Skip = !enabled; } - public async Task RunAsync(CancellationToken cancelltionToken) + public async Task RunAsync(Action logger, CancellationToken cancelltionToken) { if (Skip) { - return new TestResult( - status: TestStatus.Skipped, - standardOutput: null, - standardError: null); + return TestResult.Skipped; } var path = Path.Combine(Directory.FullName, "nuget.config"); @@ -91,15 +60,9 @@ public async Task RunAsync(CancellationToken cancelltionToken) await File.WriteAllTextAsync(path, NuGetConfig); } - var result = await UpdateProjectFilesIfPresent(); - var stdout = result.StandardOutput; - var stderr = result.StandardError; - if (!result.Success) - { - return new TestResult(TestStatus.Failed, stdout, stderr); - } + UpdateProjectFilesIfPresent(); - var testResult = await InternalRunAsync(cancelltionToken); + var testResult = await InternalRunAsync(logger, cancelltionToken); if (!string.IsNullOrEmpty(NuGetConfig)) { @@ -109,14 +72,14 @@ public async Task RunAsync(CancellationToken cancelltionToken) return testResult; } - private async Task UpdateProjectFilesIfPresent() + private void UpdateProjectFilesIfPresent() { if (SystemUnderTest.RuntimeVersion < Version.Parse("2.0")) { var projectJsonPath = Path.Combine(this.Directory.FullName, "project.json"); if (File.Exists(projectJsonPath)) { - return await CopyProjectJsonFile(); + CopyProjectJsonFile(); } } else @@ -125,13 +88,12 @@ private async Task UpdateProjectFilesIfPresent() var csprojPath = Path.Combine(this.Directory.FullName, csprojFile); if (File.Exists(csprojPath)) { - return await UpdateCsprojVersion(csprojPath); + UpdateCsprojVersion(csprojPath); } } - return new PartialResult(true, "No project file to update", ""); } - private async Task CopyProjectJsonFile() + private void CopyProjectJsonFile() { string majorMinor = "" + SystemUnderTest.RuntimeVersion.Major + SystemUnderTest.RuntimeVersion.Minor; var fileName = $"resources/project{majorMinor}xunit.json"; @@ -139,7 +101,6 @@ private async Task CopyProjectJsonFile() var source = resourceLocation; var dest = Path.Combine(this.Directory.FullName, "project.json"); File.Copy(source, dest); - return new PartialResult(true, "", ""); } private static string FindResourceFile(string name) @@ -154,20 +115,18 @@ private static string FindResourceFile(string name) return resourceLocation; } - private async Task UpdateCsprojVersion(string csprojPath) + private void UpdateCsprojVersion(string csprojPath) { var contents = File.ReadAllText(csprojPath); var updatedContents = UpdateCsprojContents(contents); File.WriteAllText(csprojPath, updatedContents); - - return new PartialResult(true, "", ""); } private string UpdateCsprojContents(string contents) => new CsprojCompatibilityPatcher().Patch(contents, this.SystemUnderTest.RuntimeVersion); - protected abstract Task InternalRunAsync(CancellationToken cancellationToken); + protected abstract Task InternalRunAsync(Action logger, CancellationToken cancellationToken); } } diff --git a/Turkey/TestOutputFormat.cs b/Turkey/TestOutputFormat.cs index cf1b64d..9b2a935 100644 --- a/Turkey/TestOutputFormat.cs +++ b/Turkey/TestOutputFormat.cs @@ -2,6 +2,7 @@ using System.Collections.Generic; using System.IO; using System.Linq; +using System.Text; using System.Text.RegularExpressions; using System.Threading.Tasks; using System.Xml; @@ -10,100 +11,38 @@ namespace Turkey { public class TestOutputFormats { - public class DotNetBunnyOutput : TestOutput - { - private readonly LogWriter _logWriter; - - public DotNetBunnyOutput(LogWriter writer) - { - this._logWriter = writer; - } - - public async override Task AtStartupAsync() - { - Console.WriteLine("\n\n(\\_/)\n(^_^)\n@(\")(\")\n\n".Replace("\n", Environment.NewLine)); - } - - public async override Task AfterParsingTestAsync(string name, bool enabled) - { - if (enabled) - { - Console.WriteLine($"Running {name}"); - } - } - - public async override Task AfterRunningTestAsync(string name, TestResult result, TimeSpan testTime) - { - string resultText; - switch (result.Status) - { - case TestStatus.Passed: - resultText = "PASS"; - Console.WriteLine($"Result: {resultText}" + Environment.NewLine); - break; - case TestStatus.Failed: - resultText = "FAIL - Code: 1"; - Console.WriteLine($"Result: {resultText}" + Environment.NewLine); - break; - default: - break; - } - - if (result.Status == TestStatus.Failed) - { - await _logWriter.WriteAsync(name, result.StandardOutput, result.StandardError); - } - - } - - public async override Task AfterRunningAllTestsAsync(TestResults results) - { - Console.WriteLine($"Total: {results.Total} Passed: {results.Passed} Failed: {results.Failed}"); - } - } - public class NewOutput : TestOutput { - - private readonly LogWriter _logWriter; - - public NewOutput(LogWriter writer) - { - this._logWriter = writer; - } - public async override Task AfterParsingTestAsync(string name, bool enabled) { var nameText = string.Format("{0,-60}", name); Console.Write(nameText); } - public async override Task AfterRunningTestAsync(string name, TestResult result, TimeSpan testTime) + public async override Task AfterRunningTestAsync(string name, TestResult result, StringBuilder testLog, TimeSpan testTime) { string elapsedTime = testTime.TotalMilliseconds.ToString(); string resultOutput = null; if (Console.IsOutputRedirected || Console.IsErrorRedirected) { - switch (result.Status) + switch (result) { - case TestStatus.Passed: resultOutput = "PASS"; break; - case TestStatus.Failed: resultOutput = "FAIL"; break; - case TestStatus.Skipped: resultOutput = "SKIP"; break; + case TestResult.Passed: resultOutput = "PASS"; break; + case TestResult.Failed: resultOutput = "FAIL"; break; + case TestResult.Skipped: resultOutput = "SKIP"; break; } Console.WriteLine($"[{resultOutput}]\t({elapsedTime}ms)"); } else { - switch (result.Status) + switch (result) { - case TestStatus.Passed: resultOutput = "\u001b[32mPASS\u001b[0m"; break; - case TestStatus.Failed: resultOutput = "\u001b[31mFAIL\u001b[0m"; break; - case TestStatus.Skipped: resultOutput = "SKIP"; break; + case TestResult.Passed: resultOutput = "\u001b[32mPASS\u001b[0m"; break; + case TestResult.Failed: resultOutput = "\u001b[31mFAIL\u001b[0m"; break; + case TestResult.Skipped: resultOutput = "SKIP"; break; } Console.WriteLine($"[{resultOutput}]\t({elapsedTime}ms)"); } - - await _logWriter.WriteAsync(name, result.StandardOutput, result.StandardError); } public async override Task AfterRunningAllTestsAsync(TestResults results) @@ -120,8 +59,7 @@ private struct TestCase { public bool Failed; public bool Skipped; public string Message; - public string StandardOutput; - public string StandardError; + public StringBuilder Log; } private List _testCases = new List(); @@ -137,21 +75,15 @@ public JUnitOutput(FileInfo resultsFile) _resultsFile = resultsFile; } - public async override Task AfterParsingTestAsync(string name, bool enabled) - { - - } - - public async override Task AfterRunningTestAsync(string name, TestResult result, TimeSpan testTime) + public async override Task AfterRunningTestAsync(string name, TestResult result, StringBuilder testLog, TimeSpan testTime) { var testCase = new TestCase(); testCase.Name = name; testCase.ClassName = "TestSuite"; - testCase.Failed = (result.Status == TestStatus.Failed); - testCase.Skipped = (result.Status == TestStatus.Skipped); - testCase.Message = "see stdout/stderr"; - testCase.StandardOutput = result.StandardOutput; - testCase.StandardError = result.StandardError; + testCase.Failed = result == TestResult.Failed; + testCase.Skipped = result == TestResult.Skipped; + testCase.Message = "see log"; + testCase.Log = testLog; _testCases.Add(testCase); } @@ -192,22 +124,14 @@ public async override Task AfterRunningAllTestsAsync(TestResults results) writer.WriteEndElement(); } - if (testCase.StandardOutput != null) + if (testCase.Log != null) { writer.WriteStartElement("system-out"); - string standardOutput = RemoveInvalidXmlCharacters(testCase.StandardOutput); + string standardOutput = RemoveInvalidXmlCharacters(testCase.Log.ToString()); writer.WriteString(standardOutput); writer.WriteEndElement(); } - if (testCase.StandardError != null) - { - writer.WriteStartElement("system-err"); - string standardError = RemoveInvalidXmlCharacters(testCase.StandardError); - writer.WriteString(standardError); - writer.WriteEndElement(); - } - writer.WriteEndElement(); } diff --git a/Turkey/TestRunner.cs b/Turkey/TestRunner.cs index 03c54ec..efdbcce 100644 --- a/Turkey/TestRunner.cs +++ b/Turkey/TestRunner.cs @@ -3,6 +3,7 @@ using System.Diagnostics; using System.IO; using System.Linq; +using System.Text; using System.Threading; using System.Threading.Tasks; @@ -43,7 +44,7 @@ public class TestOutput public async virtual Task AtStartupAsync() {} public async virtual Task BeforeTestAsync() {} public async virtual Task AfterParsingTestAsync(string name, bool enabled) {} - public async virtual Task AfterRunningTestAsync(string name, TestResult result, TimeSpan testTime) {} + public async virtual Task AfterRunningTestAsync(string name, TestResult result, StringBuilder testLog, TimeSpan testTime) {} public async virtual Task AfterRunningAllTestsAsync(TestResults results) {} } @@ -64,7 +65,7 @@ public TestRunner(SystemUnderTest system, DirectoryInfo root, bool verboseOutput this.nuGetConfig = nuGetConfig; } - public async Task ScanAndRunAsync(List outputs, Func GetNewCancellationToken) + public async Task ScanAndRunAsync(List outputs, string logDir, TimeSpan timeoutForEachTest) { await outputs.ForEachAsync(output => output.AtStartupAsync()); @@ -87,8 +88,6 @@ public async Task ScanAndRunAsync(List outputs, Func ScanAndRunAsync(List outputs, Func output.AfterParsingTestAsync(testName, !test.Skip)); + + using var cts = timeoutForEachTest == TimeSpan.Zero ? null : new CancellationTokenSource(timeoutForEachTest); + var cancellationToken = cts is null ? default : cts.Token; + Action testLogger = null; + + // Log to a file. + string logFileName = Path.Combine(logDir, $"logfile-{testName}.log"); + using var logFile = new StreamWriter(logFileName) { AutoFlush = true }; + Action logToLogFile = s => logFile.WriteLine(s); + testLogger += logToLogFile; - await outputs.ForEachAsync(output => output.AfterParsingTestAsync(test.Descriptor.Name, !test.Skip)); + // Log to a StringBuilder. + StringBuilder testLog = new(); + Action logToTestLog = s => testLog.AppendLine(s); + testLogger += logToTestLog; if (test.Descriptor.Cleanup) { await cleaner.CleanProjectLocalDotNetCruftAsync(); } - var result = await test.RunAsync(cancellationToken); + TestResult testResult; + try + { + testResult = await test.RunAsync(testLogger, cancellationToken); + } + catch (OperationCanceledException) + { + testLogger("[[TIMEOUT]]"); + testResult = TestResult.Failed; + } testTimeWatch.Stop(); results.Total++; - switch (result.Status) + switch (testResult) { - case TestStatus.Passed: results.Passed++; break; - case TestStatus.Failed: results.Failed++; break; - case TestStatus.Skipped: results.Skipped++; break; + case TestResult.Passed: results.Passed++; break; + case TestResult.Failed: results.Failed++; break; + case TestResult.Skipped: results.Skipped++; break; } - await outputs.ForEachAsync(output => output.AfterRunningTestAsync(test.Descriptor.Name, result, testTimeWatch.Elapsed)); + await outputs.ForEachAsync(output => output.AfterRunningTestAsync(testName, testResult, testLog, testTimeWatch.Elapsed)); } await outputs.ForEachAsync(output => output.AfterRunningAllTestsAsync(results)); diff --git a/Turkey/XUnitTest.cs b/Turkey/XUnitTest.cs index a3660c9..5bdd729 100644 --- a/Turkey/XUnitTest.cs +++ b/Turkey/XUnitTest.cs @@ -14,51 +14,19 @@ public XUnitTest(DirectoryInfo directory, SystemUnderTest system, string nuGetCo { } - protected override async Task InternalRunAsync(CancellationToken cancellationToken) + protected override async Task InternalRunAsync(Action logger, CancellationToken cancellationToken) { - PartialResult result; - string stdout = ""; - string stderr = ""; - try - { - result = await BuildProjectAsync(cancellationToken); - stdout += Environment.NewLine + result.StandardOutput; - stderr += Environment.NewLine + result.StandardError; - if (!result.Success) - { - return new TestResult(TestStatus.Failed, stdout, stderr); - } + bool success = await BuildProjectAsync(logger, cancellationToken) == 0 + && await RunTestProjectAsync(logger, cancellationToken) == 0; - result = await TestProjectAsync(cancellationToken); - stdout += Environment.NewLine + result.StandardOutput; - stderr += Environment.NewLine + result.StandardError; - } - catch (OperationCanceledException) - { - stdout += Environment.NewLine + "[[TIMEOUT]]" + Environment.NewLine; - stderr += Environment.NewLine + "[[TIMEOUT]]" + Environment.NewLine; - result.Success = false; - } - - return new TestResult(result.Success ? TestStatus.Passed : TestStatus.Failed, - stdout, - stderr); + return success ? TestResult.Passed : TestResult.Failed; } - private async Task BuildProjectAsync(CancellationToken token) - { - return ProcessResultToPartialResult(await DotNet.BuildAsync(Directory, SystemUnderTest.EnvironmentVariables, token)); - } - - private async Task TestProjectAsync(CancellationToken token) - { - return ProcessResultToPartialResult(await DotNet.TestAsync(Directory, SystemUnderTest.EnvironmentVariables, token)); - } + private Task BuildProjectAsync(Action logger, CancellationToken token) + => DotNet.BuildAsync(Directory, SystemUnderTest.EnvironmentVariables, logger, token); - private static PartialResult ProcessResultToPartialResult(DotNet.ProcessResult result) - { - return new PartialResult(result.ExitCode == 0, result.StandardOutput, result.StandardError); - } + private Task RunTestProjectAsync(Action logger, CancellationToken token) + => DotNet.TestAsync(Directory, SystemUnderTest.EnvironmentVariables, logger, token); } }