From 5ccaf0ff200510cc2dd56f6cd58770e92ade282a Mon Sep 17 00:00:00 2001 From: Gregorius Soedharmo Date: Tue, 12 Oct 2021 02:05:56 +0700 Subject: [PATCH] Node runner should ignore runs that are not started by MNTR (#93) Node runner should ignore runs that are not being called by MNTR --- .../MultiNodeTestExecutorSpec.cs | 2 + .../Akka.MultiNode.TestAdapter.csproj | 1 + .../Environment/MultiNodeEnvironment.cs | 18 ---- .../Internal/NodeTest.cs | 2 +- .../MultiNodeTestAdapter.cs | 6 -- .../MultiNodeTestResult.cs | 9 +- .../MultiNodeTestRunner.cs | 94 ++++++++++++------- .../NodeRunner/Executor.cs | 12 ++- src/common.props | 2 +- 9 files changed, 80 insertions(+), 66 deletions(-) delete mode 100644 src/Akka.MultiNode.TestAdapter/Internal/Environment/MultiNodeEnvironment.cs diff --git a/src/Akka.MultiNode.TestAdapter.Tests/MultiNodeTestExecutorSpec.cs b/src/Akka.MultiNode.TestAdapter.Tests/MultiNodeTestExecutorSpec.cs index 85a2608..a0e2f23 100644 --- a/src/Akka.MultiNode.TestAdapter.Tests/MultiNodeTestExecutorSpec.cs +++ b/src/Akka.MultiNode.TestAdapter.Tests/MultiNodeTestExecutorSpec.cs @@ -2,6 +2,7 @@ using Akka.MultiNode.TestAdapter.SampleTests; using Akka.MultiNode.TestAdapter.SampleTests.Metadata; using Akka.MultiNode.TestAdapter.Tests.Helpers; +using Akka.Remote.TestKit; using FluentAssertions; using Microsoft.VisualStudio.TestPlatform.ObjectModel; using Xunit; @@ -17,6 +18,7 @@ public MultiNodeTestExecutorSpec() { _sampleTestsAssemblyPath = Path.GetFullPath(SampleTestsMetadata.AssemblyFileName); File.Exists(_sampleTestsAssemblyPath).Should().BeTrue($"Assemblies with samples should exist at {_sampleTestsAssemblyPath}"); + CommandLine.Initialize(new []{"-Dmultinode.test-runner=\"multinode\""}); } [Fact] diff --git a/src/Akka.MultiNode.TestAdapter/Akka.MultiNode.TestAdapter.csproj b/src/Akka.MultiNode.TestAdapter/Akka.MultiNode.TestAdapter.csproj index d4408a7..96124f9 100644 --- a/src/Akka.MultiNode.TestAdapter/Akka.MultiNode.TestAdapter.csproj +++ b/src/Akka.MultiNode.TestAdapter/Akka.MultiNode.TestAdapter.csproj @@ -6,6 +6,7 @@ $(NetStandardLibVersion) true true + 8.0 diff --git a/src/Akka.MultiNode.TestAdapter/Internal/Environment/MultiNodeEnvironment.cs b/src/Akka.MultiNode.TestAdapter/Internal/Environment/MultiNodeEnvironment.cs deleted file mode 100644 index 84953e9..0000000 --- a/src/Akka.MultiNode.TestAdapter/Internal/Environment/MultiNodeEnvironment.cs +++ /dev/null @@ -1,18 +0,0 @@ -using Akka.Remote.TestKit; - -namespace Akka.MultiNode.TestAdapter.Internal.Environment -{ - /// - /// MultiNodeEnvironment - /// - public static class MultiNodeEnvironment - { - /// - /// Initializes multi-node test environment. Used by and NodeRunner. - /// - public static void Initialize() - { - System.Environment.SetEnvironmentVariable(MultiNodeFactAttribute.MultiNodeTestEnvironmentName, "1"); - } - } -} \ No newline at end of file diff --git a/src/Akka.MultiNode.TestAdapter/Internal/NodeTest.cs b/src/Akka.MultiNode.TestAdapter/Internal/NodeTest.cs index 5046a79..5006a9c 100644 --- a/src/Akka.MultiNode.TestAdapter/Internal/NodeTest.cs +++ b/src/Akka.MultiNode.TestAdapter/Internal/NodeTest.cs @@ -13,7 +13,7 @@ public class NodeTest public string Role { get; set; } public MultiNodeTest Test { get; set; } - public string Name => $"{Test.TestName}_node{Node}[{Role}]"; + public string Name => $"{Test.MethodName}_node{Node}[{Role}]"; } } diff --git a/src/Akka.MultiNode.TestAdapter/MultiNodeTestAdapter.cs b/src/Akka.MultiNode.TestAdapter/MultiNodeTestAdapter.cs index 416e570..e447634 100644 --- a/src/Akka.MultiNode.TestAdapter/MultiNodeTestAdapter.cs +++ b/src/Akka.MultiNode.TestAdapter/MultiNodeTestAdapter.cs @@ -6,7 +6,6 @@ using System.Linq; using System.Xml; using Akka.MultiNode.TestAdapter.Internal; -using Akka.MultiNode.TestAdapter.Internal.Environment; using Microsoft.VisualStudio.TestPlatform.ObjectModel; using Microsoft.VisualStudio.TestPlatform.ObjectModel.Adapter; using Microsoft.VisualStudio.TestPlatform.ObjectModel.Logging; @@ -27,11 +26,6 @@ namespace Akka.MultiNode.TestAdapter [ExtensionUri(Constants.ExecutorUriString)] public class MultiNodeTestAdapter : ITestDiscoverer, ITestExecutor { - public MultiNodeTestAdapter() - { - MultiNodeEnvironment.Initialize(); - } - /// /// Discovers the tests available from the provided container. /// diff --git a/src/Akka.MultiNode.TestAdapter/MultiNodeTestResult.cs b/src/Akka.MultiNode.TestAdapter/MultiNodeTestResult.cs index 3806c27..285e569 100644 --- a/src/Akka.MultiNode.TestAdapter/MultiNodeTestResult.cs +++ b/src/Akka.MultiNode.TestAdapter/MultiNodeTestResult.cs @@ -42,7 +42,12 @@ public TestStatus Status { { if (!string.IsNullOrWhiteSpace(Test.SkipReason)) return TestStatus.Skipped; - return NodeResults.Any(result => result.Result == TestStatus.Failed) ? TestStatus.Failed : TestStatus.Passed; + + if(NodeResults.Any(result => result.Result == TestStatus.Failed)) + return TestStatus.Failed; + if (NodeResults.Any(result => result.Result == TestStatus.Skipped)) + return TestStatus.Skipped; + return TestStatus.Passed; } } /// @@ -57,7 +62,7 @@ public TestStatus Status { /// public override string ToString() { - var sb = new StringBuilder($"Test {Test.TestName}: {Status}"); + var sb = new StringBuilder($"Test {Test.MethodName}: {Status}"); if (Test.SkipReason != null) sb.Append(" Skipped: ").Append(Test.SkipReason); foreach (var node in NodeResults) diff --git a/src/Akka.MultiNode.TestAdapter/MultiNodeTestRunner.cs b/src/Akka.MultiNode.TestAdapter/MultiNodeTestRunner.cs index 73b46b5..8d623d0 100644 --- a/src/Akka.MultiNode.TestAdapter/MultiNodeTestRunner.cs +++ b/src/Akka.MultiNode.TestAdapter/MultiNodeTestRunner.cs @@ -19,15 +19,12 @@ using System.Threading.Tasks; using Akka.Actor; using Akka.IO; -using Akka.MultiNode.TestAdapter.Helpers; using Akka.MultiNode.TestAdapter.Internal; -using Akka.MultiNode.TestAdapter.Internal.Environment; using Akka.MultiNode.TestAdapter.Internal.Persistence; using Akka.MultiNode.TestAdapter.Internal.Sinks; using Akka.MultiNode.TestAdapter.Internal.TrxReporter; using Akka.MultiNode.TestAdapter.NodeRunner; using Akka.Remote.TestKit; -using Akka.Util; using Xunit; using ErrorMessage = Xunit.Sdk.ErrorMessage; @@ -100,7 +97,6 @@ private void Initialize(string assemblyPath, MultiNodeTestRunnerOptions options) EnableAllSinks(assemblyPath, options); // Set MNTR environment for correct tests discovery - MultiNodeEnvironment.Initialize(); } public MultiNodeTestResult ExecuteSpec(MultiNodeTest test, MultiNodeTestRunnerOptions options) @@ -113,6 +109,7 @@ public MultiNodeTestResult ExecuteSpec(MultiNodeTest test, MultiNodeTestRunnerOp : _tcpLogger.Ask(TcpLoggingServer.GetBoundPort.Instance).Result; TestStarted?.Invoke(test); + Environment.SetEnvironmentVariable(MultiNodeFactAttribute.MultiNodeTestEnvironmentName, "1"); try { if (options.SpecNames != null && @@ -132,10 +129,19 @@ public MultiNodeTestResult ExecuteSpec(MultiNodeTest test, MultiNodeTestRunnerOp var nodes = test.Nodes; var result = RunSpec(options, test, listenPort); - if(result.Status == MultiNodeTestResult.TestStatus.Failed) - TestFailed?.Invoke(result); - else - TestPassed?.Invoke(result); + switch (result.Status) + { + case MultiNodeTestResult.TestStatus.Passed: + TestPassed?.Invoke(result); + return result; + case MultiNodeTestResult.TestStatus.Failed: + TestFailed?.Invoke(result); + return result; + case MultiNodeTestResult.TestStatus.Skipped: + TestSkipped?.Invoke(test, "Must be run using Akka.MultiNode.TestAdapter"); + return null; + } + return result; } catch (Exception e) @@ -144,6 +150,10 @@ public MultiNodeTestResult ExecuteSpec(MultiNodeTest test, MultiNodeTestRunnerOp PublishRunnerMessage(e.Message); return null; } + finally + { + Environment.SetEnvironmentVariable(MultiNodeFactAttribute.MultiNodeTestEnvironmentName, null); + } } /// @@ -173,17 +183,23 @@ public void Dispose() /// public static (List Tests, List Errors) DiscoverSpecs(string assemblyPath) { - MultiNodeEnvironment.Initialize(); - - using (var controller = new XunitFrontController(AppDomainSupport.IfAvailable, assemblyPath)) + Environment.SetEnvironmentVariable(MultiNodeFactAttribute.MultiNodeTestEnvironmentName, "1"); + try { - using (var discovery = new Discovery(assemblyPath)) + using (var controller = new XunitFrontController(AppDomainSupport.IfAvailable, assemblyPath)) { - controller.Find(false, discovery, TestFrameworkOptions.ForDiscovery()); - discovery.Finished.WaitOne(); - return (discovery.MultiNodeTests, discovery.Errors); + using (var discovery = new Discovery(assemblyPath)) + { + controller.Find(false, discovery, TestFrameworkOptions.ForDiscovery()); + discovery.Finished.WaitOne(); + return (discovery.MultiNodeTests, discovery.Errors); + } } } + finally + { + Environment.SetEnvironmentVariable(MultiNodeFactAttribute.MultiNodeTestEnvironmentName, null); + } } private List DiscoverAndRunSpecs(string assemblyPath, MultiNodeTestRunnerOptions options) @@ -226,12 +242,20 @@ private List DiscoverAndRunSpecs(string assemblyPath, Multi // Run test on several nodes and report results var result = RunSpec(options, test, listenPort); - if(result.Status == MultiNodeTestResult.TestStatus.Failed) - TestFailed?.Invoke(result); - else - TestPassed?.Invoke(result); - - testResults.Add(result); + switch (result.Status) + { + case MultiNodeTestResult.TestStatus.Failed: + TestFailed?.Invoke(result); + testResults.Add(result); + break; + case MultiNodeTestResult.TestStatus.Passed: + TestPassed?.Invoke(result); + testResults.Add(result); + break; + case MultiNodeTestResult.TestStatus.Skipped: + TestSkipped?.Invoke(test, test.SkipReason); + continue; + } } catch (Exception e) { @@ -250,7 +274,7 @@ private MultiNodeTestResult RunSpec(MultiNodeTestRunnerOptions options, MultiNod var timelineCollector = TestRunSystem.ActorOf(Props.Create(() => new TimelineLogCollectorActor())); //TODO: might need to do some validation here to avoid the 260 character max path error on Windows - var folder = Directory.CreateDirectory(Path.Combine(options.OutputDirectory, test.TestName)); + var folder = Directory.CreateDirectory(Path.Combine(options.OutputDirectory, test.MethodName)); var testOutputDir = folder.FullName; var testResult = new MultiNodeTestResult(test); @@ -270,7 +294,8 @@ private MultiNodeTestResult RunSpec(MultiNodeTestRunnerOptions options, MultiNod $@"-Dmultinode.role=""{nodeTest.Role}""", $@"-Dmultinode.listen-address={options.ListenAddress}", $@"-Dmultinode.listen-port={listenPort}", - $@"-Dmultinode.test-assembly=""{test.AssemblyPath}""" + $@"-Dmultinode.test-assembly=""{test.AssemblyPath}""", + "-Dmultinode.test-runner=\"multinode\"" }; // Configure process for node @@ -319,7 +344,7 @@ private void DumpAggregatedSpecLogs( if (result.Status == MultiNodeTestResult.TestStatus.Failed) { - var failedSpecPath = Path.GetFullPath(Path.Combine(options.OutputDirectory, options.FailedSpecsDirectory, $"{result.Test.TestName}.txt")); + var failedSpecPath = Path.GetFullPath(Path.Combine(options.OutputDirectory, options.FailedSpecsDirectory, $"{result.Test.MethodName}.txt")); var dumpFailureArtifactTask = timelineCollector.Ask(new TimelineLogCollectorActor.DumpToFile(failedSpecPath)); dumpTasks.Add(dumpFailureArtifactTask); result.Attachments.Add(new MultiNodeTestResult.Attachment{Title = "Fail log", Path = failedSpecPath}); @@ -339,9 +364,12 @@ private void WaitForNodeExit(MultiNodeTestResult result, List<(NodeTest, Process process.WaitForExit(); Console.WriteLine($"Process for test {test.Name} finished with code {process.ExitCode}"); var nodeResult = result.NodeResults.First(n => n.Index == test.Node); - nodeResult.Result = process.ExitCode == 0 - ? MultiNodeTestResult.TestStatus.Passed - : MultiNodeTestResult.TestStatus.Failed; + nodeResult.Result = process.ExitCode switch + { + 0 => MultiNodeTestResult.TestStatus.Passed, + 2 => MultiNodeTestResult.TestStatus.Skipped, + _ => MultiNodeTestResult.TestStatus.Failed + }; } } finally @@ -366,7 +394,7 @@ private Process StartNodeProcess( var nodeIndex = nodeTest.Node; var nodeRole = nodeTest.Role; var logFilePath = Path.GetFullPath(Path.Combine(specFolder.FullName, $"node{nodeIndex}__{nodeRole}__{_platformName}.txt")); - var nodeInfo = new TimelineLogCollectorActor.NodeInfo(nodeIndex, nodeRole, _platformName, nodeTest.Test.TestName); + var nodeInfo = new TimelineLogCollectorActor.NodeInfo(nodeIndex, nodeRole, _platformName, nodeTest.Test.MethodName); var fileActor = TestRunSystem.ActorOf(Props.Create(() => new FileSystemAppenderActor(logFilePath))); result.Attachments.Add(new MultiNodeTestResult.Attachment{Title = $"Node {nodeIndex} [{nodeRole}]", Path = logFilePath}); @@ -377,7 +405,7 @@ private Process StartNodeProcess( { if (p.ExitCode == 0) { - ReportSpecPassFromExitCode(nodeIndex, nodeRole, closureTest.Test.TestName); + ReportSpecPassFromExitCode(nodeIndex, nodeRole, closureTest.Test.MethodName); } }; opt.OutputDataReceived = (sender, eventArgs) => @@ -493,12 +521,8 @@ MessageSink CreateVisualizerFileSink() return new FileSystemMessageSink(visualizerProps); } - var fileSystemSink = CommandLine.GetProperty("multinode.enable-filesink"); - if (!string.IsNullOrEmpty(fileSystemSink)) - { - SinkCoordinator.Tell(new SinkCoordinator.EnableSink(CreateJsonFileSink())); - SinkCoordinator.Tell(new SinkCoordinator.EnableSink(CreateVisualizerFileSink())); - } + SinkCoordinator.Tell(new SinkCoordinator.EnableSink(CreateJsonFileSink())); + SinkCoordinator.Tell(new SinkCoordinator.EnableSink(CreateVisualizerFileSink())); } private void AbortTcpLoggingServer() diff --git a/src/Akka.MultiNode.TestAdapter/NodeRunner/Executor.cs b/src/Akka.MultiNode.TestAdapter/NodeRunner/Executor.cs index 65733ac..5ddde1f 100644 --- a/src/Akka.MultiNode.TestAdapter/NodeRunner/Executor.cs +++ b/src/Akka.MultiNode.TestAdapter/NodeRunner/Executor.cs @@ -14,7 +14,6 @@ using System.Threading; using Akka.Actor; using Akka.IO; -using Akka.MultiNode.TestAdapter.Internal.Environment; using Akka.MultiNode.TestAdapter.Internal.Sinks; using Akka.Remote.TestKit; using Xunit; @@ -32,10 +31,15 @@ public int Execute(string[] args) var maxProcessWaitTimeout = TimeSpan.FromMinutes(5); IActorRef logger = null; + Environment.SetEnvironmentVariable(MultiNodeFactAttribute.MultiNodeTestEnvironmentName, "1"); try { CommandLine.Initialize(args); + var runner = CommandLine.GetPropertyOrDefault("multinode.test-runner", null); + if (runner != "multinode") + return 2; + var nodeIndex = CommandLine.GetInt32("multinode.index"); var nodeRole = CommandLine.GetProperty("multinode.role"); var assemblyFileName = CommandLine.GetProperty("multinode.test-assembly"); @@ -53,8 +57,6 @@ public int Execute(string[] args) var tcpClient = logger = system.ActorOf(); system.Tcp().Tell(new Tcp.Connect(listenEndpoint), tcpClient); - MultiNodeEnvironment.Initialize(); - // In NetCore, if the assembly file hasn't been touched, // XunitFrontController would fail loading external assemblies and its dependencies. @@ -182,6 +184,10 @@ public int Execute(string[] args) Environment.Exit(1); //signal failure return 1; } + finally + { + Environment.SetEnvironmentVariable(MultiNodeFactAttribute.MultiNodeTestEnvironmentName, null); + } void FlushLogMessages() { diff --git a/src/common.props b/src/common.props index 0714fad..921a580 100644 --- a/src/common.props +++ b/src/common.props @@ -24,7 +24,7 @@ netstandard2.0 6.1.0 2.9.0 - 1.4.26 + 1.4.27 akka;actors;actor model;Akka;concurrency