From f70b70b4bb30d777af07acc5315ebd5d1ff1ece2 Mon Sep 17 00:00:00 2001 From: nohwnd Date: Tue, 22 Feb 2022 17:45:26 +0100 Subject: [PATCH 001/112] Add test --- .../FeatureFlag/FeatureFlag.cs | 32 ++++- .../Friends.cs | 3 + test/Intent/ConsoleLogger.cs | 9 +- test/vstest.ProgrammerTests/Fakes/Fixture.cs | 9 +- .../Fakes/FixtureOptions.cs | 9 ++ test/vstest.ProgrammerTests/UnitTest1.cs | 121 +++++++++++++++++- .../vstest.ProgrammerTests.csproj | 2 +- 7 files changed, 170 insertions(+), 15 deletions(-) create mode 100644 test/vstest.ProgrammerTests/Fakes/FixtureOptions.cs diff --git a/src/Microsoft.TestPlatform.CoreUtilities/FeatureFlag/FeatureFlag.cs b/src/Microsoft.TestPlatform.CoreUtilities/FeatureFlag/FeatureFlag.cs index 60a4f0a525..a3c3929b3f 100644 --- a/src/Microsoft.TestPlatform.CoreUtilities/FeatureFlag/FeatureFlag.cs +++ b/src/Microsoft.TestPlatform.CoreUtilities/FeatureFlag/FeatureFlag.cs @@ -9,7 +9,7 @@ namespace Microsoft.VisualStudio.TestPlatform.Utilities; using System.Collections.Generic; // !!! FEATURES MUST BE KEPT IN SYNC WITH https://github.com/dotnet/sdk/blob/main/src/Cli/dotnet/commands/dotnet-test/VSTestFeatureFlag.cs !!! -internal partial class FeatureFlag : IFeatureFlag +internal sealed class FeatureFlag : IFeatureFlag { private static readonly Dictionary FeatureFlags = new(); @@ -19,18 +19,40 @@ internal partial class FeatureFlag : IFeatureFlag static FeatureFlag() { +#pragma warning disable CS0618 // Type or member is obsolete + Reset(); +#pragma warning restore CS0618 // Type or member is obsolete + } + + [Obsolete("Use this only from tests, and ctor.")] + internal static void Reset() + { + FeatureFlags.Clear(); FeatureFlags.Add(ARTIFACTS_POSTPROCESSING, true); FeatureFlags.Add(ARTIFACTS_POSTPROCESSING_SDK_KEEP_OLD_UX, false); + FeatureFlags.Add(MULTI_TFM_RUN, true); } - // Added for artifact porst-processing, it enable/disable the post processing. + [Obsolete("Use this only from tests.")] + internal static void SetFlag(string name, bool value) + { + if (!FeatureFlags.ContainsKey(name)) + throw new ArgumentException($"Feature flag {name} is a known feature flag."); + + FeatureFlags[name] = value; + } + + // Added for artifact post-processing, it enable/disable the post processing. // Added in 17.2-preview 7.0-preview - public static string ARTIFACTS_POSTPROCESSING = VSTEST_FEATURE + "_" + "ARTIFACTS_POSTPROCESSING"; + public static string ARTIFACTS_POSTPROCESSING = VSTEST_FEATURE + "_" + nameof(ARTIFACTS_POSTPROCESSING); - // Added for artifact porst-processing, it will show old output for dotnet sdk scenario. + // Added for artifact post-processing, it will show old output for dotnet sdk scenario. // It can be useful if we need to restore old UX in case users are parsing the console output. // Added in 17.2-preview 7.0-preview - public static string ARTIFACTS_POSTPROCESSING_SDK_KEEP_OLD_UX = VSTEST_FEATURE + "_" + "ARTIFACTS_POSTPROCESSING_SDK_KEEP_OLD_UX"; + public static string ARTIFACTS_POSTPROCESSING_SDK_KEEP_OLD_UX = VSTEST_FEATURE + "_" + nameof(ARTIFACTS_POSTPROCESSING_SDK_KEEP_OLD_UX); + + // Allow vstest.console to sources from multiple TFMs + public static string MULTI_TFM_RUN = VSTEST_FEATURE + "_" + nameof(MULTI_TFM_RUN); // For now we're checking env var. // We could add it also to some section inside the runsettings. diff --git a/src/Microsoft.TestPlatform.CoreUtilities/Friends.cs b/src/Microsoft.TestPlatform.CoreUtilities/Friends.cs index b897d33e2d..81d2194843 100644 --- a/src/Microsoft.TestPlatform.CoreUtilities/Friends.cs +++ b/src/Microsoft.TestPlatform.CoreUtilities/Friends.cs @@ -4,3 +4,6 @@ using System.Runtime.CompilerServices; [assembly: InternalsVisibleTo("vstest.console, PublicKey=002400000480000094000000060200000024000052534131000400000100010007d1fa57c4aed9f0a32e84aa0faefd0de9e8fd6aec8f87fb03766c834c99921eb23be79ad9d5dcc1dd9ad236132102900b723cf980957fc4e177108fc607774f29e8320e92ea05ece4e821c0a5efe8f1645c4c0c93c1ab99285d622caa652c1dfad63d745d6f2de5f17e5eaf0fc4963d261c8a12436518206dc093344d5ad293")] + + +[assembly: InternalsVisibleTo("vstest.ProgrammerTests, PublicKey=002400000480000094000000060200000024000052534131000400000100010007d1fa57c4aed9f0a32e84aa0faefd0de9e8fd6aec8f87fb03766c834c99921eb23be79ad9d5dcc1dd9ad236132102900b723cf980957fc4e177108fc607774f29e8320e92ea05ece4e821c0a5efe8f1645c4c0c93c1ab99285d622caa652c1dfad63d745d6f2de5f17e5eaf0fc4963d261c8a12436518206dc093344d5ad293")] diff --git a/test/Intent/ConsoleLogger.cs b/test/Intent/ConsoleLogger.cs index b41c484532..326794c4ef 100644 --- a/test/Intent/ConsoleLogger.cs +++ b/test/Intent/ConsoleLogger.cs @@ -44,13 +44,14 @@ public void WriteFrameworkError(Exception ex) private static string FormatMethodName(string methodName) { - var noUnderscores = methodName.Replace('_', ' '); // insert space before every capital letter or number that is after a non-capital letter - var spaced = Regex.Replace(noUnderscores, "(?<=[a-z])([A-Z0-9])", " $1"); + var spaced = Regex.Replace(methodName, "(?<=[a-z])([A-Z0-9])", " $1"); // insert space before every capital leter that is after a number var spaced2 = Regex.Replace(spaced, "(?<=[0-9]|^)([A-Z])", " $1"); - var newLines = spaced2.Replace("When", $"{Environment.NewLine} When") - .Replace("Then", $"{Environment.NewLine} Then"); + var newLines = spaced2 + .Replace("_When", $"{Environment.NewLine} When") + .Replace("_And", $"{Environment.NewLine} And") + .Replace("_Then", $"{Environment.NewLine} Then"); return newLines.ToLowerInvariant(); } diff --git a/test/vstest.ProgrammerTests/Fakes/Fixture.cs b/test/vstest.ProgrammerTests/Fakes/Fixture.cs index e8b66c1c4f..72eb4febf8 100644 --- a/test/vstest.ProgrammerTests/Fakes/Fixture.cs +++ b/test/vstest.ProgrammerTests/Fakes/Fixture.cs @@ -14,6 +14,7 @@ namespace vstest.ProgrammerTests.Fakes; using Microsoft.VisualStudio.TestPlatform.CrossPlatEngine.TestRunAttachmentsProcessing; using Microsoft.VisualStudio.TestPlatform.ObjectModel; using Microsoft.VisualStudio.TestPlatform.ObjectModel.Client; +using Microsoft.VisualStudio.TestPlatform.Utilities; internal class Fixture : IDisposable { @@ -36,7 +37,7 @@ internal class Fixture : IDisposable public ProtocolConfig ProtocolConfig { get; internal set; } - public Fixture() + public Fixture(FixtureOptions? fixtureOptions = null) { // This type is compiled only in DEBUG, and won't exist otherwise. #if DEBUG @@ -51,6 +52,12 @@ public Fixture() } #endif +#pragma warning disable CS0618 // Type or member is obsolete + FeatureFlag.Reset(); + fixtureOptions?.FeatureFlags?.ToList().ForEach(flag => FeatureFlag.SetFlag(flag.Key, flag.Value)); +#pragma warning restore CS0618 // Type or member is obsolete + + CurrentProcess = new FakeProcess(ErrorAggregator, @"X:\fake\vstest.console.exe", string.Empty, null, null, null, null, null); ProcessHelper = new FakeProcessHelper(ErrorAggregator, CurrentProcess); FileHelper = new FakeFileHelper(ErrorAggregator); diff --git a/test/vstest.ProgrammerTests/Fakes/FixtureOptions.cs b/test/vstest.ProgrammerTests/Fakes/FixtureOptions.cs new file mode 100644 index 0000000000..215c3649a6 --- /dev/null +++ b/test/vstest.ProgrammerTests/Fakes/FixtureOptions.cs @@ -0,0 +1,9 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +namespace vstest.ProgrammerTests.Fakes; + +internal class FixtureOptions +{ + public Dictionary? FeatureFlags { get; init; } +} diff --git a/test/vstest.ProgrammerTests/UnitTest1.cs b/test/vstest.ProgrammerTests/UnitTest1.cs index 61ca59cd84..0ab51d643c 100644 --- a/test/vstest.ProgrammerTests/UnitTest1.cs +++ b/test/vstest.ProgrammerTests/UnitTest1.cs @@ -25,6 +25,9 @@ namespace vstest.ProgrammerTests; #endif using vstest.ProgrammerTests.Fakes; +using Microsoft.VisualStudio.TestPlatform.Utilities; +using Intent; +using System.Linq.Expressions; // Tests are run by Intent library that is executed from our Program.Main. To debug press F5 in VS, and maybe mark just a single test with [Only]. // To just run, press Ctrl+F5 to run without debugging. It will use short timeout for abort in case something is wrong with your test. @@ -237,12 +240,19 @@ public async Task GivenMultipleMsTestAssembliesThatUseTheSameTargetFrameworkAndA fixture.ExecutedTests.Should().HaveCount(mstest1Dll.TestCount + mstest2Dll.TestCount); } - public async Task GivenMultipleMsTestAssembliesThatUseDifferentTargetFrameworkAndTheSameArchitecture_WhenTestsAreRun_ThenTwoTesthostsAreStartedBothForTheSameTFM() + [Only] + public async Task GivenMultipleMsTestAssembliesThatHaveTheSameArchitecture_AndHaveDifferentTargetFrameworks_AndMULTI_TFM_RUNFeatureFlagIsDisabled_WhenTestsAreRun_ThenTwoTesthostsAreStartedBothForTheSameTFM() { - // TODO: make vstest.console not start testhosts for incompatible sources. - // -- arrange - using var fixture = new Fixture(); + using var fixture = new Fixture( + new FixtureOptions + { + FeatureFlags = new Dictionary + { + [FeatureFlag.MULTI_TFM_RUN] = false + } + } + ); var mstest1Dll = new FakeTestDllBuilder() .WithPath(@"X:\fake\mstest1.dll") @@ -335,6 +345,109 @@ public async Task GivenMultipleMsTestAssembliesThatUseDifferentTargetFrameworkAn fixture.ExecutedTests.Should().HaveCount(mstest1Dll.TestCount); } + + [Only] + public async Task GivenMultipleMsTestAssembliesThatHaveTheSameArchitecture_AndHaveDifferentTargetFrameworks_AndMULTI_TFM_RUNFeatureFlagIsEnabled_WhenTestsAreRun_ThenTwoTesthostsWithTheCorrectTFMForTheDllAreUsed() + { + // -- arrange + using var fixture = new Fixture( + new FixtureOptions { + FeatureFlags = new Dictionary + { + [FeatureFlag.MULTI_TFM_RUN] = true + } + } + ); + + var mstest1Dll = new FakeTestDllBuilder() + .WithPath(@"X:\fake\mstest1.dll") + .WithFramework(KnownFrameworkNames.Net5) // <--- + .WithArchitecture(Architecture.X64) + .WithTestCount(11, 5) + .Build(); + + var testhost1Process = new FakeProcess(fixture.ErrorAggregator, @"X:\fake\testhost1.exe"); + + var runTests1 = new FakeTestHostResponsesBuilder() + .VersionCheck(5) + .ExecutionInitialize(FakeMessage.NoResponse) + .StartTestExecutionWithSources(mstest1Dll.TestResultBatches) + .SessionEnd(FakeMessage.NoResponse, afterAction: _ => testhost1Process.Exit()) + .Build(); + + var testhost1 = new FakeTestHostFixtureBuilder(fixture) + .WithTestDll(mstest1Dll) + .WithProcess(testhost1Process) + .WithResponses(runTests1) + .Build(); + + // -- + + var mstest2Dll = new FakeTestDllBuilder() + .WithPath(@"X:\fake\mstest2.dll") + .WithFramework(KnownFrameworkNames.Net48) // <--- + .WithArchitecture(Architecture.X64) + .WithTestCount(21, 5) + .Build(); + + var testhost2Process = new FakeProcess(fixture.ErrorAggregator, @"X:\fake\testhost2.exe"); + + var runTests2 = new FakeTestHostResponsesBuilder() + .VersionCheck(5) + .ExecutionInitialize(FakeMessage.NoResponse) + .StartTestExecutionWithSources(mstest2Dll.TestResultBatches) + .SessionEnd(FakeMessage.NoResponse, _ => testhost2Process.Exit()) + .Build(); + + var testhost2 = new FakeTestHostFixtureBuilder(fixture) + .WithTestDll(mstest2Dll) + .WithProcess(testhost2Process) + .WithResponses(runTests2) + .Build(); + + fixture.AddTestHostFixtures(testhost1, testhost2); + + var testRequestManager = fixture.BuildTestRequestManager(); + + mstest1Dll.FrameworkName.Should().NotBe(mstest2Dll.FrameworkName); + + // -- act + // TODO: Building whole default runconfiguration is needed here, because TestRequestManager does not ensure the basic settings are populated, + // and all methods that populate them just silently fail, so TestHostProvider does not get any useful settings. + var runConfiguration = new RunConfiguration().ToXml().OuterXml; + var testRunRequestPayload = new TestRunRequestPayload + { + Sources = new List { mstest1Dll.Path, mstest2Dll.Path }, + + RunSettings = $"{runConfiguration}" + }; + + await testRequestManager.ExecuteWithAbort(tm => tm.RunTests(testRunRequestPayload, testHostLauncher: null, fixture.TestRunEventsRegistrar, fixture.ProtocolConfig)); + + // -- assert + fixture.AssertNoErrors(); + // We figure out the framework for each assembly so there should be no incompatibility warnings + // REVIEW: This should be uncommented. Commenting it now, because it is helpful to see those warnings. + // fixture.TestRunEventsRegistrar.LoggedWarnings.Should().NotContainMatch("Test run detected DLL(s) which were built for different framework*"); + + // We started both testhosts, even thought we know one of them is incompatible. + fixture.ProcessHelper.Processes.Where(p => p.Started).Should().HaveCount(2); + var startWithSources1 = testhost1.FakeCommunicationChannel.ProcessedMessages.Single(m => m.Request.MessageType == MessageType.StartTestExecutionWithSources); + var startWithSources1Text = startWithSources1.Request.Payload.Select(t => t.ToString()).JoinBySpace(); + // We sent mstest1.dll. + startWithSources1Text.Should().Contain("mstest1.dll"); + // And we sent net5 as the target framework, because that is the framework of mstest1.dll. + startWithSources1Text.Should().Contain(KnownFrameworkStrings.Net5); + + var startWithSources2 = testhost2.FakeCommunicationChannel.ProcessedMessages.Single(m => m.Request.MessageType == MessageType.StartTestExecutionWithSources); + var startWithSources2Text = startWithSources2.Request.Payload.Select(t => t.ToString()).JoinBySpace(); + // We sent mstest2.dll. + startWithSources2Text.Should().Contain("mstest2.dll"); + // And we sent net48 as the target framework, because that is the framework of mstest2.dll. + startWithSources2Text.Should().Contain(mstest2Dll.FrameworkName.ToString()); + + fixture.ExecutedTests.Should().HaveCount(mstest1Dll.TestCount + mstest2Dll.TestCount); + } } // Test and improvmement ideas: diff --git a/test/vstest.ProgrammerTests/vstest.ProgrammerTests.csproj b/test/vstest.ProgrammerTests/vstest.ProgrammerTests.csproj index 21a59d0ab8..a87cdbc1a5 100644 --- a/test/vstest.ProgrammerTests/vstest.ProgrammerTests.csproj +++ b/test/vstest.ProgrammerTests/vstest.ProgrammerTests.csproj @@ -16,7 +16,7 @@ Exe - + From a1c902889c3fde1d2ed1a5a7c0c2c4266bb30a27 Mon Sep 17 00:00:00 2001 From: nohwnd Date: Wed, 23 Feb 2022 15:58:40 +0100 Subject: [PATCH 002/112] Passthru options. Discover request needs to fix settings on criteria as well. --- .../TestPlatform.cs | 31 +- .../Engine/ClientProtocol/ITestEngine.cs | 10 +- .../Parallel/ParallelOperationManager.cs | 62 +-- .../Parallel/ParallelProxyDiscoveryManager.cs | 140 ++++--- .../Parallel/ParallelProxyExecutionManager.cs | 151 +++++-- .../Client/ProxyDiscoveryManager.cs | 1 + .../TestEngine.cs | 147 ++++--- .../Utilities/SourceDetailHelper.cs | 29 ++ .../Client/Interfaces/ITestPlatform.cs | 6 +- .../SourceDetail.cs | 13 + .../Hosting/DefaultTestHostManager.cs | 3 +- src/vstest.console/CommandLine/InferHelper.cs | 178 ++++---- .../TestPlatformHelpers/TestRequestManager.cs | 380 +++++++++++------- .../Fakes/FakeTestRuntimeProviderManager.cs | 3 - .../Fakes/KnownFrameworkStrings.cs | 2 +- test/vstest.ProgrammerTests/UnitTest1.cs | 6 +- .../CommandLine/InferHelperTests.cs | 28 +- 17 files changed, 730 insertions(+), 460 deletions(-) create mode 100644 src/Microsoft.TestPlatform.CrossPlatEngine/Utilities/SourceDetailHelper.cs create mode 100644 src/Microsoft.TestPlatform.ObjectModel/SourceDetail.cs diff --git a/src/Microsoft.TestPlatform.Client/TestPlatform.cs b/src/Microsoft.TestPlatform.Client/TestPlatform.cs index f46702a5fc..70a8452dbb 100644 --- a/src/Microsoft.TestPlatform.Client/TestPlatform.cs +++ b/src/Microsoft.TestPlatform.Client/TestPlatform.cs @@ -85,7 +85,8 @@ protected internal TestPlatform( public IDiscoveryRequest CreateDiscoveryRequest( IRequestData requestData, DiscoveryCriteria discoveryCriteria!!, - TestPlatformOptions options) + TestPlatformOptions options, + Dictionary sourceToSourceDetailMap) { PopulateExtensions(discoveryCriteria.RunSettings, discoveryCriteria.Sources); @@ -93,12 +94,7 @@ public IDiscoveryRequest CreateDiscoveryRequest( ITestLoggerManager loggerManager = TestEngine.GetLoggerManager(requestData); loggerManager.Initialize(discoveryCriteria.RunSettings); - ITestRuntimeProvider testHostManager = _testHostProviderManager.GetTestHostManagerByRunConfiguration(discoveryCriteria.RunSettings); - TestPlatform.ThrowExceptionIfTestHostManagerIsNull(testHostManager, discoveryCriteria.RunSettings); - - testHostManager.Initialize(TestSessionMessageLogger.Instance, discoveryCriteria.RunSettings); - - IProxyDiscoveryManager discoveryManager = TestEngine.GetDiscoveryManager(requestData, testHostManager, discoveryCriteria); + IProxyDiscoveryManager discoveryManager = TestEngine.GetDiscoveryManager(requestData, discoveryCriteria, sourceToSourceDetailMap); discoveryManager.Initialize(options?.SkipDefaultAdapters ?? false); return new DiscoveryRequest(requestData, discoveryCriteria, discoveryManager, loggerManager); @@ -108,7 +104,8 @@ public IDiscoveryRequest CreateDiscoveryRequest( public ITestRunRequest CreateTestRunRequest( IRequestData requestData, TestRunCriteria testRunCriteria!!, - TestPlatformOptions options) + TestPlatformOptions options, + Dictionary sourceToSourceDetailMap) { IEnumerable sources = GetSources(testRunCriteria); PopulateExtensions(testRunCriteria.TestRunSettings, sources); @@ -117,23 +114,7 @@ public ITestRunRequest CreateTestRunRequest( ITestLoggerManager loggerManager = TestEngine.GetLoggerManager(requestData); loggerManager.Initialize(testRunCriteria.TestRunSettings); - // TODO: PERF: this will create a testhost manager, and then it will pass that to GetExecutionManager, where it will - // be used only when we will run in-process. If we don't run in process, we will throw away the manager we just - // created and let the proxy parallel callbacks to create a new one. This seems to be very easy to move to the GetExecutionManager, - // and safe as well, so we create the manager only once. - // TODO: Of course TestEngine.GetExecutionManager is public api... - ITestRuntimeProvider testHostManager = _testHostProviderManager.GetTestHostManagerByRunConfiguration(testRunCriteria.TestRunSettings); - TestPlatform.ThrowExceptionIfTestHostManagerIsNull(testHostManager, testRunCriteria.TestRunSettings); - - testHostManager.Initialize(TestSessionMessageLogger.Instance, testRunCriteria.TestRunSettings); - - // NOTE: The custom launcher should not be set when we have test session info available. - if (testRunCriteria.TestHostLauncher != null) - { - testHostManager.SetCustomLauncher(testRunCriteria.TestHostLauncher); - } - - IProxyExecutionManager executionManager = TestEngine.GetExecutionManager(requestData, testHostManager, testRunCriteria); + IProxyExecutionManager executionManager = TestEngine.GetExecutionManager(requestData, testRunCriteria, sourceToSourceDetailMap); executionManager.Initialize(options?.SkipDefaultAdapters ?? false); return new TestRunRequest(requestData, testRunCriteria, executionManager, loggerManager); diff --git a/src/Microsoft.TestPlatform.Common/Interfaces/Engine/ClientProtocol/ITestEngine.cs b/src/Microsoft.TestPlatform.Common/Interfaces/Engine/ClientProtocol/ITestEngine.cs index 8a1351842b..e7fc4e340d 100644 --- a/src/Microsoft.TestPlatform.Common/Interfaces/Engine/ClientProtocol/ITestEngine.cs +++ b/src/Microsoft.TestPlatform.Common/Interfaces/Engine/ClientProtocol/ITestEngine.cs @@ -5,6 +5,8 @@ namespace Microsoft.VisualStudio.TestPlatform.ObjectModel.Engine; +using System.Collections.Generic; + using Client; using Host; @@ -28,8 +30,8 @@ public interface ITestEngine /// An IProxyDiscoveryManager object that can do discovery. IProxyDiscoveryManager GetDiscoveryManager( IRequestData requestData, - ITestRuntimeProvider testHostManager, - DiscoveryCriteria discoveryCriteria); + DiscoveryCriteria discoveryCriteria, + Dictionary sourceToSourceDetailMap); /// /// Fetches the ExecutionManager for this engine. This manager would provide all @@ -45,8 +47,8 @@ IProxyDiscoveryManager GetDiscoveryManager( /// An IProxyExecutionManager object that can do execution. IProxyExecutionManager GetExecutionManager( IRequestData requestData, - ITestRuntimeProvider testHostManager, - TestRunCriteria testRunCriteria); + TestRunCriteria testRunCriteria, + Dictionary sourceToSourceDetailMap); /// /// Fetches the TestSessionManager for this engine. This manager would provide all diff --git a/src/Microsoft.TestPlatform.CrossPlatEngine/Client/Parallel/ParallelOperationManager.cs b/src/Microsoft.TestPlatform.CrossPlatEngine/Client/Parallel/ParallelOperationManager.cs index d4f3b23b47..7d8df891ec 100644 --- a/src/Microsoft.TestPlatform.CrossPlatEngine/Client/Parallel/ParallelOperationManager.cs +++ b/src/Microsoft.TestPlatform.CrossPlatEngine/Client/Parallel/ParallelOperationManager.cs @@ -16,31 +16,32 @@ namespace Microsoft.VisualStudio.TestPlatform.CrossPlatEngine.Client; using ObjectModel.Engine; /// -/// Abstract class having common parallel manager implementation +/// Common parallel manager functionality. /// -internal abstract class ParallelOperationManager : IParallelOperationManager, IDisposable +internal abstract class ParallelOperationManager : IParallelOperationManager, IDisposable { #region ConcurrentManagerInstanceData - protected Func CreateNewConcurrentManager { get; set; } + protected Func CreateNewConcurrentManager { get; } /// - /// Gets a value indicating whether hosts are shared. + /// Holds all active managers, so we can do actions on all of them, like initialize, run, cancel or close. /// - protected bool SharedHosts { get; private set; } - - private IDictionary _concurrentManagerHandlerMap; + // TODO: make this ConcurrentDictionary and use it's concurrent api, if we have the need. + private readonly IDictionary _concurrentManagerHandlerMap = new ConcurrentDictionary(); /// /// Singleton Instance of this class /// - protected static T s_instance; + protected static TParallelManager s_instance; /// /// Default number of Processes /// private int _currentParallelLevel; + protected int MaxParallelLevel { get; private set; } + #endregion #region Concurrency Keeper Objects @@ -53,20 +54,27 @@ internal abstract class ParallelOperationManager : IParallelOperationMana #endregion - protected ParallelOperationManager(Func createNewManager, int parallelLevel, bool sharedHosts) + protected ParallelOperationManager(Func createNewManager, int parallelLevel) { CreateNewConcurrentManager = createNewManager; - SharedHosts = sharedHosts; // Update Parallel Level - UpdateParallelLevel(parallelLevel); + // REVIEW: this "pre-starts" testhosts so we have a pool of them, this is the reason the number or + // parallel hosts is capped to the amount of sources so we don't "pre-start" too many of them + // instead we should take each source, look if it can be run by shared host, and if so try to + // grab a free host, run new one if we are below parallel level, or wait if we are at parallel + // level and everyone is busy if we have non-shared host we do just the two last options, run new + // one if current count is under parallel level, or wait till we can run new one. + // this.UpdateParallelLevel(parallelLevel); + + MaxParallelLevel = parallelLevel; } /// /// Remove and dispose a manager from concurrent list of manager. /// /// Manager to remove - public void RemoveManager(T manager) + public void RemoveManager(TParallelManager manager) { _concurrentManagerHandlerMap.Remove(manager); } @@ -76,7 +84,7 @@ public void RemoveManager(T manager) /// /// Manager to add /// eventHandler of the manager - public void AddManager(T manager, TU handler) + public void AddManager(TParallelManager manager, TEventHandler handler) { _concurrentManagerHandlerMap.Add(manager, handler); } @@ -87,7 +95,7 @@ public void AddManager(T manager, TU handler) /// /// Manager to update /// event handler to update for manager - public void UpdateHandlerForManager(T manager, TU handler) + public void UpdateHandlerForManager(TParallelManager manager, TEventHandler handler) { if (_concurrentManagerHandlerMap.ContainsKey(manager)) { @@ -103,7 +111,7 @@ public void UpdateHandlerForManager(T manager, TU handler) /// Get the event handler associated with the manager. /// /// Manager - public TU GetHandlerForGivenManager(T manager) + public TEventHandler GetHandlerForGivenManager(TParallelManager manager) { return _concurrentManagerHandlerMap[manager]; } @@ -119,7 +127,7 @@ public int GetConcurrentManagersCount() /// /// Get instances of all active concurrent manager /// - public IEnumerable GetConcurrentManagerInstances() + public IEnumerable GetConcurrentManagerInstances() { return _concurrentManagerHandlerMap.Keys.ToList(); } @@ -133,13 +141,7 @@ public void UpdateParallelLevel(int newParallelLevel) { if (_concurrentManagerHandlerMap == null) { - // not initialized yet - // create rest of concurrent clients other than default one - _concurrentManagerHandlerMap = new ConcurrentDictionary(); - for (int i = 0; i < newParallelLevel; i++) - { - AddManager(CreateNewConcurrentManager(), default); - } + throw new Exception("ParallelOperationManager.UpdateParallelLevel: This should not be used anymore, to pre-start hosts."); } else if (_currentParallelLevel != newParallelLevel) { @@ -147,10 +149,9 @@ public void UpdateParallelLevel(int newParallelLevel) // Create more concurrent clients and update the list if (_currentParallelLevel < newParallelLevel) { - for (int i = 0; i < newParallelLevel - _currentParallelLevel; i++) - { - AddManager(CreateNewConcurrentManager(), default); - } + // This path does not even seem to be used anywhere. + + throw new Exception("ParallelOperationManager.UpdateParallelLevel: This should not be used anymore, to ensure we add more hosts."); } else { @@ -190,7 +191,7 @@ public void Dispose() s_instance = default; } - protected void DoActionOnAllManagers(Action action, bool doActionsInParallel = false) + protected void DoActionOnAllManagers(Action action, bool doActionsInParallel = false) { if (_concurrentManagerHandlerMap != null && _concurrentManagerHandlerMap.Count > 0) { @@ -237,15 +238,16 @@ private void DoManagerAction(Action action) /// /// source data to work on - source file or testCaseList /// True, if data exists. False otherwise - protected bool TryFetchNextSource(IEnumerator enumerator, out TY source) + protected bool TryFetchNextSource(IEnumerator enumerator, out TSource source) { + // TODO: If only something like a concurrent queue existed. source = default; var hasNext = false; lock (_sourceEnumeratorLockObject) { if (enumerator != null && enumerator.MoveNext()) { - source = (TY)enumerator.Current; + source = (TSource)enumerator.Current; hasNext = source != null; } } diff --git a/src/Microsoft.TestPlatform.CrossPlatEngine/Client/Parallel/ParallelProxyDiscoveryManager.cs b/src/Microsoft.TestPlatform.CrossPlatEngine/Client/Parallel/ParallelProxyDiscoveryManager.cs index 114e6b0c44..326716efe5 100644 --- a/src/Microsoft.TestPlatform.CrossPlatEngine/Client/Parallel/ParallelProxyDiscoveryManager.cs +++ b/src/Microsoft.TestPlatform.CrossPlatEngine/Client/Parallel/ParallelProxyDiscoveryManager.cs @@ -21,10 +21,10 @@ namespace Microsoft.VisualStudio.TestPlatform.CrossPlatEngine.Client.Parallel; /// /// ParallelProxyDiscoveryManager that manages parallel discovery /// -internal class ParallelProxyDiscoveryManager : ParallelOperationManager, IParallelProxyDiscoveryManager +internal class ParallelProxyDiscoveryManager : ParallelOperationManager, IParallelProxyDiscoveryManager { private readonly IDataSerializer _dataSerializer; - + private readonly Dictionary _sourceToSourceDetailMap; private int _discoveryCompletedClients; private int _availableTestSources = -1; @@ -35,7 +35,7 @@ internal class ParallelProxyDiscoveryManager : ParallelOperationManager private readonly object _discoveryStatusLockObject = new(); - public ParallelProxyDiscoveryManager(IRequestData requestData, Func actualProxyManagerCreator, int parallelLevel, bool sharedHosts) - : this(requestData, actualProxyManagerCreator, JsonDataSerializer.Instance, parallelLevel, sharedHosts) + public ParallelProxyDiscoveryManager(IRequestData requestData, Func actualProxyManagerCreator, int parallelLevel, Dictionary sourceToSourceDetailMap) + : this(requestData, actualProxyManagerCreator, JsonDataSerializer.Instance, parallelLevel, sourceToSourceDetailMap) { } - internal ParallelProxyDiscoveryManager(IRequestData requestData, Func actualProxyManagerCreator, IDataSerializer dataSerializer, int parallelLevel, bool sharedHosts) - : base(actualProxyManagerCreator, parallelLevel, sharedHosts) + internal ParallelProxyDiscoveryManager(IRequestData requestData, Func actualProxyManagerCreator, IDataSerializer dataSerializer, int parallelLevel, Dictionary sourceToSourceDetailMap) + : base(actualProxyManagerCreator, parallelLevel) { _requestData = requestData; _dataSerializer = dataSerializer; + _sourceToSourceDetailMap = sourceToSourceDetailMap; } #region IProxyDiscoveryManager @@ -62,7 +63,9 @@ internal ParallelProxyDiscoveryManager(IRequestData requestData, Func public void Initialize(bool skipDefaultAdapters) { - DoActionOnAllManagers((proxyManager) => proxyManager.Initialize(skipDefaultAdapters), doActionsInParallel: true); + // The parent ctor does not pre-create any managers, save the info for later. + // DoActionOnAllManagers((proxyManager) => proxyManager.Initialize(skipDefaultAdapters), doActionsInParallel: true); + _skipDefaultAdapters = skipDefaultAdapters; } /// @@ -114,6 +117,7 @@ public void Close() public bool HandlePartialDiscoveryComplete(IProxyDiscoveryManager proxyDiscoveryManager, long totalTests, IEnumerable lastChunk, bool isAborted) { var allDiscoverersCompleted = false; + // REVIEW: Interlocked.Increment the count, and the condition below probably does not need to be in a lock? lock (_discoveryStatusLockObject) { // Each concurrent Executor calls this method @@ -146,27 +150,40 @@ when testhost crashed by itself and when user requested it (f.e. through TW) return true; } + // REVIEW: this was here before I did multi tfm work, this should be reviewed, because + // the comment builds on false premise, so maybe too much work is done here, because we should take shared hosts into account, and schedule + // the next source on the same manager if we have the possibility. + // and not kill testhost for every source. /* Discovery is not complete. Now when both.net framework and.net core projects can run in parallel we should clear manager and create new one for both cases. Otherwise `proxyDiscoveryManager` instance is alredy closed by now and it will give exception when trying to do some operation on it. */ + var SharedHosts = false; EqtTrace.Verbose("ParallelProxyDiscoveryManager: HandlePartialDiscoveryComplete: Replace discovery manager. Shared: {0}, Aborted: {1}.", SharedHosts, isAborted); RemoveManager(proxyDiscoveryManager); - proxyDiscoveryManager = CreateNewConcurrentManager(); - var parallelEventsHandler = new ParallelDiscoveryEventsHandler( - _requestData, - proxyDiscoveryManager, - _currentDiscoveryEventsHandler, - this, - _currentDiscoveryDataAggregator); - AddManager(proxyDiscoveryManager, parallelEventsHandler); + // If we have more sources, create manager for that source. + // The source determines which type of host to create, because it can have a framework + // and architecture associated with it. + if (TryFetchNextSource(_sourceEnumerator, out string source)) + { + var sourceDetail = _sourceToSourceDetailMap[source]; + proxyDiscoveryManager = CreateNewConcurrentManager(sourceDetail); + var parallelEventsHandler = new ParallelDiscoveryEventsHandler( + _requestData, + proxyDiscoveryManager, + _currentDiscoveryEventsHandler, + this, + _currentDiscoveryDataAggregator); + AddManager(proxyDiscoveryManager, parallelEventsHandler); + } - // Second, let's attempt to trigger discovery for the next source. - DiscoverTestsOnConcurrentManager(proxyDiscoveryManager); + // REVIEW: is this really how it should be done? Proxy manager can be if we don't pass any and if we don't have more sources? + // Let's attempt to trigger discovery for the source. + DiscoverTestsOnConcurrentManager(source, proxyDiscoveryManager); return false; } @@ -180,8 +197,16 @@ private void DiscoverTestsPrivate(ITestDiscoveryEventsHandler2 discoveryEventsHa // Reset the discovery complete data _discoveryCompletedClients = 0; + // REVIEW: what did I meant in the ERR comment below?? :D + // ERR: I need to schedule them until I reach maxParallelLevel or until I run out of sources. + // This won't schedule any source for discovery, because there are not concurrent managers. foreach (var concurrentManager in GetConcurrentManagerInstances()) { + if (!TryFetchNextSource(_sourceEnumerator, out string source)) + { + throw new InvalidOperationException("There are no more sources"); + } + var parallelEventsHandler = new ParallelDiscoveryEventsHandler( _requestData, concurrentManager, @@ -190,7 +215,7 @@ private void DiscoverTestsPrivate(ITestDiscoveryEventsHandler2 discoveryEventsHa _currentDiscoveryDataAggregator); UpdateHandlerForManager(concurrentManager, parallelEventsHandler); - DiscoverTestsOnConcurrentManager(concurrentManager); + DiscoverTestsOnConcurrentManager(source, concurrentManager); } } @@ -199,45 +224,50 @@ private void DiscoverTestsPrivate(ITestDiscoveryEventsHandler2 discoveryEventsHa /// Each concurrent discoverer calls this method, once its completed working on previous data /// /// Proxy discovery manager instance. - private void DiscoverTestsOnConcurrentManager(IProxyDiscoveryManager proxyDiscoveryManager) + private void DiscoverTestsOnConcurrentManager(string source, IProxyDiscoveryManager proxyDiscoveryManager) { - // Peek to see if we have sources to trigger a discovery - if (TryFetchNextSource(_sourceEnumerator, out string nextSource)) + if (source == null) { - EqtTrace.Verbose("ProxyParallelDiscoveryManager: Triggering test discovery for next source: {0}", nextSource); - - // Kick off another discovery task for the next source - var discoveryCriteria = new DiscoveryCriteria(new[] { nextSource }, _actualDiscoveryCriteria.FrequencyOfDiscoveredTestsEvent, _actualDiscoveryCriteria.DiscoveredTestEventTimeout, _actualDiscoveryCriteria.RunSettings); - discoveryCriteria.TestCaseFilter = _actualDiscoveryCriteria.TestCaseFilter; - Task.Run(() => - { - EqtTrace.Verbose("ParallelProxyDiscoveryManager: Discovery started."); - - proxyDiscoveryManager.DiscoverTests(discoveryCriteria, GetHandlerForGivenManager(proxyDiscoveryManager)); - }) - .ContinueWith(t => - { - // Just in case, the actual discovery couldn't start for an instance. Ensure that - // we call discovery complete since we have already fetched a source. Otherwise - // discovery will not terminate - EqtTrace.Error("ParallelProxyDiscoveryManager: Failed to trigger discovery. Exception: " + t.Exception); - - var handler = GetHandlerForGivenManager(proxyDiscoveryManager); - var testMessagePayload = new TestMessagePayload { MessageLevel = TestMessageLevel.Error, Message = t.Exception.ToString() }; - handler.HandleRawMessage(_dataSerializer.SerializePayload(MessageType.TestMessage, testMessagePayload)); - handler.HandleLogMessage(TestMessageLevel.Error, t.Exception.ToString()); - - // Send discovery complete. Similar logic is also used in ProxyDiscoveryManager.DiscoverTests. - // Differences: - // Total tests must be zero here since parallel discovery events handler adds the count - // Keep `lastChunk` as null since we don't want a message back to the IDE (discovery didn't even begin) - // Set `isAborted` as true since we want this instance of discovery manager to be replaced - var discoveryCompleteEventsArgs = new DiscoveryCompleteEventArgs(-1, true); - handler.HandleDiscoveryComplete(discoveryCompleteEventsArgs, null); - }, - TaskContinuationOptions.OnlyOnFaulted); + EqtTrace.Verbose("ProxyParallelDiscoveryManager: No sources available for discovery."); + return; } - EqtTrace.Verbose("ProxyParallelDiscoveryManager: No sources available for discovery."); + EqtTrace.Verbose("ProxyParallelDiscoveryManager: Triggering test discovery for next source: {0}", source); + + // Kick off another discovery task for the next source + var discoveryCriteria = new DiscoveryCriteria( + new[] { source }, + _actualDiscoveryCriteria.FrequencyOfDiscoveredTestsEvent, + _actualDiscoveryCriteria.DiscoveredTestEventTimeout, + _actualDiscoveryCriteria.RunSettings + ); + discoveryCriteria.TestCaseFilter = _actualDiscoveryCriteria.TestCaseFilter; + Task.Run(() => + { + EqtTrace.Verbose("ParallelProxyDiscoveryManager: Discovery started."); + + proxyDiscoveryManager.DiscoverTests(discoveryCriteria, GetHandlerForGivenManager(proxyDiscoveryManager)); + }) + .ContinueWith(t => + { + // Just in case, the actual discovery couldn't start for an instance. Ensure that + // we call discovery complete since we have already fetched a source. Otherwise + // discovery will not terminate + EqtTrace.Error("ParallelProxyDiscoveryManager: Failed to trigger discovery. Exception: " + t.Exception); + + var handler = GetHandlerForGivenManager(proxyDiscoveryManager); + var testMessagePayload = new TestMessagePayload { MessageLevel = TestMessageLevel.Error, Message = t.Exception.ToString() }; + handler.HandleRawMessage(_dataSerializer.SerializePayload(MessageType.TestMessage, testMessagePayload)); + handler.HandleLogMessage(TestMessageLevel.Error, t.Exception.ToString()); + + // Send discovery complete. Similar logic is also used in ProxyDiscoveryManager.DiscoverTests. + // Differences: + // Total tests must be zero here since parallel discovery events handler adds the count + // Keep `lastChunk` as null since we don't want a message back to the IDE (discovery didn't even begin) + // Set `isAborted` as true since we want this instance of discovery manager to be replaced + var discoveryCompleteEventsArgs = new DiscoveryCompleteEventArgs(-1, true); + handler.HandleDiscoveryComplete(discoveryCompleteEventsArgs, null); + }, + TaskContinuationOptions.OnlyOnFaulted); } } diff --git a/src/Microsoft.TestPlatform.CrossPlatEngine/Client/Parallel/ParallelProxyExecutionManager.cs b/src/Microsoft.TestPlatform.CrossPlatEngine/Client/Parallel/ParallelProxyExecutionManager.cs index 641d5cfcef..43a8d507f4 100644 --- a/src/Microsoft.TestPlatform.CrossPlatEngine/Client/Parallel/ParallelProxyExecutionManager.cs +++ b/src/Microsoft.TestPlatform.CrossPlatEngine/Client/Parallel/ParallelProxyExecutionManager.cs @@ -23,13 +23,15 @@ namespace Microsoft.VisualStudio.TestPlatform.CrossPlatEngine.Client.Parallel; using Microsoft.VisualStudio.TestPlatform.ObjectModel.Client; using ObjectModel.Engine; using ObjectModel.Logging; +using Microsoft.VisualStudio.TestPlatform.CrossPlatEngine.Utilities; /// /// ParallelProxyExecutionManager that manages parallel execution /// -internal class ParallelProxyExecutionManager : ParallelOperationManager, IParallelProxyExecutionManager +internal class ParallelProxyExecutionManager : ParallelOperationManager, IParallelProxyExecutionManager { private readonly IDataSerializer _dataSerializer; + private readonly Dictionary _sourceToSourceDetailMap; #region TestRunSpecificData @@ -69,21 +71,17 @@ internal class ParallelProxyExecutionManager : ParallelOperationManager actualProxyManagerCreator, int parallelLevel) - : this(requestData, actualProxyManagerCreator, JsonDataSerializer.Instance, parallelLevel, true) + public ParallelProxyExecutionManager(IRequestData requestData, Func actualProxyManagerCreator, int parallelLevel, Dictionary sourceToSourceDetailMap) + : this(requestData, actualProxyManagerCreator, JsonDataSerializer.Instance, parallelLevel, sourceToSourceDetailMap) { } - public ParallelProxyExecutionManager(IRequestData requestData, Func actualProxyManagerCreator, int parallelLevel, bool sharedHosts) - : this(requestData, actualProxyManagerCreator, JsonDataSerializer.Instance, parallelLevel, sharedHosts) - { - } - - internal ParallelProxyExecutionManager(IRequestData requestData, Func actualProxyManagerCreator, IDataSerializer dataSerializer, int parallelLevel, bool sharedHosts) - : base(actualProxyManagerCreator, parallelLevel, sharedHosts) + internal ParallelProxyExecutionManager(IRequestData requestData, Func actualProxyManagerCreator, IDataSerializer dataSerializer, int parallelLevel, Dictionary sourceToSourceDetailMap) + : base(actualProxyManagerCreator, parallelLevel) { _requestData = requestData; _dataSerializer = dataSerializer; + _sourceToSourceDetailMap = sourceToSourceDetailMap; } #region IProxyExecutionManager @@ -91,8 +89,7 @@ internal ParallelProxyExecutionManager(IRequestData requestData, Func proxyManager.Initialize(skipDefaultAdapters), doActionsInParallel: true); - IsInitialized = true; + // base class does not create any manager just save the info } public int StartTestRun(TestRunCriteria testRunCriteria, ITestRunEventsHandler eventHandler) @@ -173,6 +170,7 @@ public bool HandlePartialRunComplete( ICollection executorUris) { var allRunsCompleted = false; + // REVIEW: Interlocked.Increment _runCompletedClients, and the condition on the bottom probably does not need to be under lock? lock (_executionStatusLockObject) { // Each concurrent Executor calls this method @@ -203,11 +201,43 @@ public bool HandlePartialRunComplete( return true; } - + var SharedHosts = false; EqtTrace.Verbose("ParallelProxyExecutionManager: HandlePartialRunComplete: Replace execution manager. Shared: {0}, Aborted: {1}.", SharedHosts, testRunCompleteArgs.IsAborted); - + // TODO: this should not always happen, this should happen only if we are going to the next non-shared host RemoveManager(proxyExecutionManager); - proxyExecutionManager = CreateNewConcurrentManager(); + + TestRunCriteria testRunCriteria = null; + string source = null; + if (!_hasSpecificTestsRun) + { + if (TryFetchNextSource(_sourceEnumerator, out source)) + { + EqtTrace.Info("ProxyParallelExecutionManager: Triggering test run for next source: {0}", source); + testRunCriteria = new TestRunCriteria(new[] { source }, _actualTestRunCriteria); + } + } + else + { + if (TryFetchNextSource(_testCaseListEnumerator, out List nextSetOfTests)) + { + // TODO: How could this not have a value? None of the code I found should actually fail when this is null. + // We will just mix results from multiple sources. But I doubt we would get here if we had a testcase without path. + source = nextSetOfTests?.FirstOrDefault()?.Source; + EqtTrace.Info("ProxyParallelExecutionManager: Triggering test run for next source: {0}", source); + testRunCriteria = new TestRunCriteria(nextSetOfTests, _actualTestRunCriteria); + } + } + + if (testRunCriteria == null) + { + // We most likely have nothing to run, but let's still return false. This way we let the logic on the top + // of this method to decide if we are done with the run or not. + return false; + } + + // We have something more to run. + var sourceDetail = _sourceToSourceDetailMap[source]; + proxyExecutionManager = CreateNewConcurrentManager(sourceDetail); var parallelEventsHandler = GetEventsHandler(proxyExecutionManager); AddManager(proxyExecutionManager, parallelEventsHandler); @@ -215,7 +245,7 @@ public bool HandlePartialRunComplete( // and queue another test run if (!testRunCompleteArgs.IsCanceled && !_abortRequested) { - StartTestRunOnConcurrentManager(proxyExecutionManager); + StartTestRunOnConcurrentManager(proxyExecutionManager, testRunCriteria); } return false; @@ -233,11 +263,76 @@ private int StartTestRunPrivate(ITestRunEventsHandler runEventsHandler) // One data aggregator per parallel run _currentRunDataAggregator = new ParallelRunDataAggregator(_actualTestRunCriteria.TestRunSettings); - foreach (var concurrentManager in GetConcurrentManagerInstances()) + // REVIEW: Create as many handlers as we can, until we reach the parallel level or the number of sources.Originally this was done in the UpdateParallelLevel in base constructor, but we did not know which source will be the next there. + var parallel = 0; + while (parallel <= MaxParallelLevel) { + parallel++; + TestRunCriteria testRunCriteria = null; + string source = null; + if (!_hasSpecificTestsRun) + { + if (TryFetchNextSource(_sourceEnumerator, out source)) + { + EqtTrace.Info("ProxyParallelExecutionManager: Triggering test run for next source: {0}", source); + var sourceDetail2 = _sourceToSourceDetailMap[source]; + var runSettingsXml = SourceDetailHelper.UpdateRunSettingsFromSourceDetail(_actualTestRunCriteria.TestRunSettings, sourceDetail2); + + // Copy the test run criteria, but use a single source, and updated runsettings. + testRunCriteria = new TestRunCriteria(new[] { source }, // <- + _actualTestRunCriteria.FrequencyOfRunStatsChangeEvent, + _actualTestRunCriteria.KeepAlive, + runSettingsXml, // <- + _actualTestRunCriteria.RunStatsChangeEventTimeout, + _actualTestRunCriteria.TestHostLauncher, + _actualTestRunCriteria.TestCaseFilter, + _actualTestRunCriteria.FilterOptions, + _actualTestRunCriteria.TestSessionInfo, + _actualTestRunCriteria.DebugEnabledForTestSession); + } + } + else + { + if (TryFetchNextSource(_testCaseListEnumerator, out List nextSetOfTests)) + { + source = nextSetOfTests?.FirstOrDefault()?.Source; + + var sourceDetail2 = _sourceToSourceDetailMap[source]; + var runSettingsXml = SourceDetailHelper.UpdateRunSettingsFromSourceDetail(_actualTestRunCriteria.TestRunSettings, sourceDetail2); + + EqtTrace.Info("ProxyParallelExecutionManager: Triggering test run for next source: {0}", source); + // Copy the test run criteria, but use the next set of tests, and updated runsettings. + testRunCriteria = new TestRunCriteria(nextSetOfTests, // <- + _actualTestRunCriteria.FrequencyOfRunStatsChangeEvent, + _actualTestRunCriteria.KeepAlive, + runSettingsXml, // <- + _actualTestRunCriteria.RunStatsChangeEventTimeout, + _actualTestRunCriteria.TestHostLauncher, + _actualTestRunCriteria.TestSessionInfo, + _actualTestRunCriteria.DebugEnabledForTestSession); + } + } + + if (source == null) + { + // Why 1? Because this is supposed to be a processId, and that is just the default that was chosen, + // and maybe is checked somewhere, but I don't see it checked in our codebase. + return 1; + } + + var sourceDetail = _sourceToSourceDetailMap[source]; + var concurrentManager = CreateNewConcurrentManager(sourceDetail); + + var parallelEventsHandler = GetEventsHandler(concurrentManager); UpdateHandlerForManager(concurrentManager, parallelEventsHandler); - StartTestRunOnConcurrentManager(concurrentManager); + if (!concurrentManager.IsInitialized) + { + concurrentManager.Initialize(_skipDefaultAdapters); + } + IsInitialized = true; + + StartTestRunOnConcurrentManager(concurrentManager, testRunCriteria); } return 1; @@ -274,26 +369,8 @@ private ParallelRunEventsHandler GetEventsHandler(IProxyExecutionManager concurr /// /// Proxy execution manager instance. /// True, if execution triggered - private void StartTestRunOnConcurrentManager(IProxyExecutionManager proxyExecutionManager) + private void StartTestRunOnConcurrentManager(IProxyExecutionManager proxyExecutionManager, TestRunCriteria testRunCriteria) { - TestRunCriteria testRunCriteria = null; - if (!_hasSpecificTestsRun) - { - if (TryFetchNextSource(_sourceEnumerator, out string nextSource)) - { - EqtTrace.Info("ProxyParallelExecutionManager: Triggering test run for next source: {0}", nextSource); - testRunCriteria = new TestRunCriteria(new[] { nextSource }, _actualTestRunCriteria); - } - } - else - { - if (TryFetchNextSource(_testCaseListEnumerator, out List nextSetOfTests)) - { - EqtTrace.Info("ProxyParallelExecutionManager: Triggering test run for next source: {0}", nextSetOfTests?.FirstOrDefault()?.Source); - testRunCriteria = new TestRunCriteria(nextSetOfTests, _actualTestRunCriteria); - } - } - if (testRunCriteria != null) { if (!proxyExecutionManager.IsInitialized) diff --git a/src/Microsoft.TestPlatform.CrossPlatEngine/Client/ProxyDiscoveryManager.cs b/src/Microsoft.TestPlatform.CrossPlatEngine/Client/ProxyDiscoveryManager.cs index a328d53dfb..a4503cdf20 100644 --- a/src/Microsoft.TestPlatform.CrossPlatEngine/Client/ProxyDiscoveryManager.cs +++ b/src/Microsoft.TestPlatform.CrossPlatEngine/Client/ProxyDiscoveryManager.cs @@ -131,6 +131,7 @@ public void DiscoverTests(DiscoveryCriteria discoveryCriteria, ITestDiscoveryEve if (_proxyOperationManager == null) { _proxyOperationManager = _proxyOperationManagerCreator( + // REVIEW: why only first? discoveryCriteria.Sources.First(), this); diff --git a/src/Microsoft.TestPlatform.CrossPlatEngine/TestEngine.cs b/src/Microsoft.TestPlatform.CrossPlatEngine/TestEngine.cs index d8d5373ef8..26d4eaa960 100644 --- a/src/Microsoft.TestPlatform.CrossPlatEngine/TestEngine.cs +++ b/src/Microsoft.TestPlatform.CrossPlatEngine/TestEngine.cs @@ -28,6 +28,8 @@ namespace Microsoft.VisualStudio.TestPlatform.CrossPlatEngine; using PlatformAbstractions; using PlatformAbstractions.Interfaces; using Microsoft.VisualStudio.TestPlatform.Utilities; +using System.IO; +using System.Xml; /// /// Cross platform test engine entry point for the client. @@ -44,7 +46,7 @@ public class TestEngine : ITestEngine protected internal TestEngine( TestRuntimeProviderManager testHostProviderManager, - IProcessHelper processHelper) : this((ITestRuntimeProviderManager) testHostProviderManager, processHelper) + IProcessHelper processHelper) : this((ITestRuntimeProviderManager)testHostProviderManager, processHelper) { } @@ -61,34 +63,59 @@ internal TestEngine( /// public IProxyDiscoveryManager GetDiscoveryManager( IRequestData requestData, - ITestRuntimeProvider testHostManager, - DiscoveryCriteria discoveryCriteria) + DiscoveryCriteria discoveryCriteria, + Dictionary sourceToSourceDetailMap) { + // Parallel level determines how many processes at most we should start at the same time. We take the number from settings, and if user + // has no preference or the preference is 0 then we use the number of logical processors. Or the number of sources, whatever is lower. + // We don't know for sure if we will start that many processes as some of the sources can run in a single testhost. This is determined by + // Shared on the test runtime provider. At this point we need to know only if the parallel level is more than 1, and so if we will do parallel + // run or not. var parallelLevel = VerifyParallelSettingAndCalculateParallelLevel( discoveryCriteria.Sources.Count(), discoveryCriteria.RunSettings); + var isParallelRun = parallelLevel > 1; + // Collecting IsParallel enabled. - requestData.MetricsCollection.Add( - TelemetryDataConstants.ParallelEnabledDuringDiscovery, - parallelLevel > 1 ? "True" : "False"); - requestData.MetricsCollection.Add( - TelemetryDataConstants.TestSessionId, - discoveryCriteria.TestSessionInfo?.Id.ToString() ?? string.Empty); + requestData.MetricsCollection.Add(TelemetryDataConstants.ParallelEnabledDuringDiscovery, isParallelRun ? "True" : "False"); + requestData.MetricsCollection.Add(TelemetryDataConstants.TestSessionId, discoveryCriteria.TestSessionInfo?.Id.ToString() ?? string.Empty); - if (ShouldRunInNoIsolation(discoveryCriteria.RunSettings, parallelLevel > 1, false)) + // This is a big if that figures out if we can run in process. In process run is very restricted, it is non-parallel run + // that has the same target framework as the current process, and it also must not be running in DesignMode (server mode / under IDE) + // and more conditions. In all other cases we run in a separate testhost process. + if (ShouldRunInProcess(discoveryCriteria.RunSettings, isParallelRun, isDataCollectorEnabled: false)) { + // Get testhost manager by configuration, and either use it for in-process run. or for single source run. + // If there are more sources throw it away. We cannot avoid creating this manager because we don't know if it will be shared or not + // and we don't have any metadata for that info. + + // We are running in process, so whatever the architecture and framework that was figured out is, it must be compatible. If we have more + // changes that we want to to runsettings in the future, based on SourceDetail then it will depend on those details. But in general + // we will have to check that all source details are the same. Otherwise we for sure cannot run in process. + // E.g. if we get list of sources where one of them has different architecture we for sure cannot run in process, because the current + // process can handle only single runsettings. + var runSettingsXml = discoveryCriteria.RunSettings; + ITestRuntimeProvider testHostManager = _testHostProviderManager.GetTestHostManagerByRunConfiguration(runSettingsXml); + ThrowExceptionIfTestHostManagerIsNull(testHostManager, runSettingsXml); + testHostManager.Initialize(TestSessionMessageLogger.Instance, runSettingsXml); + var isTelemetryOptedIn = requestData.IsTelemetryOptedIn; var newRequestData = GetRequestData(isTelemetryOptedIn); - return new InProcessProxyDiscoveryManager( - testHostManager, - new TestHostManagerFactory(newRequestData)); + return new InProcessProxyDiscoveryManager(testHostManager, new TestHostManagerFactory(newRequestData)); } - Func proxyDiscoveryManagerCreator = () => + Func proxyDiscoveryManagerCreator = sourceDetail => { - var hostManager = _testHostProviderManager.GetTestHostManagerByRunConfiguration(discoveryCriteria.RunSettings); - hostManager?.Initialize(TestSessionMessageLogger.Instance, discoveryCriteria.RunSettings); + // Update runsettings to have the correct architecture and framework for the current source, + // we are (potentially) running with multiple target frameworks or multiple architectures. + // And this allows the ITestRuntimeProvider to have no idea about that, because it only needs to know about run settings for the source it is about to run. + // TODO: No idea why I did not actually patch the settings when I infer them, because then everyone along the path will know about the same runsettings. But at least here I know + // that I am making changes on the most up-to-date settings. + var runSettingsXml = SourceDetailHelper.UpdateRunSettingsFromSourceDetail(discoveryCriteria.RunSettings, sourceDetail); + + var hostManager = _testHostProviderManager.GetTestHostManagerByRunConfiguration(runSettingsXml); + hostManager?.Initialize(TestSessionMessageLogger.Instance, runSettingsXml); // This function is used to either take a pre-existing proxy operation manager from // the test pool or to create a new proxy operation manager on the spot. @@ -102,7 +129,7 @@ public IProxyDiscoveryManager GetDiscoveryManager( var proxyOperationManager = TestSessionPool.Instance.TryTakeProxy( discoveryCriteria.TestSessionInfo, source, - discoveryCriteria.RunSettings); + runSettingsXml); if (proxyOperationManager == null) { @@ -136,42 +163,50 @@ public IProxyDiscoveryManager GetDiscoveryManager( hostManager); }; - return (parallelLevel > 1 || !testHostManager.Shared) - ? new ParallelProxyDiscoveryManager( - requestData, - proxyDiscoveryManagerCreator, - parallelLevel, - sharedHosts: testHostManager.Shared) - : proxyDiscoveryManagerCreator(); + return new ParallelProxyDiscoveryManager(requestData, proxyDiscoveryManagerCreator, parallelLevel, sourceToSourceDetailMap); } /// public IProxyExecutionManager GetExecutionManager( IRequestData requestData, - ITestRuntimeProvider testHostManager, - TestRunCriteria testRunCriteria) + TestRunCriteria testRunCriteria, + Dictionary sourceToSourceDetailMap) { + // We use mulitple "different" runsettings here. We have runsettings that come with the testRunCriteria, + // and we use that to figure out the common stuff before we try to setup the run. Later we patch the settings + // from the additional details that were passed. Those should not affect the common properties that are used for setup. + // Right now the only two things that change there are the architecture and framework so we can mix them in a single run. var distinctSources = GetDistinctNumberOfSources(testRunCriteria); - var parallelLevel = VerifyParallelSettingAndCalculateParallelLevel( - distinctSources, - testRunCriteria.TestRunSettings); + var parallelLevel = VerifyParallelSettingAndCalculateParallelLevel(distinctSources, testRunCriteria.TestRunSettings); + + // See comments in GetDiscoveryManager for more info about what is happening in this method. + var isParallelRun = parallelLevel > 1; // Collecting IsParallel enabled. - requestData.MetricsCollection.Add( - TelemetryDataConstants.ParallelEnabledDuringExecution, - parallelLevel > 1 ? "True" : "False"); - requestData.MetricsCollection.Add( - TelemetryDataConstants.TestSessionId, - testRunCriteria.TestSessionInfo?.Id.ToString() ?? string.Empty); + requestData.MetricsCollection.Add(TelemetryDataConstants.ParallelEnabledDuringExecution, isParallelRun ? "True" : "False"); + requestData.MetricsCollection.Add(TelemetryDataConstants.TestSessionId, testRunCriteria.TestSessionInfo?.Id.ToString() ?? string.Empty); var isDataCollectorEnabled = XmlRunSettingsUtilities.IsDataCollectionEnabled(testRunCriteria.TestRunSettings); var isInProcDataCollectorEnabled = XmlRunSettingsUtilities.IsInProcDataCollectionEnabled(testRunCriteria.TestRunSettings); - if (ShouldRunInNoIsolation( + if (ShouldRunInProcess( testRunCriteria.TestRunSettings, - parallelLevel > 1, + isParallelRun, isDataCollectorEnabled || isInProcDataCollectorEnabled)) { + // Not updating runsettings from source detail on purpose here. We are running in process, so whatever the settings we figured out at the start. They must be compatible + // with the current process, otherwise we would not be able to run inside of the current process. + ITestRuntimeProvider testHostManager = _testHostProviderManager.GetTestHostManagerByRunConfiguration(testRunCriteria.TestRunSettings); + ThrowExceptionIfTestHostManagerIsNull(testHostManager, testRunCriteria.TestRunSettings); + + testHostManager.Initialize(TestSessionMessageLogger.Instance, testRunCriteria.TestRunSettings); + + // NOTE: The custom launcher should not be set when we have test session info available. + if (testRunCriteria.TestHostLauncher != null) + { + testHostManager.SetCustomLauncher(testRunCriteria.TestHostLauncher); + } + var isTelemetryOptedIn = requestData.IsTelemetryOptedIn; var newRequestData = GetRequestData(isTelemetryOptedIn); return new InProcessProxyExecutionManager( @@ -181,12 +216,13 @@ public IProxyExecutionManager GetExecutionManager( // SetupChannel ProxyExecutionManager with data collection if data collectors are // specified in run settings. - Func proxyExecutionManagerCreator = () => + Func proxyExecutionManagerCreator = sourceDetail => { // Create a new host manager, to be associated with individual // ProxyExecutionManager(&POM) - var hostManager = _testHostProviderManager.GetTestHostManagerByRunConfiguration(testRunCriteria.TestRunSettings); - hostManager?.Initialize(TestSessionMessageLogger.Instance, testRunCriteria.TestRunSettings); + string runSettingsXml = SourceDetailHelper.UpdateRunSettingsFromSourceDetail(testRunCriteria.TestRunSettings, sourceDetail); + var hostManager = _testHostProviderManager.GetTestHostManagerByRunConfiguration(runSettingsXml); + hostManager?.Initialize(TestSessionMessageLogger.Instance, runSettingsXml); if (testRunCriteria.TestHostLauncher != null) { @@ -207,7 +243,7 @@ public IProxyExecutionManager GetExecutionManager( var proxyOperationManager = TestSessionPool.Instance.TryTakeProxy( testRunCriteria.TestSessionInfo, source, - testRunCriteria.TestRunSettings); + runSettingsXml); if (proxyOperationManager == null) { @@ -246,7 +282,7 @@ public IProxyExecutionManager GetExecutionManager( hostManager, new ProxyDataCollectionManager( requestData, - testRunCriteria.TestRunSettings, + runSettingsXml, GetSourcesFromTestRunCriteria(testRunCriteria))) : new ProxyExecutionManager( requestData, @@ -254,16 +290,9 @@ public IProxyExecutionManager GetExecutionManager( hostManager); }; - // parallelLevel = 1 for desktop should go via else route. - var executionManager = (parallelLevel > 1 || !testHostManager.Shared) - ? new ParallelProxyExecutionManager( - requestData, - proxyExecutionManagerCreator, - parallelLevel, - sharedHosts: testHostManager.Shared) - : proxyExecutionManagerCreator(); + var executionManager = new ParallelProxyExecutionManager(requestData, proxyExecutionManagerCreator, parallelLevel, sourceToSourceDetailMap); - EqtTrace.Verbose($"TestEngine.GetExecutionManager: Chosen execution manager '{executionManager.GetType().AssemblyQualifiedName}' ParallelLevel '{parallelLevel}' Shared host '{testHostManager.Shared}'"); + EqtTrace.Verbose($"TestEngine.GetExecutionManager: Chosen execution manager '{executionManager.GetType().AssemblyQualifiedName}' ParallelLevel '{parallelLevel}'."); return executionManager; } @@ -284,7 +313,7 @@ public IProxyTestSessionManager GetTestSessionManager( var isDataCollectorEnabled = XmlRunSettingsUtilities.IsDataCollectionEnabled(testSessionCriteria.RunSettings); var isInProcDataCollectorEnabled = XmlRunSettingsUtilities.IsInProcDataCollectionEnabled(testSessionCriteria.RunSettings); - if (ShouldRunInNoIsolation( + if (ShouldRunInProcess( testSessionCriteria.RunSettings, parallelLevel > 1, isDataCollectorEnabled || isInProcDataCollectorEnabled)) @@ -399,6 +428,7 @@ private int VerifyParallelSettingAndCalculateParallelLevel( // Check the user parallel setting. int userParallelSetting = RunSettingsUtilities.GetMaxCpuCount(runSettings); parallelLevelToUse = userParallelSetting == 0 + // TODO: use environment helper so we can control this from tests. ? Environment.ProcessorCount : userParallelSetting; var enableParallel = parallelLevelToUse > 1; @@ -436,7 +466,7 @@ private int VerifyParallelSettingAndCalculateParallelLevel( return parallelLevelToUse; } - private bool ShouldRunInNoIsolation( + private bool ShouldRunInProcess( string runsettings, bool isParallelEnabled, bool isDataCollectorEnabled) @@ -514,4 +544,15 @@ private IEnumerable GetSourcesFromTestRunCriteria(TestRunCriteria testRu ? TestSourcesUtility.GetSources(testRunCriteria.Tests) : testRunCriteria.Sources; } + + private static void ThrowExceptionIfTestHostManagerIsNull( + ITestRuntimeProvider testHostManager, + string settingsXml) + { + if (testHostManager == null) + { + EqtTrace.Error($"{nameof(TestPlatform)}.{nameof(ThrowExceptionIfTestHostManagerIsNull)}: No suitable testHostProvider found for runsettings: {settingsXml}"); + throw new TestPlatformException(string.Format(CultureInfo.CurrentCulture, Resources.Resources.NoTestHostProviderFound)); + } + } } diff --git a/src/Microsoft.TestPlatform.CrossPlatEngine/Utilities/SourceDetailHelper.cs b/src/Microsoft.TestPlatform.CrossPlatEngine/Utilities/SourceDetailHelper.cs new file mode 100644 index 0000000000..586a5aa34c --- /dev/null +++ b/src/Microsoft.TestPlatform.CrossPlatEngine/Utilities/SourceDetailHelper.cs @@ -0,0 +1,29 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +namespace Microsoft.VisualStudio.TestPlatform.CrossPlatEngine.Utilities; + +using System.IO; +using System.Xml; + +using Microsoft.VisualStudio.TestPlatform.ObjectModel; +using Microsoft.VisualStudio.TestPlatform.ObjectModel.Utilities; +using Microsoft.VisualStudio.TestPlatform.Utilities; + +internal static class SourceDetailHelper +{ + internal static string UpdateRunSettingsFromSourceDetail(string runSettings, SourceDetail sourceDetail) + { + var stream = new StringReader(runSettings); + var reader = XmlReader.Create(stream, XmlRunSettingsUtilities.ReaderSettings); + var document = new XmlDocument(); + document.Load(reader); + var navigator = document.CreateNavigator(); + + InferRunSettingsHelper.UpdateTargetFramework(document, sourceDetail.Framework.ToString(), overwrite: true); + InferRunSettingsHelper.UpdateTargetPlatform(document, sourceDetail.Architecture.ToString(), overwrite: true); + + var updatedRunSettings = navigator.OuterXml; + return updatedRunSettings; + } +} diff --git a/src/Microsoft.TestPlatform.ObjectModel/Client/Interfaces/ITestPlatform.cs b/src/Microsoft.TestPlatform.ObjectModel/Client/Interfaces/ITestPlatform.cs index 37e5ee2710..f263f14c54 100644 --- a/src/Microsoft.TestPlatform.ObjectModel/Client/Interfaces/ITestPlatform.cs +++ b/src/Microsoft.TestPlatform.ObjectModel/Client/Interfaces/ITestPlatform.cs @@ -45,7 +45,8 @@ void UpdateExtensions( IDiscoveryRequest CreateDiscoveryRequest( IRequestData requestData, DiscoveryCriteria discoveryCriteria, - TestPlatformOptions options); + TestPlatformOptions options, + Dictionary sourceToSourceDetailMap); /// /// Creates a test run request. @@ -59,7 +60,8 @@ IDiscoveryRequest CreateDiscoveryRequest( ITestRunRequest CreateTestRunRequest( IRequestData requestData, TestRunCriteria testRunCriteria, - TestPlatformOptions options); + TestPlatformOptions options, + Dictionary sourceToSourceDetailMap); /// /// Starts a test session. diff --git a/src/Microsoft.TestPlatform.ObjectModel/SourceDetail.cs b/src/Microsoft.TestPlatform.ObjectModel/SourceDetail.cs new file mode 100644 index 0000000000..4270aa9518 --- /dev/null +++ b/src/Microsoft.TestPlatform.ObjectModel/SourceDetail.cs @@ -0,0 +1,13 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +#nullable disable + +namespace Microsoft.VisualStudio.TestPlatform.ObjectModel; + +public class SourceDetail +{ + public string Source { get; internal set; } + public Architecture Architecture { get; internal set; } + public Framework Framework { get; internal set; } +} diff --git a/src/Microsoft.TestPlatform.TestHostProvider/Hosting/DefaultTestHostManager.cs b/src/Microsoft.TestPlatform.TestHostProvider/Hosting/DefaultTestHostManager.cs index 58cde217b3..a88d0ac568 100644 --- a/src/Microsoft.TestPlatform.TestHostProvider/Hosting/DefaultTestHostManager.cs +++ b/src/Microsoft.TestPlatform.TestHostProvider/Hosting/DefaultTestHostManager.cs @@ -305,7 +305,8 @@ public void Initialize(IMessageLogger logger, string runsettingsXml) _targetFramework = runConfiguration.TargetFramework; _testHostProcess = null; - Shared = !runConfiguration.DisableAppDomain; + // REVIEW: this is only here to force the use of the parallel proxy manager even when the default is netfmw and single source, so I don't have to fix all the logic right now + Shared = false; // !runConfiguration.DisableAppDomain; _hostExitedEventRaised = false; } diff --git a/src/vstest.console/CommandLine/InferHelper.cs b/src/vstest.console/CommandLine/InferHelper.cs index 835c99e285..e62e7dfb4c 100644 --- a/src/vstest.console/CommandLine/InferHelper.cs +++ b/src/vstest.console/CommandLine/InferHelper.cs @@ -26,54 +26,61 @@ internal InferHelper(IAssemblyMetadataProvider assemblyMetadataProvider) /// /// Determines Architecture from sources. /// - public Architecture AutoDetectArchitecture(IList sources, IDictionary sourcePlatforms, Architecture defaultArchitecture) + public Architecture AutoDetectArchitecture(IList sources, Architecture defaultArchitecture, out IDictionary sourceToPlatformMap) { + sourceToPlatformMap = new Dictionary(); + if (sources == null || sources.Count == 0) + return defaultArchitecture; + var architecture = defaultArchitecture; + foreach (var source in sources) + { + sourceToPlatformMap.Add(source, defaultArchitecture); + } + try { - if (sources != null && sources.Count > 0) + // REVIEW: I had changes in this class in my prototype that I could not understand anymore, are there any changes needed? + Architecture? finalArch = null; + foreach (string source in sources) { - Architecture? finalArch = null; - foreach (string source in sources) + Architecture arch; + if (IsDotNetAssembly(source)) { - Architecture arch; - if (IsDotNetAssembly(source)) - { - arch = _assemblyMetadataProvider.GetArchitecture(source); - } - else - { - // Set AnyCPU for non dotnet test sources (js, py and other). Otherwise warning will - // show up if there is mismatch with user provided platform. - arch = Architecture.AnyCPU; - } - EqtTrace.Info("Determined platform for source '{0}' is '{1}'", source, arch); - sourcePlatforms[source] = arch; - - if (Architecture.AnyCPU.Equals(arch)) - { - // If arch is AnyCPU ignore it. - continue; - } + arch = _assemblyMetadataProvider.GetArchitecture(source); + } + else + { + // Set AnyCPU for non dotnet test sources (js, py and other). Otherwise warning will + // show up if there is mismatch with user provided platform. + arch = Architecture.AnyCPU; + } + EqtTrace.Info("Determined platform for source '{0}' is '{1}'", source, arch); + sourceToPlatformMap[source] = arch; - if (finalArch == null) - { - finalArch = arch; - continue; - } + if (Architecture.AnyCPU.Equals(arch)) + { + // If arch is AnyCPU ignore it. + continue; + } - if (!finalArch.Equals(arch)) - { - finalArch = defaultArchitecture; - EqtTrace.Info("Conflict in platform architecture, using default platform:{0}", finalArch); - } + if (finalArch == null) + { + finalArch = arch; + continue; } - if (finalArch != null) + if (!finalArch.Equals(arch)) { - architecture = (Architecture)finalArch; + finalArch = defaultArchitecture; + EqtTrace.Info("Conflict in platform architecture, using default platform:{0}", finalArch); } } + + if (finalArch != null) + { + architecture = (Architecture)finalArch; + } } catch (Exception ex) { @@ -88,22 +95,24 @@ public Architecture AutoDetectArchitecture(IList sources, IDictionary /// Determines Framework from sources. /// - public Framework AutoDetectFramework(IList sources, IDictionary sourceFrameworkVersions) + public Framework AutoDetectFramework(IList sources, out IDictionary sourceToFrameworkMap) { + sourceToFrameworkMap = new Dictionary(); Framework framework = Framework.DefaultFramework; + + if (sources == null || sources.Count == 0) + return framework; + try { - if (sources != null && sources.Count > 0) + var finalFx = DetermineFrameworkName(sources, out sourceToFrameworkMap, out var conflictInFxIdentifier); + framework = Framework.FromString(finalFx.FullName); + if (conflictInFxIdentifier) { - var finalFx = DetermineFrameworkName(sources, sourceFrameworkVersions, out var conflictInFxIdentifier); - framework = Framework.FromString(finalFx.FullName); - if (conflictInFxIdentifier) - { - // TODO Log to console and client. - EqtTrace.Info( - "conflicts in Framework identifier of provided sources(test assemblies), using default framework:{0}", - framework); - } + // TODO Log to console and client. + EqtTrace.Info( + "conflicts in Framework identifier of provided sources(test assemblies), using default framework:{0}", + framework); } } catch (Exception ex) @@ -116,58 +125,71 @@ public Framework AutoDetectFramework(IList sources, IDictionary sources, IDictionary sourceFrameworkVersions, out bool conflictInFxIdentifier) + private FrameworkName DetermineFrameworkName(IEnumerable sources, out IDictionary sourceToFrameworkMap, out bool conflictInFxIdentifier) { + sourceToFrameworkMap = new Dictionary(); + + var defaultFramework = Framework.DefaultFramework; FrameworkName finalFx = null; conflictInFxIdentifier = false; foreach (string source in sources) { - FrameworkName fx; - if (IsDotNetAssembly(source)) - { - fx = _assemblyMetadataProvider.GetFrameWork(source); - } - else + try { - // TODO What else to do with appx, js and other? - var extension = Path.GetExtension(source); - if (extension.Equals(".js", StringComparison.OrdinalIgnoreCase)) + FrameworkName fx; + if (IsDotNetAssembly(source)) { - // Currently to run tests for .NET Core, assembly need dependency to Microsoft.NET.Test.Sdk. Which is not - // possible for js files. So using default .NET Full framework version. - fx = new FrameworkName(Constants.DotNetFramework40); + fx = _assemblyMetadataProvider.GetFrameWork(source); } else { - fx = extension.Equals(".appx", StringComparison.OrdinalIgnoreCase) - || extension.Equals(".msix", StringComparison.OrdinalIgnoreCase) - || extension.Equals(".appxrecipe", StringComparison.OrdinalIgnoreCase) - ? new FrameworkName(Constants.DotNetFrameworkUap10) - : new FrameworkName(Framework.DefaultFramework.Name); + // TODO What else to do with appx, js and other? + var extension = Path.GetExtension(source); + if (extension.Equals(".js", StringComparison.OrdinalIgnoreCase)) + { + // Currently to run tests for .NET Core, assembly need dependency to Microsoft.NET.Test.Sdk. Which is not + // possible for js files. So using default .NET Full framework version. + fx = new FrameworkName(Constants.DotNetFramework40); + } + else + { + fx = extension.Equals(".appx", StringComparison.OrdinalIgnoreCase) + || extension.Equals(".msix", StringComparison.OrdinalIgnoreCase) + || extension.Equals(".appxrecipe", StringComparison.OrdinalIgnoreCase) + ? new FrameworkName(Constants.DotNetFrameworkUap10) + : new FrameworkName(Framework.DefaultFramework.Name); + } } - } - sourceFrameworkVersions[source] = Framework.FromString(fx.FullName); - if (finalFx == null) - { - finalFx = fx; - continue; - } + sourceToFrameworkMap.Add(source, Framework.FromString(fx.FullName)); - if (finalFx.Identifier.Equals(fx.Identifier)) - { - // Use latest version. - if (finalFx.Version < fx.Version) + if (finalFx == null) { finalFx = fx; + continue; + } + + if (finalFx.Identifier.Equals(fx.Identifier)) + { + // Use latest version. + if (finalFx.Version < fx.Version) + { + finalFx = fx; + } + } + else + { + conflictInFxIdentifier = true; + finalFx = new FrameworkName(defaultFramework.Name); } } - else + catch (Exception ex) { - conflictInFxIdentifier = true; - finalFx = new FrameworkName(Framework.DefaultFramework.Name); + sourceToFrameworkMap.Add(source, defaultFramework); + EqtTrace.Error("Failed to determine framework for source: {0} using default framework: {1}, exception: {2}", source, defaultFramework.Name, ex); } } + return finalFx; } diff --git a/src/vstest.console/TestPlatformHelpers/TestRequestManager.cs b/src/vstest.console/TestPlatformHelpers/TestRequestManager.cs index 0888ad1e9b..c9e6fe7713 100644 --- a/src/vstest.console/TestPlatformHelpers/TestRequestManager.cs +++ b/src/vstest.console/TestPlatformHelpers/TestRequestManager.cs @@ -167,6 +167,8 @@ public void DiscoverTests( { EqtTrace.Info("TestRequestManager.DiscoverTests: Discovery tests started."); + // TODO: Normalize rest of the data on the request as well + discoveryPayload.Sources = discoveryPayload.Sources?.Distinct().ToList() ?? new List(); var runsettings = discoveryPayload.RunSettings; if (discoveryPayload.TestPlatformOptions != null) @@ -177,13 +179,22 @@ public void DiscoverTests( var requestData = GetRequestData(protocolConfig); if (UpdateRunSettingsIfRequired( runsettings, - discoveryPayload.Sources?.ToList(), + discoveryPayload.Sources.ToList(), discoveryEventsRegistrar, - out string updatedRunsettings)) + out string updatedRunsettings, + out IDictionary sourceToArchitectureMap, + out IDictionary sourceToFrameworkMap)) { runsettings = updatedRunsettings; } + var sourceToSourceDetailMap = discoveryPayload.Sources.Select(source => new SourceDetail + { + Source = source, + Architecture = sourceToArchitectureMap[source], + Framework = sourceToFrameworkMap[source], + }).ToDictionary(k => k.Source); + var runConfiguration = XmlRunSettingsUtilities.GetRunConfigurationNode(runsettings); var batchSize = runConfiguration.BatchSize; var testCaseFilterFromRunsettings = runConfiguration.TestCaseFilter; @@ -220,7 +231,8 @@ public void DiscoverTests( _currentDiscoveryRequest = _testPlatform.CreateDiscoveryRequest( requestData, criteria, - discoveryPayload.TestPlatformOptions); + discoveryPayload.TestPlatformOptions, + sourceToSourceDetailMap); discoveryEventsRegistrar?.RegisterDiscoveryEvents(_currentDiscoveryRequest); // Notify start of discovery start. @@ -275,11 +287,20 @@ public void RunTests( runsettings, sources, testRunEventsRegistrar, - out string updatedRunsettings)) + out string updatedRunsettings, + out IDictionary sourceToArchitectureMap, + out IDictionary sourceToFrameworkMap)) { runsettings = updatedRunsettings; } + var sourceToSourceDetailMap = sources.Select(source => new SourceDetail + { + Source = source, + Architecture = sourceToArchitectureMap[source], + Framework = sourceToFrameworkMap[source], + }).ToDictionary(k => k.Source); + if (InferRunSettingsHelper.AreRunSettingsCollectorsIncompatibleWithTestSettings(runsettings)) { throw new SettingsException( @@ -306,6 +327,10 @@ public void RunTests( // Get Fakes data collector settings. if (!string.Equals(Environment.GetEnvironmentVariable("VSTEST_SKIP_FAKES_CONFIGURATION"), "1")) { + // TODO: Are the sources in _commandLineOptions any different from the ones we get on the request? + // because why would they be? We never pass that forward to the executor, so this probably should + // just look at sources anyway. + // The commandline options do not have sources in design time mode, // and so we fall back to using sources instead. if (_commandLineOptions.Sources.Any()) @@ -324,6 +349,11 @@ public void RunTests( } } + // We can have either a run that contains string as test container (usually a DLL), which is later resolved to the actual path + // and all tests that match filter are run from that container. + // + // OR we already did discovery and have a list of TestCases that have concrete test method information + // and so we only pass those. TestCase also has the test container path (usually a DLL). TestRunCriteria runCriteria = testRunRequestPayload.Sources != null && testRunRequestPayload.Sources.Any() ? new TestRunCriteria( testRunRequestPayload.Sources, @@ -355,7 +385,8 @@ public void RunTests( requestData, runCriteria, testRunEventsRegistrar, - testRunRequestPayload.TestPlatformOptions); + testRunRequestPayload.TestPlatformOptions, + sourceToSourceDetailMap); EqtTrace.Info("TestRequestManager.RunTests: run tests completed."); } finally @@ -438,7 +469,9 @@ public void StartTestSession( payload.RunSettings, payload.Sources, null, - out string updatedRunsettings)) + out string updatedRunsettings, + out IDictionary sourceToArchitectureMap, + out IDictionary sourceToFrameworkMap)) { payload.RunSettings = updatedRunsettings; } @@ -614,90 +647,97 @@ private bool UpdateRunSettingsIfRequired( string runsettingsXml, IList sources, IBaseTestEventsRegistrar registrar, - out string updatedRunSettingsXml) + out string updatedRunSettingsXml, + out IDictionary sourceToArchitectureMap, + out IDictionary sourceToFrameworkMap) { bool settingsUpdated = false; updatedRunSettingsXml = runsettingsXml; - var sourcePlatforms = new Dictionary(); - var sourceFrameworks = new Dictionary(); - if (!string.IsNullOrEmpty(runsettingsXml)) + // TODO: maybe check the settings on the edge and normalize the incoming requests rather than + // force validating that we even got runsettings in here. + if (string.IsNullOrWhiteSpace(runsettingsXml)) + { + sourceToArchitectureMap = new Dictionary(); + sourceToFrameworkMap = new Dictionary(); + return false; + } + + // TargetFramework is full CLR. Set DesignMode based on current context. + using var stream = new StringReader(runsettingsXml); + using var reader = XmlReader.Create( + stream, + XmlRunSettingsUtilities.ReaderSettings); + var document = new XmlDocument(); + document.Load(reader); + var navigator = document.CreateNavigator(); + var runConfiguration = XmlRunSettingsUtilities.GetRunConfigurationNode(runsettingsXml); + var loggerRunSettings = XmlRunSettingsUtilities.GetLoggerRunSettings(runsettingsXml) + ?? new LoggerRunSettings(); + + settingsUpdated |= UpdateFrameworkInRunSettingsIfRequired( + document, + navigator, + sources, + registrar, + out Framework chosenFramework, + out sourceToFrameworkMap); + + // Choose default architecture based on the framework. + // For .NET core, the default platform architecture should be based on the process. + Architecture defaultArchitecture = Architecture.X86; + if (chosenFramework.Name.IndexOf("netstandard", StringComparison.OrdinalIgnoreCase) >= 0 + || chosenFramework.Name.IndexOf("netcoreapp", StringComparison.OrdinalIgnoreCase) >= 0 + // This is a special case for 1 version of Nuget.Frameworks that was shipped with using identifier NET5 instead of NETCoreApp5 for .NET 5. + || chosenFramework.Name.IndexOf("net5", StringComparison.OrdinalIgnoreCase) >= 0) { - // TargetFramework is full CLR. Set DesignMode based on current context. - using var stream = new StringReader(runsettingsXml); - using var reader = XmlReader.Create( - stream, - XmlRunSettingsUtilities.ReaderSettings); - var document = new XmlDocument(); - document.Load(reader); - var navigator = document.CreateNavigator(); - var runConfiguration = XmlRunSettingsUtilities.GetRunConfigurationNode(runsettingsXml); - var loggerRunSettings = XmlRunSettingsUtilities.GetLoggerRunSettings(runsettingsXml) - ?? new LoggerRunSettings(); - - settingsUpdated |= UpdateFramework( - document, - navigator, - sources, - sourceFrameworks, - registrar, - out Framework chosenFramework); - - // Choose default architecture based on the framework. - // For .NET core, the default platform architecture should be based on the process. - Architecture defaultArchitecture = Architecture.X86; - if (chosenFramework.Name.IndexOf("netstandard", StringComparison.OrdinalIgnoreCase) >= 0 - || chosenFramework.Name.IndexOf("netcoreapp", StringComparison.OrdinalIgnoreCase) >= 0 - // This is a special case for 1 version of Nuget.Frameworks that was shipped with using identifier NET5 instead of NETCoreApp5 for .NET 5. - || chosenFramework.Name.IndexOf("net5", StringComparison.OrdinalIgnoreCase) >= 0) - { #if NETCOREAPP - // We are running in vstest.console that is either started via dotnet.exe - // or via vstest.console.exe .NET Core executable. For AnyCPU dlls this - // should resolve 32-bit SDK when running from 32-bit dotnet process and - // 64-bit SDK when running from 64-bit dotnet process. - // As default architecture we specify the expected test host architecture, - // it can be specified by user on the command line with --arch or through runsettings. - // If it's not specified by user will be filled by current processor architecture; - // should be the same as SDK. - defaultArchitecture = RunSettingsHelper.Instance.IsDefaultTargetArchitecture ? - TranslateToArchitecture(_processHelper.GetCurrentProcessArchitecture()) : - runConfiguration.TargetPlatform; + // We are running in vstest.console that is either started via dotnet.exe + // or via vstest.console.exe .NET Core executable. For AnyCPU dlls this + // should resolve 32-bit SDK when running from 32-bit dotnet process and + // 64-bit SDK when running from 64-bit dotnet process. + // As default architecture we specify the expected test host architecture, + // it can be specified by user on the command line with --arch or through runsettings. + // If it's not specified by user will be filled by current processor architecture; + // should be the same as SDK. + defaultArchitecture = RunSettingsHelper.Instance.IsDefaultTargetArchitecture ? + TranslateToArchitecture(_processHelper.GetCurrentProcessArchitecture()) : + runConfiguration.TargetPlatform; #else - // We are running in vstest.console.exe that was built against .NET - // Framework. This console prefers 32-bit because it needs to run as 32-bit - // to be compatible with QTAgent. It runs as 32-bit both under VS and in - // Developer console. Set the default architecture based on the OS - // architecture, to find 64-bit dotnet SDK when running AnyCPU dll on 64-bit - // system, and 32-bit SDK when running AnyCPU dll on 32-bit OS. - // We want to find 64-bit SDK because it is more likely to be installed. - defaultArchitecture = Environment.Is64BitOperatingSystem ? Architecture.X64 : Architecture.X86; + // We are running in vstest.console.exe that was built against .NET + // Framework. This console prefers 32-bit because it needs to run as 32-bit + // to be compatible with QTAgent. It runs as 32-bit both under VS and in + // Developer console. Set the default architecture based on the OS + // architecture, to find 64-bit dotnet SDK when running AnyCPU dll on 64-bit + // system, and 32-bit SDK when running AnyCPU dll on 32-bit OS. + // We want to find 64-bit SDK because it is more likely to be installed. + defaultArchitecture = Environment.Is64BitOperatingSystem ? Architecture.X64 : Architecture.X86; #endif - EqtTrace.Verbose($"TestRequestManager.UpdateRunSettingsIfRequired: Default architecture: {defaultArchitecture} IsDefaultTargetArchitecture: {RunSettingsHelper.Instance.IsDefaultTargetArchitecture}, Current process architecture: {_processHelper.GetCurrentProcessArchitecture()}."); - } - - settingsUpdated |= UpdatePlatform( - document, - navigator, - sources, - sourcePlatforms, - defaultArchitecture, - out Architecture chosenPlatform); - CheckSourcesForCompatibility( - chosenFramework, - chosenPlatform, - defaultArchitecture, - sourcePlatforms, - sourceFrameworks, - registrar); - settingsUpdated |= UpdateDesignMode(document, runConfiguration); - settingsUpdated |= UpdateCollectSourceInformation(document, runConfiguration); - settingsUpdated |= UpdateTargetDevice(navigator, document); - settingsUpdated |= AddOrUpdateConsoleLogger(document, runConfiguration, loggerRunSettings); - - updatedRunSettingsXml = navigator.OuterXml; + EqtTrace.Verbose($"TestRequestManager.UpdateRunSettingsIfRequired: Default architecture: {defaultArchitecture} IsDefaultTargetArchitecture: {RunSettingsHelper.Instance.IsDefaultTargetArchitecture}, Current process architecture: {_processHelper.GetCurrentProcessArchitecture()}."); } + settingsUpdated |= UpdatePlatform( + document, + navigator, + sources, + defaultArchitecture, + out Architecture chosenPlatform, + out sourceToArchitectureMap); + + CheckSourcesForCompatibility( + chosenFramework, + chosenPlatform, + defaultArchitecture, + sourceToArchitectureMap, + sourceToFrameworkMap, + registrar); + settingsUpdated |= UpdateDesignMode(document, runConfiguration); + settingsUpdated |= UpdateCollectSourceInformation(document, runConfiguration); + settingsUpdated |= UpdateTargetDevice(navigator, document); + settingsUpdated |= AddOrUpdateConsoleLogger(document, runConfiguration, loggerRunSettings); + + updatedRunSettingsXml = navigator.OuterXml; + return settingsUpdated; #if NETCOREAPP @@ -822,72 +862,81 @@ private bool UpdatePlatform( XmlDocument document, XPathNavigator navigator, IList sources, - IDictionary sourcePlatforms, Architecture defaultArchitecture, - out Architecture chosenPlatform) + out Architecture commonPlatform, + out IDictionary sourceToPlatformMap) { - // Get platform from sources. - var inferedPlatform = _inferHelper.AutoDetectArchitecture( - sources, - sourcePlatforms, - defaultArchitecture); - - EqtTrace.Info($"Infered platform '{inferedPlatform}'."); + // Get platform from runsettings. If runsettings specify a platform, we don't need to + // auto detect it and update it, because it is forced by run settings to be a single given platform + // for all the provided sources. + bool platformSetByRunsettings = IsPlatformSetByRunSettings(navigator, out commonPlatform); - // Get platform from runsettings. - bool updatePlatform = IsAutoPlatformDetectRequired(navigator, out chosenPlatform); - - // Update platform if required. For command line scenario update happens in - // ArgumentProcessor. - if (updatePlatform) + if (platformSetByRunsettings) { - EqtTrace.Info($"Platform update to '{inferedPlatform}' required."); - InferRunSettingsHelper.UpdateTargetPlatform( - document, - inferedPlatform.ToString(), - overwrite: true); - chosenPlatform = inferedPlatform; + EqtTrace.Info($"Platform is set by runsettings to be '{commonPlatform}' for all sources."); + sourceToPlatformMap = new Dictionary(); + foreach (var source in sources) + { + sourceToPlatformMap.Add(source, commonPlatform); + } + + // Return false, because we did not update runsettings. + return false; } - return updatePlatform; + // REVIEW: this should make the target platform empty, or "default" rather than the inferred platform, and we will then use the sources map to choose the correct testhost provider, and populate the runsettings there + // Autodetect platform from sources, and return a single common platform. + commonPlatform = _inferHelper.AutoDetectArchitecture(sources, defaultArchitecture, out sourceToPlatformMap); + InferRunSettingsHelper.UpdateTargetPlatform(document, commonPlatform.ToString(), overwrite: true); + + EqtTrace.Info($"Platform was updated to '{commonPlatform}'."); + // Return true because we updated runsettings. + return true; } - private bool UpdateFramework( + private bool UpdateFrameworkInRunSettingsIfRequired( XmlDocument document, XPathNavigator navigator, IList sources, - IDictionary sourceFrameworks, IBaseTestEventsRegistrar registrar, - out Framework chosenFramework) + out Framework commonFramework, + out IDictionary sourceToFrameworkMap) { - // Get framework from sources. - // This looks like you can optimize it by moving it down to if (updateFramework), but it has a side-effect of - // populating the sourceFrameworks, which is later checked when source compatibility check is done against the value - // that we either inferred as the common framework, or that is forced in runsettings. - var inferedFramework = _inferHelper.AutoDetectFramework(sources, sourceFrameworks); - - // See if framework is forced by runsettings. If not autodetect it. - bool updateFramework = IsAutoFrameworkDetectRequired(navigator, out chosenFramework); - - // Update framework if required. For command line scenario update happens in - // ArgumentProcessor. - if (updateFramework) + bool frameworkSetByRunsettings = IsFrameworkSetByRunSettings(navigator, out commonFramework); + + if (frameworkSetByRunsettings) { - InferRunSettingsHelper.UpdateTargetFramework( - document, - inferedFramework.ToString(), - overwrite: true); - chosenFramework = inferedFramework; + sourceToFrameworkMap = new Dictionary(); + foreach (var source in sources) + { + sourceToFrameworkMap.Add(source, commonFramework); + } + + WriteWarningForNetFramework35IsUnsupported(registrar, commonFramework); + // Return false because we did not update runsettings. + return false; } + + // REVIEW: this should make the target framework empty, or "default" rather than the inferred framework, and we will then use the sources map to choose the correct testhost provider, and populate the runsettings there + // Autodetect framework from sources, and return a single common platform. + commonFramework = _inferHelper.AutoDetectFramework(sources, out sourceToFrameworkMap); + InferRunSettingsHelper.UpdateTargetFramework(document, commonFramework.ToString(), overwrite: true); + + WriteWarningForNetFramework35IsUnsupported(registrar, commonFramework); + + return frameworkSetByRunsettings; + } + + private static void WriteWarningForNetFramework35IsUnsupported(IBaseTestEventsRegistrar registrar, Framework commonFramework) + { // Raise warnings for unsupported frameworks. - if (ObjectModel.Constants.DotNetFramework35.Equals(chosenFramework.Name)) + // TODO: Look at the sourceToFrameworkMap, and report paths to the sources that use that framework, rather than the chosen framework + if (ObjectModel.Constants.DotNetFramework35.Equals(commonFramework.Name)) { EqtTrace.Warning("TestRequestManager.UpdateRunSettingsIfRequired: throw warning on /Framework:Framework35 option."); registrar.LogWarning(Resources.Framework35NotSupported); } - - return updateFramework; } /// @@ -952,7 +1001,8 @@ private void RunTests( IRequestData requestData, TestRunCriteria testRunCriteria, ITestRunEventsRegistrar testRunEventsRegistrar, - TestPlatformOptions options) + TestPlatformOptions options, + Dictionary sourceToSourceDetailMap) { // Make sure to run the run request inside a lock as the below section is not thread-safe. // TranslationLayer can process faster as it directly gets the raw un-serialized messages @@ -966,7 +1016,8 @@ private void RunTests( _currentTestRunRequest = _testPlatform.CreateTestRunRequest( requestData, testRunCriteria, - options); + options, + sourceToSourceDetailMap); _testRunResultAggregator.RegisterTestRunEvents(_currentTestRunRequest); testRunEventsRegistrar?.RegisterTestRunEvents(_currentTestRunRequest); @@ -998,61 +1049,79 @@ private void RunTests( } } - private bool IsAutoFrameworkDetectRequired( + /// + /// Check runsettings, to see if framework was specified by the user, if yes then use that for all sources. + /// This method either looks at runsettings directly when running as a server (DesignMode / IDE / via VSTestConsoleWrapper, or how you wanna call it) + /// or uses the pre-parsed runsettings when in console mode. + /// + /// + /// + private bool IsFrameworkSetByRunSettings( XPathNavigator navigator, out Framework chosenFramework) { - bool required = true; - chosenFramework = null; if (_commandLineOptions.IsDesignMode) { - bool isValidFx = - InferRunSettingsHelper.TryGetFrameworkXml( - navigator, - out var frameworkFromrunsettingsXml); - required = !isValidFx || string.IsNullOrWhiteSpace(frameworkFromrunsettingsXml); - if (!required) + bool isValidFrameworkXml = InferRunSettingsHelper.TryGetFrameworkXml(navigator, out var frameworkXml); + var runSettingsHaveValidFramework = isValidFrameworkXml && !string.IsNullOrWhiteSpace(frameworkXml); + if (!runSettingsHaveValidFramework) { - chosenFramework = Framework.FromString(frameworkFromrunsettingsXml); + // TODO: this should just ask the runsettings to give that value so we always parse it the same way + chosenFramework = Framework.FromString(frameworkXml); + return true; } + + chosenFramework = Framework.DefaultFramework; + return false; } - else if (!_commandLineOptions.IsDesignMode - && _commandLineOptions.FrameworkVersionSpecified) + + if (_commandLineOptions.FrameworkVersionSpecified) { - required = false; chosenFramework = _commandLineOptions.TargetFrameworkVersion; + return true; } - return required; + chosenFramework = Framework.DefaultFramework; + return false; } - private bool IsAutoPlatformDetectRequired( - XPathNavigator navigator, - out Architecture chosenPlatform) + /// + /// Check runsettings, to see if platform was specified by the user, if yes then use that for all sources. + /// This method either looks at runsettings directly when running as a server (DesignMode / IDE / via VSTestConsoleWrapper, or how you wanna call it) + /// or uses the pre-parsed runsettings when in console mode. + /// + /// + /// + private bool IsPlatformSetByRunSettings( + XPathNavigator navigator, out Architecture chosenPlatform) { - bool required = true; - chosenPlatform = Architecture.Default; if (_commandLineOptions.IsDesignMode) { - bool isValidPlatform = InferRunSettingsHelper.TryGetPlatformXml( + bool isValidPlatformXml = InferRunSettingsHelper.TryGetPlatformXml( navigator, out var platformXml); - required = !isValidPlatform || string.IsNullOrWhiteSpace(platformXml); - if (!required) + bool runSettingsHaveValidPlatform = isValidPlatformXml && !string.IsNullOrWhiteSpace(platformXml); + if (runSettingsHaveValidPlatform) { - chosenPlatform = (Architecture)Enum.Parse( - typeof(Architecture), - platformXml, true); + // TODO: this should be checking if the enum has the value specified, or ideally just ask the runsettings to give that value + // so we parse the same way always + chosenPlatform = (Architecture)Enum.Parse(typeof(Architecture), platformXml, ignoreCase: true); + return true; } + + chosenPlatform = Architecture.Default; + return false; } - else if (!_commandLineOptions.IsDesignMode && _commandLineOptions.ArchitectureSpecified) + + if (_commandLineOptions.ArchitectureSpecified) { - required = false; chosenPlatform = _commandLineOptions.TargetArchitecture; + return true; } - return required; + chosenPlatform = Architecture.Default; + return false; } /// @@ -1217,6 +1286,7 @@ private IRequestData GetRequestData(ProtocolConfig protocolConfig) private List GetSources(TestRunRequestPayload testRunRequestPayload) { + // TODO: This should also use hashset to only return distinct sources. List sources = new(); if (testRunRequestPayload.Sources != null && testRunRequestPayload.Sources.Count > 0) diff --git a/test/vstest.ProgrammerTests/Fakes/FakeTestRuntimeProviderManager.cs b/test/vstest.ProgrammerTests/Fakes/FakeTestRuntimeProviderManager.cs index 9668bd3f99..e665ea3a09 100644 --- a/test/vstest.ProgrammerTests/Fakes/FakeTestRuntimeProviderManager.cs +++ b/test/vstest.ProgrammerTests/Fakes/FakeTestRuntimeProviderManager.cs @@ -22,12 +22,9 @@ public FakeTestRuntimeProviderManager(FakeErrorAggregator fakeErrorAggregator) public void AddTestRuntimeProviders(params FakeTestRuntimeProvider[] runtimeProviders) { - // This is not a bug, I am registering each provider twice because TestPlatform resolves - // them twice for every request that does not run in-process. foreach (var runtimeProvider in runtimeProviders) { TestRuntimeProviders.Enqueue(runtimeProvider); - TestRuntimeProviders.Enqueue(runtimeProvider); } } diff --git a/test/vstest.ProgrammerTests/Fakes/KnownFrameworkStrings.cs b/test/vstest.ProgrammerTests/Fakes/KnownFrameworkStrings.cs index 9605a8ff59..ffa7961349 100644 --- a/test/vstest.ProgrammerTests/Fakes/KnownFrameworkStrings.cs +++ b/test/vstest.ProgrammerTests/Fakes/KnownFrameworkStrings.cs @@ -6,7 +6,7 @@ namespace vstest.ProgrammerTests.Fakes; internal static class KnownFrameworkStrings { public static string NetCore(int major, int minor = 0) => $".NETCoreApp,Version=v{major}.{minor}"; - private static string NetFramework(int major, int minor, int patch = 0) => $".NETFramework,Version=v{major}.{minor}.{patch}"; + private static string NetFramework(int major, int minor, int patch = 0) => $".NETFramework,Version=v{major}.{minor}{(patch != 0 ? $".{patch}" : null)}"; public static string Netcoreapp1 = NetCore(1); public static string Netcoreapp2 = NetCore(2); diff --git a/test/vstest.ProgrammerTests/UnitTest1.cs b/test/vstest.ProgrammerTests/UnitTest1.cs index 0ab51d643c..15382443a9 100644 --- a/test/vstest.ProgrammerTests/UnitTest1.cs +++ b/test/vstest.ProgrammerTests/UnitTest1.cs @@ -27,7 +27,6 @@ namespace vstest.ProgrammerTests; using vstest.ProgrammerTests.Fakes; using Microsoft.VisualStudio.TestPlatform.Utilities; using Intent; -using System.Linq.Expressions; // Tests are run by Intent library that is executed from our Program.Main. To debug press F5 in VS, and maybe mark just a single test with [Only]. // To just run, press Ctrl+F5 to run without debugging. It will use short timeout for abort in case something is wrong with your test. @@ -240,7 +239,6 @@ public async Task GivenMultipleMsTestAssembliesThatUseTheSameTargetFrameworkAndA fixture.ExecutedTests.Should().HaveCount(mstest1Dll.TestCount + mstest2Dll.TestCount); } - [Only] public async Task GivenMultipleMsTestAssembliesThatHaveTheSameArchitecture_AndHaveDifferentTargetFrameworks_AndMULTI_TFM_RUNFeatureFlagIsDisabled_WhenTestsAreRun_ThenTwoTesthostsAreStartedBothForTheSameTFM() { // -- arrange @@ -373,6 +371,8 @@ public async Task GivenMultipleMsTestAssembliesThatHaveTheSameArchitecture_AndHa .ExecutionInitialize(FakeMessage.NoResponse) .StartTestExecutionWithSources(mstest1Dll.TestResultBatches) .SessionEnd(FakeMessage.NoResponse, afterAction: _ => testhost1Process.Exit()) + // We actually do get asked to terminate multiple times. + .SessionEnd(FakeMessage.NoResponse) .Build(); var testhost1 = new FakeTestHostFixtureBuilder(fixture) @@ -397,6 +397,8 @@ public async Task GivenMultipleMsTestAssembliesThatHaveTheSameArchitecture_AndHa .ExecutionInitialize(FakeMessage.NoResponse) .StartTestExecutionWithSources(mstest2Dll.TestResultBatches) .SessionEnd(FakeMessage.NoResponse, _ => testhost2Process.Exit()) + // We actually do get asked to terminate multiple times. + .SessionEnd(FakeMessage.NoResponse) .Build(); var testhost2 = new FakeTestHostFixtureBuilder(fixture) diff --git a/test/vstest.console.UnitTests/CommandLine/InferHelperTests.cs b/test/vstest.console.UnitTests/CommandLine/InferHelperTests.cs index 8e21b824a3..1c661c1a4a 100644 --- a/test/vstest.console.UnitTests/CommandLine/InferHelperTests.cs +++ b/test/vstest.console.UnitTests/CommandLine/InferHelperTests.cs @@ -40,46 +40,46 @@ public InferHelperTests() [TestMethod] public void AutoDetectArchitectureShouldReturnDefaultArchitectureOnNullSources() { - Assert.AreEqual(_defaultArchitecture, _inferHelper.AutoDetectArchitecture(null, _sourceArchitectures, _defaultArchitecture)); + Assert.AreEqual(_defaultArchitecture, _inferHelper.AutoDetectArchitecture(null, _defaultArchitecture, _sourceArchitectures)); } [TestMethod] public void AutoDetectArchitectureShouldReturnDefaultArchitectureOnEmptySources() { - Assert.AreEqual(_defaultArchitecture, _inferHelper.AutoDetectArchitecture(new List(0), _sourceArchitectures, _defaultArchitecture)); + Assert.AreEqual(_defaultArchitecture, _inferHelper.AutoDetectArchitecture(new List(0), _defaultArchitecture, _sourceArchitectures)); } [TestMethod] public void AutoDetectArchitectureShouldReturnDefaultArchitectureOnNullItemInSources() { - Assert.AreEqual(_defaultArchitecture, _inferHelper.AutoDetectArchitecture(new List() { null }, _sourceArchitectures, _defaultArchitecture)); + Assert.AreEqual(_defaultArchitecture, _inferHelper.AutoDetectArchitecture(new List() { null }, _defaultArchitecture, _sourceArchitectures)); } [TestMethod] public void AutoDetectArchitectureShouldReturnDefaultArchitectureOnWhiteSpaceItemInSources() { - Assert.AreEqual(_defaultArchitecture, _inferHelper.AutoDetectArchitecture(new List() { " " }, _sourceArchitectures, _defaultArchitecture)); + Assert.AreEqual(_defaultArchitecture, _inferHelper.AutoDetectArchitecture(new List() { " " }, _defaultArchitecture, _sourceArchitectures)); } [TestMethod] public void AutoDetectArchitectureShouldReturnCorrectArchForOneSource() { _mockAssemblyHelper.Setup(ah => ah.GetArchitecture(It.IsAny())).Returns(Architecture.X86); - Assert.AreEqual(Architecture.X86, _inferHelper.AutoDetectArchitecture(new List() { "1.dll" }, _sourceArchitectures, _defaultArchitecture)); + Assert.AreEqual(Architecture.X86, _inferHelper.AutoDetectArchitecture(new List() { "1.dll" }, _defaultArchitecture, _sourceArchitectures)); _mockAssemblyHelper.Verify(ah => ah.GetArchitecture(It.IsAny())); } [TestMethod] public void AutoDetectArchitectureShouldReturnCorrectDefaultArchForNotDotNetAssembly() { - Assert.AreEqual(_defaultArchitecture, _inferHelper.AutoDetectArchitecture(new List() { "NotDotNetAssebly.appx" }, _sourceArchitectures, _defaultArchitecture)); + Assert.AreEqual(_defaultArchitecture, _inferHelper.AutoDetectArchitecture(new List() { "NotDotNetAssebly.appx" }, _defaultArchitecture, _sourceArchitectures)); _mockAssemblyHelper.Verify(ah => ah.GetArchitecture(It.IsAny()), Times.Never); } [TestMethod] public void AutoDetectArchitectureShouldSetAnyCpuArchForNotDotNetAssembly() { - _inferHelper.AutoDetectArchitecture(new List() { "NotDotNetAssebly.appx" }, _sourceArchitectures, _defaultArchitecture); + _inferHelper.AutoDetectArchitecture(new List() { "NotDotNetAssebly.appx" }, _defaultArchitecture, _sourceArchitectures); Assert.AreEqual(Architecture.AnyCPU, _sourceArchitectures["NotDotNetAssebly.appx"]); } @@ -88,7 +88,7 @@ public void AutoDetectArchitectureShouldReturnDefaultArchForAllAnyCpuAssemblies( { _mockAssemblyHelper.SetupSequence(ah => ah.GetArchitecture(It.IsAny())) .Returns(Architecture.AnyCPU).Returns(Architecture.AnyCPU).Returns(Architecture.AnyCPU); - Assert.AreEqual(_defaultArchitecture, _inferHelper.AutoDetectArchitecture(new List() { "AnyCPU1.dll", "AnyCPU2.exe", "AnyCPU3.dll" }, _sourceArchitectures, _defaultArchitecture)); + Assert.AreEqual(_defaultArchitecture, _inferHelper.AutoDetectArchitecture(new List() { "AnyCPU1.dll", "AnyCPU2.exe", "AnyCPU3.dll" }, _defaultArchitecture, _sourceArchitectures)); _mockAssemblyHelper.Verify(ah => ah.GetArchitecture(It.IsAny()), Times.Exactly(3)); } @@ -97,7 +97,7 @@ public void AutoDetectArchitectureShouldReturnX86ArchIfOneX86AssemblyAndRestAnyC { _mockAssemblyHelper.SetupSequence(ah => ah.GetArchitecture(It.IsAny())) .Returns(Architecture.AnyCPU).Returns(Architecture.AnyCPU).Returns(Architecture.X86); - Assert.AreEqual(Architecture.X86, _inferHelper.AutoDetectArchitecture(new List() { "AnyCPU1.dll", "AnyCPU2.exe", "x86.dll" }, _sourceArchitectures, _defaultArchitecture)); + Assert.AreEqual(Architecture.X86, _inferHelper.AutoDetectArchitecture(new List() { "AnyCPU1.dll", "AnyCPU2.exe", "x86.dll" }, _defaultArchitecture, _sourceArchitectures)); _mockAssemblyHelper.Verify(ah => ah.GetArchitecture(It.IsAny()), Times.Exactly(3)); } @@ -106,7 +106,7 @@ public void AutoDetectArchitectureShouldReturnARMArchIfOneARMAssemblyAndRestAnyC { _mockAssemblyHelper.SetupSequence(ah => ah.GetArchitecture(It.IsAny())) .Returns(Architecture.ARM).Returns(Architecture.ARM).Returns(Architecture.ARM); - Assert.AreEqual(Architecture.ARM, _inferHelper.AutoDetectArchitecture(new List() { "ARM1.dll", "ARM2.dll", "ARM3.dll" }, _sourceArchitectures, _defaultArchitecture)); + Assert.AreEqual(Architecture.ARM, _inferHelper.AutoDetectArchitecture(new List() { "ARM1.dll", "ARM2.dll", "ARM3.dll" }, _defaultArchitecture, _sourceArchitectures)); _mockAssemblyHelper.Verify(ah => ah.GetArchitecture(It.IsAny()), Times.Exactly(3)); } @@ -115,7 +115,7 @@ public void AutoDetectArchitectureShouldReturnX64ArchIfOneX64AssemblyAndRestAnyC { _mockAssemblyHelper.SetupSequence(ah => ah.GetArchitecture(It.IsAny())) .Returns(Architecture.AnyCPU).Returns(Architecture.AnyCPU).Returns(Architecture.X64); - Assert.AreEqual(Architecture.X64, _inferHelper.AutoDetectArchitecture(new List() { "x64.dll", "AnyCPU2.exe", "x64.dll" }, _sourceArchitectures, _defaultArchitecture)); + Assert.AreEqual(Architecture.X64, _inferHelper.AutoDetectArchitecture(new List() { "x64.dll", "AnyCPU2.exe", "x64.dll" }, _defaultArchitecture, _sourceArchitectures)); _mockAssemblyHelper.Verify(ah => ah.GetArchitecture(It.IsAny()), Times.Exactly(3)); } @@ -124,7 +124,7 @@ public void AutoDetectArchitectureShouldReturnDefaultArchOnConflictArches() { _mockAssemblyHelper.SetupSequence(ah => ah.GetArchitecture(It.IsAny())) .Returns(Architecture.AnyCPU).Returns(Architecture.X64).Returns(Architecture.X86); - Assert.AreEqual(_defaultArchitecture, _inferHelper.AutoDetectArchitecture(new List() { "AnyCPU1.dll", "x64.exe", "x86.dll" }, _sourceArchitectures, _defaultArchitecture)); + Assert.AreEqual(_defaultArchitecture, _inferHelper.AutoDetectArchitecture(new List() { "AnyCPU1.dll", "x64.exe", "x86.dll" }, _defaultArchitecture, _sourceArchitectures)); _mockAssemblyHelper.Verify(ah => ah.GetArchitecture(It.IsAny()), Times.Exactly(3)); } @@ -134,7 +134,7 @@ public void AutoDetectArchitectureShouldPoulateSourceArchitectureDictionary() _mockAssemblyHelper.SetupSequence(ah => ah.GetArchitecture(It.IsAny())) .Returns(Architecture.AnyCPU).Returns(Architecture.X64).Returns(Architecture.X86); - Assert.AreEqual(_defaultArchitecture, _inferHelper.AutoDetectArchitecture(new List() { "AnyCPU1.dll", "x64.exe", "x86.dll" }, _sourceArchitectures, _defaultArchitecture)); + Assert.AreEqual(_defaultArchitecture, _inferHelper.AutoDetectArchitecture(new List() { "AnyCPU1.dll", "x64.exe", "x86.dll" }, _defaultArchitecture, _sourceArchitectures)); Assert.AreEqual(3, _sourceArchitectures.Count); Assert.AreEqual(Architecture.AnyCPU, _sourceArchitectures["AnyCPU1.dll"]); Assert.AreEqual(Architecture.X64, _sourceArchitectures["x64.exe"]); @@ -148,7 +148,7 @@ public void AutoDetectArchitectureShouldReturnDefaultArchIfthereIsNotDotNetAssem { _mockAssemblyHelper.SetupSequence(ah => ah.GetArchitecture(It.IsAny())) .Returns(Architecture.AnyCPU); - Assert.AreEqual(_defaultArchitecture, _inferHelper.AutoDetectArchitecture(new List() { "AnyCPU1.dll", "NotDotNetAssebly.appx" }, _sourceArchitectures, _defaultArchitecture)); + Assert.AreEqual(_defaultArchitecture, _inferHelper.AutoDetectArchitecture(new List() { "AnyCPU1.dll", "NotDotNetAssebly.appx" }, _defaultArchitecture, _sourceArchitectures)); _mockAssemblyHelper.Verify(ah => ah.GetArchitecture(It.IsAny()), Times.Exactly(1)); } From 409cca35d19d08dfcddb1f3b31d1464448d552e6 Mon Sep 17 00:00:00 2001 From: nohwnd Date: Wed, 23 Feb 2022 18:05:57 +0100 Subject: [PATCH 003/112] Discovery partially works, I don't update the testsettings in the request I send --- .../Parallel/ParallelProxyDiscoveryManager.cs | 15 ++- .../Parallel/ParallelProxyExecutionManager.cs | 4 +- .../Fakes/FakeCommunicationChannel.cs | 1 - .../Fakes/FakeDiscoveryEventsRegistrar.cs | 75 ++++++++++++ .../Fakes/FakeTestHostResponsesBuilder.cs | 45 +++++++ test/vstest.ProgrammerTests/Fakes/Fixture.cs | 2 + test/vstest.ProgrammerTests/UnitTest1.cs | 113 +++++++++++++++++- 7 files changed, 245 insertions(+), 10 deletions(-) create mode 100644 test/vstest.ProgrammerTests/Fakes/FakeDiscoveryEventsRegistrar.cs diff --git a/src/Microsoft.TestPlatform.CrossPlatEngine/Client/Parallel/ParallelProxyDiscoveryManager.cs b/src/Microsoft.TestPlatform.CrossPlatEngine/Client/Parallel/ParallelProxyDiscoveryManager.cs index 326716efe5..80459e8604 100644 --- a/src/Microsoft.TestPlatform.CrossPlatEngine/Client/Parallel/ParallelProxyDiscoveryManager.cs +++ b/src/Microsoft.TestPlatform.CrossPlatEngine/Client/Parallel/ParallelProxyDiscoveryManager.cs @@ -197,16 +197,21 @@ private void DiscoverTestsPrivate(ITestDiscoveryEventsHandler2 discoveryEventsHa // Reset the discovery complete data _discoveryCompletedClients = 0; - // REVIEW: what did I meant in the ERR comment below?? :D - // ERR: I need to schedule them until I reach maxParallelLevel or until I run out of sources. - // This won't schedule any source for discovery, because there are not concurrent managers. - foreach (var concurrentManager in GetConcurrentManagerInstances()) + + // ERRR: GetConcurrentManagerInstances() was called before that touches proxyOperationManager, not CreateNewConcurrentManager directly + // is this still compatible? I guess I am skipping the testhost pool? Or am I getting the manager from the pool via a creator? + var parallel = 0; + while (parallel <= MaxParallelLevel) { + parallel++; if (!TryFetchNextSource(_sourceEnumerator, out string source)) { - throw new InvalidOperationException("There are no more sources"); + break; } + var sourceDetail = _sourceToSourceDetailMap[source]; + var concurrentManager = CreateNewConcurrentManager(sourceDetail); + var parallelEventsHandler = new ParallelDiscoveryEventsHandler( _requestData, concurrentManager, diff --git a/src/Microsoft.TestPlatform.CrossPlatEngine/Client/Parallel/ParallelProxyExecutionManager.cs b/src/Microsoft.TestPlatform.CrossPlatEngine/Client/Parallel/ParallelProxyExecutionManager.cs index 43a8d507f4..3f6695f862 100644 --- a/src/Microsoft.TestPlatform.CrossPlatEngine/Client/Parallel/ParallelProxyExecutionManager.cs +++ b/src/Microsoft.TestPlatform.CrossPlatEngine/Client/Parallel/ParallelProxyExecutionManager.cs @@ -263,7 +263,9 @@ private int StartTestRunPrivate(ITestRunEventsHandler runEventsHandler) // One data aggregator per parallel run _currentRunDataAggregator = new ParallelRunDataAggregator(_actualTestRunCriteria.TestRunSettings); - // REVIEW: Create as many handlers as we can, until we reach the parallel level or the number of sources.Originally this was done in the UpdateParallelLevel in base constructor, but we did not know which source will be the next there. + // Create as many handlers as we can, until we reach the max parallel level (this is number of logical processors, or the number of sources, or user + // preference. Originally this was done in the UpdateParallelLevel in base constructor, but we did not know which source will be the next there. So we could not + // start the manager specifically for the source with the correct tfm and platform. var parallel = 0; while (parallel <= MaxParallelLevel) { diff --git a/test/vstest.ProgrammerTests/Fakes/FakeCommunicationChannel.cs b/test/vstest.ProgrammerTests/Fakes/FakeCommunicationChannel.cs index 3d26712557..6965bde793 100644 --- a/test/vstest.ProgrammerTests/Fakes/FakeCommunicationChannel.cs +++ b/test/vstest.ProgrammerTests/Fakes/FakeCommunicationChannel.cs @@ -182,7 +182,6 @@ private void ProcessIncomingMessages(TContext context) Debugger.Break(); } - // TODO: passing the raw message in, is strange responsePair.BeforeAction?.Invoke(context); var responses = responsePair.Responses; ProcessedMessages.Add(new RequestResponsePair(requestMessage, responses, false)); diff --git a/test/vstest.ProgrammerTests/Fakes/FakeDiscoveryEventsRegistrar.cs b/test/vstest.ProgrammerTests/Fakes/FakeDiscoveryEventsRegistrar.cs new file mode 100644 index 0000000000..63a8f82354 --- /dev/null +++ b/test/vstest.ProgrammerTests/Fakes/FakeDiscoveryEventsRegistrar.cs @@ -0,0 +1,75 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +namespace vstest.ProgrammerTests.Fakes; + +using Microsoft.VisualStudio.TestPlatform.Common.Interfaces; +using Microsoft.VisualStudio.TestPlatform.ObjectModel.Client; +using Microsoft.VisualStudio.TestPlatform.ObjectModel.Logging; + +internal class FakeTestDiscoveryEventsRegistrar : ITestDiscoveryEventsRegistrar +{ + private readonly FakeErrorAggregator _fakeErrorAggregator; + + public List AllEvents { get; } = new(); + public List LoggedWarnings { get; } = new(); + public List> DiscoveryCompleteEvents { get; } = new(); + public List> DiscoveredTestsEvents { get; } = new(); + public List> DiscoveryStartEvents { get; } = new(); + public List> DiscoveryMessageEvents { get; } = new(); + + public FakeTestDiscoveryEventsRegistrar(FakeErrorAggregator fakeErrorAggregator) + { + _fakeErrorAggregator = fakeErrorAggregator; + } + + public void LogWarning(string message) + { + AllEvents.Add(message); + LoggedWarnings.Add(message); + } + + public void RegisterDiscoveryEvents(IDiscoveryRequest discoveryRequest) + { + discoveryRequest.OnDiscoveryMessage += OnDiscoveryMessage; + discoveryRequest.OnDiscoveryStart += OnDiscoveryStart; + discoveryRequest.OnDiscoveredTests += OnDiscoveredTests; + discoveryRequest.OnDiscoveryComplete += OnDiscoveryComplete; + } + + public void UnregisterDiscoveryEvents(IDiscoveryRequest discoveryRequest) + { + discoveryRequest.OnDiscoveryMessage -= OnDiscoveryMessage; + discoveryRequest.OnDiscoveryStart -= OnDiscoveryStart; + discoveryRequest.OnDiscoveredTests -= OnDiscoveredTests; + discoveryRequest.OnDiscoveryComplete -= OnDiscoveryComplete; + } + + private void OnDiscoveryComplete(object? sender, DiscoveryCompleteEventArgs e) + { + var eventRecord = new EventRecord(sender, e); + AllEvents.Add(eventRecord); + DiscoveryCompleteEvents.Add(eventRecord); + } + + private void OnDiscoveredTests(object? sender, DiscoveredTestsEventArgs e) + { + var eventRecord = new EventRecord(sender, e); + AllEvents.Add(eventRecord); + DiscoveredTestsEvents.Add(eventRecord); + } + + private void OnDiscoveryStart(object? sender, DiscoveryStartEventArgs e) + { + var eventRecord = new EventRecord(sender, e); + AllEvents.Add(eventRecord); + DiscoveryStartEvents.Add(eventRecord); + } + + private void OnDiscoveryMessage(object? sender, TestRunMessageEventArgs e) + { + var eventRecord = new EventRecord(sender, e); + AllEvents.Add(eventRecord); + DiscoveryMessageEvents.Add(eventRecord); + } +} diff --git a/test/vstest.ProgrammerTests/Fakes/FakeTestHostResponsesBuilder.cs b/test/vstest.ProgrammerTests/Fakes/FakeTestHostResponsesBuilder.cs index 2f3302e565..f1dc61b4c8 100644 --- a/test/vstest.ProgrammerTests/Fakes/FakeTestHostResponsesBuilder.cs +++ b/test/vstest.ProgrammerTests/Fakes/FakeTestHostResponsesBuilder.cs @@ -132,4 +132,49 @@ internal List> Bui { return _responses; } + + internal FakeTestHostResponsesBuilder DiscoveryInitialize(FakeMessage fakeMessage) + { + AddPairWithFakeMessage(MessageType.DiscoveryInitialize, fakeMessage); + return this; + } + + internal FakeTestHostResponsesBuilder StartDiscovery(List> testResultBatches) + { + // Discovery returns back test cases, not test results, but it is easier to take test results, because + // we have a builder that can be re-used for both test run and test discovery. + + List messages; + if (testResultBatches.Count != 0) + { + // this will create as many test stats changes messages, as there are batches -1 + // the last batch will be sent as test run complete event + + // see TestRequestSender.OnDiscoveryMessageReceived to see how the vstest.console receives the data + List changeMessages = testResultBatches.Take(testResultBatches.Count - 1).Select(batch => + new FakeMessage>(MessageType.TestCasesFound, batch.Select(testResult => testResult.TestCase).ToList())).ToList(); + + // TODO: if we send this incorrectly the handler just continues, check logs if we can understand it from there. We should at least write a warning. + // because otherwise it hangs. + FakeMessage completedMessage = new FakeMessage(MessageType.DiscoveryComplete, new DiscoveryCompletePayload + { + LastDiscoveredTests = testResultBatches.Last().Select(testResult => testResult.TestCase).ToList(), + }); + messages = changeMessages.Concat(new[] { completedMessage }).ToList(); + } + else + { + FakeMessage completedMessage = new FakeMessage(MessageType.DiscoveryComplete, new DiscoveryCompletePayload + { + LastDiscoveredTests = new List(), + }); + + messages = completedMessage.AsList(); + } + + + AddPairWithMultipleFakeMessages(MessageType.StartDiscovery, messages); + + return this; + } } diff --git a/test/vstest.ProgrammerTests/Fakes/Fixture.cs b/test/vstest.ProgrammerTests/Fakes/Fixture.cs index 72eb4febf8..bbf6dd7d9d 100644 --- a/test/vstest.ProgrammerTests/Fakes/Fixture.cs +++ b/test/vstest.ProgrammerTests/Fakes/Fixture.cs @@ -24,6 +24,7 @@ internal class Fixture : IDisposable public FakeFileHelper FileHelper { get; } public FakeTestRuntimeProviderManager TestRuntimeProviderManager { get; } public FakeTestRunEventsRegistrar TestRunEventsRegistrar { get; } + public FakeTestDiscoveryEventsRegistrar TestDiscoveryEventsRegistrar { get; } public TestEngine? TestEngine { get; private set; } public TestPlatform? TestPlatform { get; private set; } public TestRunResultAggregator? TestRunResultAggregator { get; private set; } @@ -63,6 +64,7 @@ public Fixture(FixtureOptions? fixtureOptions = null) FileHelper = new FakeFileHelper(ErrorAggregator); TestRuntimeProviderManager = new FakeTestRuntimeProviderManager(ErrorAggregator); TestRunEventsRegistrar = new FakeTestRunEventsRegistrar(ErrorAggregator); + TestRunEventsRegistrar = new FakeTestRunEventsRegistrar(ErrorAggregator); ProtocolConfig = new ProtocolConfig(); } public void Dispose() diff --git a/test/vstest.ProgrammerTests/UnitTest1.cs b/test/vstest.ProgrammerTests/UnitTest1.cs index 15382443a9..afc4109c54 100644 --- a/test/vstest.ProgrammerTests/UnitTest1.cs +++ b/test/vstest.ProgrammerTests/UnitTest1.cs @@ -344,12 +344,12 @@ public async Task GivenMultipleMsTestAssembliesThatHaveTheSameArchitecture_AndHa fixture.ExecutedTests.Should().HaveCount(mstest1Dll.TestCount); } - [Only] public async Task GivenMultipleMsTestAssembliesThatHaveTheSameArchitecture_AndHaveDifferentTargetFrameworks_AndMULTI_TFM_RUNFeatureFlagIsEnabled_WhenTestsAreRun_ThenTwoTesthostsWithTheCorrectTFMForTheDllAreUsed() { // -- arrange using var fixture = new Fixture( - new FixtureOptions { + new FixtureOptions + { FeatureFlags = new Dictionary { [FeatureFlag.MULTI_TFM_RUN] = true @@ -420,7 +420,6 @@ public async Task GivenMultipleMsTestAssembliesThatHaveTheSameArchitecture_AndHa var testRunRequestPayload = new TestRunRequestPayload { Sources = new List { mstest1Dll.Path, mstest2Dll.Path }, - RunSettings = $"{runConfiguration}" }; @@ -450,6 +449,114 @@ public async Task GivenMultipleMsTestAssembliesThatHaveTheSameArchitecture_AndHa fixture.ExecutedTests.Should().HaveCount(mstest1Dll.TestCount + mstest2Dll.TestCount); } + + [Only] + public async Task GivenMultipleMsTestAssembliesThatHaveTheSameArchitecture_AndHaveDifferentTargetFrameworks_AndMULTI_TFM_RUNFeatureFlagIsEnabled_WhenTestsAreDiscovered_ThenTwoTesthostsWithTheCorrectTFMForTheDllAreUsed() + { + EqtTrace.InitializeVerboseTrace(@"C:\temp\log.txt"); + // -- arrange + using var fixture = new Fixture( + new FixtureOptions + { + FeatureFlags = new Dictionary + { + [FeatureFlag.MULTI_TFM_RUN] = true + } + } + ); + + var mstest1Dll = new FakeTestDllBuilder() + .WithPath(@"X:\fake\mstest1.dll") + .WithFramework(KnownFrameworkNames.Net5) // <--- + .WithArchitecture(Architecture.X64) + .WithTestCount(11, 5) + .Build(); + + var testhost1Process = new FakeProcess(fixture.ErrorAggregator, @"X:\fake\testhost1.exe"); + + var runTests1 = new FakeTestHostResponsesBuilder() + .VersionCheck(5) + .DiscoveryInitialize(FakeMessage.NoResponse) + .StartDiscovery(mstest1Dll.TestResultBatches) + .SessionEnd(FakeMessage.NoResponse, afterAction: _ => testhost1Process.Exit()) + // We actually do get asked to terminate multiple times. + .SessionEnd(FakeMessage.NoResponse) + .Build(); + + var testhost1 = new FakeTestHostFixtureBuilder(fixture) + .WithTestDll(mstest1Dll) + .WithProcess(testhost1Process) + .WithResponses(runTests1) + .Build(); + + // -- + + var mstest2Dll = new FakeTestDllBuilder() + .WithPath(@"X:\fake\mstest2.dll") + .WithFramework(KnownFrameworkNames.Net48) // <--- + .WithArchitecture(Architecture.X64) + .WithTestCount(21, 5) + .Build(); + + var testhost2Process = new FakeProcess(fixture.ErrorAggregator, @"X:\fake\testhost2.exe"); + + var runTests2 = new FakeTestHostResponsesBuilder() + .VersionCheck(5) + .DiscoveryInitialize(FakeMessage.NoResponse) + .StartDiscovery(mstest2Dll.TestResultBatches) + .SessionEnd(FakeMessage.NoResponse, _ => testhost2Process.Exit()) + // We actually do get asked to terminate multiple times. + .SessionEnd(FakeMessage.NoResponse) + .Build(); + + var testhost2 = new FakeTestHostFixtureBuilder(fixture) + .WithTestDll(mstest2Dll) + .WithProcess(testhost2Process) + .WithResponses(runTests2) + .Build(); + + fixture.AddTestHostFixtures(testhost1, testhost2); + + var testRequestManager = fixture.BuildTestRequestManager(); + + mstest1Dll.FrameworkName.Should().NotBe(mstest2Dll.FrameworkName); + + // -- act + // TODO: Building whole default runconfiguration is needed here, because TestRequestManager does not ensure the basic settings are populated, + // and all methods that populate them just silently fail, so TestHostProvider does not get any useful settings. + var runConfiguration = new RunConfiguration().ToXml().OuterXml; + var testDiscoveryPayload = new DiscoveryRequestPayload + { + Sources = new List { mstest1Dll.Path, mstest2Dll.Path }, + RunSettings = $"{runConfiguration}" + }; + + await testRequestManager.ExecuteWithAbort(tm => tm.DiscoverTests(testDiscoveryPayload, fixture.TestDiscoveryEventsRegistrar, fixture.ProtocolConfig)); + + // -- assert + fixture.AssertNoErrors(); + // We figure out the framework for each assembly so there should be no incompatibility warnings + // REVIEW: This should be uncommented. Commenting it now, because it is helpful to see those warnings. + // fixture.TestRunEventsRegistrar.LoggedWarnings.Should().NotContainMatch("Test run detected DLL(s) which were built for different framework*"); + + // We started both testhosts, even thought we know one of them is incompatible. + fixture.ProcessHelper.Processes.Where(p => p.Started).Should().HaveCount(2); + var startWithSources1 = testhost1.FakeCommunicationChannel.ProcessedMessages.Single(m => m.Request.MessageType == MessageType.StartDiscovery); + var startWithSources1Text = startWithSources1.Request.Payload.Select(t => t.ToString()).JoinBySpace(); + // We sent mstest1.dll. + startWithSources1Text.Should().Contain("mstest1.dll"); + // And we sent net5 as the target framework, because that is the framework of mstest1.dll. + startWithSources1Text.Should().Contain(KnownFrameworkStrings.Net5); + + var startWithSources2 = testhost2.FakeCommunicationChannel.ProcessedMessages.Single(m => m.Request.MessageType == MessageType.StartDiscovery); + var startWithSources2Text = startWithSources2.Request.Payload.Select(t => t.ToString()).JoinBySpace(); + // We sent mstest2.dll. + startWithSources2Text.Should().Contain("mstest2.dll"); + // And we sent net48 as the target framework, because that is the framework of mstest2.dll. + startWithSources2Text.Should().Contain(mstest2Dll.FrameworkName.ToString()); + + fixture.ExecutedTests.Should().HaveCount(mstest1Dll.TestCount + mstest2Dll.TestCount); + } } // Test and improvmement ideas: From 8e5a28918e1685e2b598fcdd85ea26face9c5d8f Mon Sep 17 00:00:00 2001 From: nohwnd Date: Thu, 24 Feb 2022 15:54:13 +0100 Subject: [PATCH 004/112] Add tests for run and discovery with different TFMs. --- .../Parallel/ParallelProxyDiscoveryManager.cs | 11 +- .../Client/ProxyDiscoveryManager.cs | 2 +- .../TestEngine.cs | 2 - .../InferRunSettingsHelper.cs | 13 +- src/vstest.console/CommandLine/InferHelper.cs | 102 ++-- .../TestPlatformHelpers/TestRequestManager.cs | 17 +- test/Intent.Primitives/ExcludeAttribute.cs | 8 +- test/Intent.Primitives/IRunLogger.cs | 7 +- test/Intent.Primitives/OnlyAttribute.cs | 2 +- test/Intent.Primitives/TestAttribute.cs | 15 + test/Intent/ConsoleLogger.cs | 56 ++- test/Intent/Runner.cs | 100 ++-- .../BasicRunAndDiscovery.cs | 265 +++++++++++ .../Fakes/FakeCommunicationChannel.cs | 8 +- .../Fakes/FakeTestHost.cs | 7 +- test/vstest.ProgrammerTests/Fakes/Fixture.cs | 30 +- .../Fakes/TestRequestManagerHelper.cs | 21 +- ...nitTest1.cs => MultiTFMRunAndDiscovery.cs} | 441 +++++++++--------- 18 files changed, 751 insertions(+), 356 deletions(-) create mode 100644 test/Intent.Primitives/TestAttribute.cs create mode 100644 test/vstest.ProgrammerTests/BasicRunAndDiscovery.cs rename test/vstest.ProgrammerTests/{UnitTest1.cs => MultiTFMRunAndDiscovery.cs} (59%) diff --git a/src/Microsoft.TestPlatform.CrossPlatEngine/Client/Parallel/ParallelProxyDiscoveryManager.cs b/src/Microsoft.TestPlatform.CrossPlatEngine/Client/Parallel/ParallelProxyDiscoveryManager.cs index 80459e8604..0d4198f334 100644 --- a/src/Microsoft.TestPlatform.CrossPlatEngine/Client/Parallel/ParallelProxyDiscoveryManager.cs +++ b/src/Microsoft.TestPlatform.CrossPlatEngine/Client/Parallel/ParallelProxyDiscoveryManager.cs @@ -17,6 +17,7 @@ namespace Microsoft.VisualStudio.TestPlatform.CrossPlatEngine.Client.Parallel; using Microsoft.VisualStudio.TestPlatform.ObjectModel.Client; using ObjectModel.Engine; using ObjectModel.Logging; +using Microsoft.VisualStudio.TestPlatform.CrossPlatEngine.Utilities; /// /// ParallelProxyDiscoveryManager that manages parallel discovery @@ -239,14 +240,20 @@ private void DiscoverTestsOnConcurrentManager(string source, IProxyDiscoveryMana EqtTrace.Verbose("ProxyParallelDiscoveryManager: Triggering test discovery for next source: {0}", source); - // Kick off another discovery task for the next source + var sourceDetail = _sourceToSourceDetailMap[source]; + var runsettingsXml = SourceDetailHelper.UpdateRunSettingsFromSourceDetail(_actualDiscoveryCriteria.RunSettings, sourceDetail); + + var discoveryCriteria = new DiscoveryCriteria( new[] { source }, _actualDiscoveryCriteria.FrequencyOfDiscoveredTestsEvent, _actualDiscoveryCriteria.DiscoveredTestEventTimeout, - _actualDiscoveryCriteria.RunSettings + runsettingsXml ); + discoveryCriteria.TestCaseFilter = _actualDiscoveryCriteria.TestCaseFilter; + + // Kick off another discovery task for the next source Task.Run(() => { EqtTrace.Verbose("ParallelProxyDiscoveryManager: Discovery started."); diff --git a/src/Microsoft.TestPlatform.CrossPlatEngine/Client/ProxyDiscoveryManager.cs b/src/Microsoft.TestPlatform.CrossPlatEngine/Client/ProxyDiscoveryManager.cs index a4503cdf20..02d0e01b43 100644 --- a/src/Microsoft.TestPlatform.CrossPlatEngine/Client/ProxyDiscoveryManager.cs +++ b/src/Microsoft.TestPlatform.CrossPlatEngine/Client/ProxyDiscoveryManager.cs @@ -131,7 +131,7 @@ public void DiscoverTests(DiscoveryCriteria discoveryCriteria, ITestDiscoveryEve if (_proxyOperationManager == null) { _proxyOperationManager = _proxyOperationManagerCreator( - // REVIEW: why only first? + // Passing only first because that is how the testhost pool is keyed. discoveryCriteria.Sources.First(), this); diff --git a/src/Microsoft.TestPlatform.CrossPlatEngine/TestEngine.cs b/src/Microsoft.TestPlatform.CrossPlatEngine/TestEngine.cs index 26d4eaa960..83b1e83fb5 100644 --- a/src/Microsoft.TestPlatform.CrossPlatEngine/TestEngine.cs +++ b/src/Microsoft.TestPlatform.CrossPlatEngine/TestEngine.cs @@ -28,8 +28,6 @@ namespace Microsoft.VisualStudio.TestPlatform.CrossPlatEngine; using PlatformAbstractions; using PlatformAbstractions.Interfaces; using Microsoft.VisualStudio.TestPlatform.Utilities; -using System.IO; -using System.Xml; /// /// Cross platform test engine entry point for the client. diff --git a/src/Microsoft.TestPlatform.Utilities/InferRunSettingsHelper.cs b/src/Microsoft.TestPlatform.Utilities/InferRunSettingsHelper.cs index 4e7e37a311..6e4ad33959 100644 --- a/src/Microsoft.TestPlatform.Utilities/InferRunSettingsHelper.cs +++ b/src/Microsoft.TestPlatform.Utilities/InferRunSettingsHelper.cs @@ -499,9 +499,16 @@ private static void AddNodeIfNotPresent(XmlDocument xmlDocument, string nodeP if (root.SelectSingleNode(RunConfigurationNodePath) == null) { - // TODO: When runsettings are incomplete this will silently return, when we run just TestRequestManager we don't get full settings. - EqtTrace.Error("InferRunSettingsHelper.UpdateNodeIfNotPresent: Unable to navigate to RunConfiguration. Current node: " + xmlDocument.LocalName); - return; + if (root.Name == RunSettingsNodeName) + { + // When just is provided to as runsettings, then this will add the common inner node. + XmlUtilities.AppendOrModifyChild(xmlDocument, RunConfigurationNodePath, RunConfigurationNodeName, innerXml: null); + } + else + { + EqtTrace.Error("InferRunSettingsHelper.UpdateNodeIfNotPresent: Unable to navigate to RunConfiguration. Current node: " + xmlDocument.LocalName); + return; + } } var node = xmlDocument.SelectSingleNode(nodePath); diff --git a/src/vstest.console/CommandLine/InferHelper.cs b/src/vstest.console/CommandLine/InferHelper.cs index e62e7dfb4c..493c0b6295 100644 --- a/src/vstest.console/CommandLine/InferHelper.cs +++ b/src/vstest.console/CommandLine/InferHelper.cs @@ -32,7 +32,7 @@ public Architecture AutoDetectArchitecture(IList sources, Architecture d if (sources == null || sources.Count == 0) return defaultArchitecture; - var architecture = defaultArchitecture; + // Set the default for all sources. foreach (var source in sources) { sourceToPlatformMap.Add(source, defaultArchitecture); @@ -40,56 +40,84 @@ public Architecture AutoDetectArchitecture(IList sources, Architecture d try { - // REVIEW: I had changes in this class in my prototype that I could not understand anymore, are there any changes needed? - Architecture? finalArch = null; + Architecture? commonArchitecture = null; foreach (string source in sources) { - Architecture arch; - if (IsDotNetAssembly(source)) + try { - arch = _assemblyMetadataProvider.GetArchitecture(source); - } - else - { - // Set AnyCPU for non dotnet test sources (js, py and other). Otherwise warning will - // show up if there is mismatch with user provided platform. - arch = Architecture.AnyCPU; - } - EqtTrace.Info("Determined platform for source '{0}' is '{1}'", source, arch); - sourceToPlatformMap[source] = arch; + Architecture detectedArchitecture; + if (IsDllOrExe(source)) + { + detectedArchitecture = _assemblyMetadataProvider.GetArchitecture(source); + + if (detectedArchitecture == Architecture.AnyCPU) + { + // This is AnyCPU .NET assembly, this source should run using the default architecture, + // which we've already set for the source. + EqtTrace.Info("Determined platform for source '{0}' was AnyCPU and it will use the default plaform {1}.", source, defaultArchitecture); + } + else + { + sourceToPlatformMap[source] = detectedArchitecture; + EqtTrace.Info("Determined platform for source '{0}' was '{1}'.", source, detectedArchitecture); + } + } + else + { + // This is non-dll source, this source should run using the default architecture, + // which we've already set for the source. + EqtTrace.Info("No platform was determined for source '{0}' because it is not a dll or an executable.", source); - if (Architecture.AnyCPU.Equals(arch)) - { - // If arch is AnyCPU ignore it. - continue; - } + // This source has no associated architecture so it does not help use determine a common architecture for + // all the sources, so we continue to next one. + continue; + } - if (finalArch == null) - { - finalArch = arch; - continue; - } + if (Architecture.AnyCPU.Equals(detectedArchitecture)) + { + // The architecture of the source is AnyCPU and so we can skip to the next one, + // because it does not help use determine a common architecture for all the sources. + continue; + } - if (!finalArch.Equals(arch)) + // This is the first source that provided some architecture use that as a candidate + // for the common architecture. + if (commonArchitecture == null) + { + commonArchitecture = detectedArchitecture; + continue; + } + + // The detected architecture, is different than the common architecture. So at least + // one of the sources is incompatible with the others. Use the default architecture as the common + // fallback. + if (!commonArchitecture.Equals(detectedArchitecture)) + { + commonArchitecture = defaultArchitecture; + } + } + catch (Exception ex) { - finalArch = defaultArchitecture; - EqtTrace.Info("Conflict in platform architecture, using default platform:{0}", finalArch); + EqtTrace.Error("Failed to determine platform for source: {0}, using default: {1}, exception: {2}", source, defaultArchitecture, ex); + sourceToPlatformMap[source] = defaultArchitecture; } } - if (finalArch != null) + if (commonArchitecture != null) { - architecture = (Architecture)finalArch; + EqtTrace.Info("Determined platform for all sources: {0}", commonArchitecture); + return commonArchitecture.Value; } + + EqtTrace.Info("None of the sources provided any runnable platform, using the default platform: {0}", defaultArchitecture); + + return defaultArchitecture; } catch (Exception ex) { - EqtTrace.Error("Failed to determine platform: {0}, using default: {1}", ex, architecture); + EqtTrace.Error("Failed to determine platform for all sources: {0}, using default: {1}", ex, defaultArchitecture); + return defaultArchitecture; } - - EqtTrace.Info("Determined platform for all sources: {0}", architecture); - - return architecture; } /// @@ -137,7 +165,7 @@ private FrameworkName DetermineFrameworkName(IEnumerable sources, out ID try { FrameworkName fx; - if (IsDotNetAssembly(source)) + if (IsDllOrExe(source)) { fx = _assemblyMetadataProvider.GetFrameWork(source); } @@ -193,7 +221,7 @@ private FrameworkName DetermineFrameworkName(IEnumerable sources, out ID return finalFx; } - private bool IsDotNetAssembly(string filePath) + private bool IsDllOrExe(string filePath) { var extType = Path.GetExtension(filePath); return extType != null && (extType.Equals(".dll", StringComparison.OrdinalIgnoreCase) || diff --git a/src/vstest.console/TestPlatformHelpers/TestRequestManager.cs b/src/vstest.console/TestPlatformHelpers/TestRequestManager.cs index c9e6fe7713..f5957a3147 100644 --- a/src/vstest.console/TestPlatformHelpers/TestRequestManager.cs +++ b/src/vstest.console/TestPlatformHelpers/TestRequestManager.cs @@ -644,7 +644,7 @@ private void Dispose(bool disposing) } private bool UpdateRunSettingsIfRequired( - string runsettingsXml, + string runsettingsXml!!, IList sources, IBaseTestEventsRegistrar registrar, out string updatedRunSettingsXml, @@ -654,15 +654,6 @@ private bool UpdateRunSettingsIfRequired( bool settingsUpdated = false; updatedRunSettingsXml = runsettingsXml; - // TODO: maybe check the settings on the edge and normalize the incoming requests rather than - // force validating that we even got runsettings in here. - if (string.IsNullOrWhiteSpace(runsettingsXml)) - { - sourceToArchitectureMap = new Dictionary(); - sourceToFrameworkMap = new Dictionary(); - return false; - } - // TargetFramework is full CLR. Set DesignMode based on current context. using var stream = new StringReader(runsettingsXml); using var reader = XmlReader.Create( @@ -925,7 +916,8 @@ private bool UpdateFrameworkInRunSettingsIfRequired( WriteWarningForNetFramework35IsUnsupported(registrar, commonFramework); - return frameworkSetByRunsettings; + // Return true because we updated runsettings. + return true; } private static void WriteWarningForNetFramework35IsUnsupported(IBaseTestEventsRegistrar registrar, Framework commonFramework) @@ -1060,11 +1052,12 @@ private bool IsFrameworkSetByRunSettings( XPathNavigator navigator, out Framework chosenFramework) { + if (_commandLineOptions.IsDesignMode) { bool isValidFrameworkXml = InferRunSettingsHelper.TryGetFrameworkXml(navigator, out var frameworkXml); var runSettingsHaveValidFramework = isValidFrameworkXml && !string.IsNullOrWhiteSpace(frameworkXml); - if (!runSettingsHaveValidFramework) + if (runSettingsHaveValidFramework) { // TODO: this should just ask the runsettings to give that value so we always parse it the same way chosenFramework = Framework.FromString(frameworkXml); diff --git a/test/Intent.Primitives/ExcludeAttribute.cs b/test/Intent.Primitives/ExcludeAttribute.cs index 4fe517e902..8c538f2e9d 100644 --- a/test/Intent.Primitives/ExcludeAttribute.cs +++ b/test/Intent.Primitives/ExcludeAttribute.cs @@ -3,7 +3,13 @@ namespace Intent; -[AttributeUsage(AttributeTargets.Assembly | AttributeTargets.Class | AttributeTargets.Constructor | AttributeTargets.Method)] +[AttributeUsage(AttributeTargets.Assembly | AttributeTargets.Class | AttributeTargets.Method)] public class ExcludeAttribute : Attribute { + public ExcludeAttribute(string? reason = null) + { + Reason = reason; + } + + public string? Reason { get; } } diff --git a/test/Intent.Primitives/IRunLogger.cs b/test/Intent.Primitives/IRunLogger.cs index 474369e78c..e4f2681195 100644 --- a/test/Intent.Primitives/IRunLogger.cs +++ b/test/Intent.Primitives/IRunLogger.cs @@ -3,12 +3,13 @@ namespace Intent; +using System.Collections.Generic; using System.Reflection; public interface IRunLogger { - void WriteTestPassed(MethodInfo m); - void WriteTestInconclusive(MethodInfo m); - void WriteTestFailure(MethodInfo m, Exception ex); + void WriteTestPassed(MethodInfo m, TimeSpan t); + void WriteTestFailure(MethodInfo m, Exception ex, TimeSpan t); void WriteFrameworkError(Exception ex); + void WriteSummary(int passed, List<(MethodInfo method, Exception exception, TimeSpan time)> failures, TimeSpan duration); } diff --git a/test/Intent.Primitives/OnlyAttribute.cs b/test/Intent.Primitives/OnlyAttribute.cs index 3d327e1f55..684bd1142d 100644 --- a/test/Intent.Primitives/OnlyAttribute.cs +++ b/test/Intent.Primitives/OnlyAttribute.cs @@ -3,7 +3,7 @@ namespace Intent; -[AttributeUsage(AttributeTargets.Assembly | AttributeTargets.Class | AttributeTargets.Constructor | AttributeTargets.Method)] +[AttributeUsage(AttributeTargets.Assembly | AttributeTargets.Class | AttributeTargets.Method)] public class OnlyAttribute : Attribute { } diff --git a/test/Intent.Primitives/TestAttribute.cs b/test/Intent.Primitives/TestAttribute.cs new file mode 100644 index 0000000000..57144b4239 --- /dev/null +++ b/test/Intent.Primitives/TestAttribute.cs @@ -0,0 +1,15 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +namespace Intent; + +[AttributeUsage(AttributeTargets.Method)] +public class TestAttribute : Attribute +{ + public TestAttribute(string name) + { + Name = name; + } + + public string Name { get; } +} diff --git a/test/Intent/ConsoleLogger.cs b/test/Intent/ConsoleLogger.cs index 326794c4ef..1dcf500ef5 100644 --- a/test/Intent/ConsoleLogger.cs +++ b/test/Intent/ConsoleLogger.cs @@ -2,35 +2,30 @@ // Licensed under the MIT license. See LICENSE file in the project root for full license information. namespace Intent.Console; + +using System; +using System.Collections.Generic; +using System.Linq; using System.Reflection; -using System.Text.RegularExpressions; using static System.Console; using static System.ConsoleColor; internal class ConsoleLogger : IRunLogger { - public void WriteTestInconclusive(MethodInfo m) - { - var currentColor = ForegroundColor; - ForegroundColor = Yellow; - WriteLine($"[?] {FormatMethodName(m.Name)}"); - ForegroundColor = currentColor; - } - - public void WriteTestPassed(MethodInfo m) + public void WriteTestPassed(MethodInfo m, TimeSpan t) { var currentColor = ForegroundColor; ForegroundColor = Green; - WriteLine($"[+] {FormatMethodName(m.Name)}"); + WriteLine($"[+] {FormatMethodName(m)} {(int)t.TotalMilliseconds} ms"); ForegroundColor = currentColor; } - public void WriteTestFailure(MethodInfo m, Exception ex) + public void WriteTestFailure(MethodInfo m, Exception ex, TimeSpan t) { var currentColor = ForegroundColor; ForegroundColor = Red; - WriteLine($"[-] {FormatMethodName(m.Name)}{Environment.NewLine}{ex}"); + WriteLine($"[-] {FormatMethodName(m)} {(int)t.TotalMilliseconds} ms{Environment.NewLine}{ex}"); ForegroundColor = currentColor; } @@ -42,17 +37,30 @@ public void WriteFrameworkError(Exception ex) ForegroundColor = currentColor; } - private static string FormatMethodName(string methodName) + private static string FormatMethodName(MethodInfo method) + { + var methodName = method.GetCustomAttribute() is TestAttribute test ? test.Name : method.Name; + if (!methodName.Contains('\n')) + { + return methodName; + } + + var lines = methodName.Split('\n').Select(line => line.Trim()); + var first = lines.Take(1).ToList(); + var rest = lines.Skip(1).Select(l => $"{Environment.NewLine} {l}").ToList(); + + return string.Join(null, first.Concat(rest)); + } + + public void WriteSummary(int passed, List<(MethodInfo method, Exception exception, TimeSpan time)> failures, TimeSpan duration) { - // insert space before every capital letter or number that is after a non-capital letter - var spaced = Regex.Replace(methodName, "(?<=[a-z])([A-Z0-9])", " $1"); - // insert space before every capital leter that is after a number - var spaced2 = Regex.Replace(spaced, "(?<=[0-9]|^)([A-Z])", " $1"); - var newLines = spaced2 - .Replace("_When", $"{Environment.NewLine} When") - .Replace("_And", $"{Environment.NewLine} And") - .Replace("_Then", $"{Environment.NewLine} Then"); - - return newLines.ToLowerInvariant(); + WriteLine(); + WriteLine(); + if (failures.Count > 0) { + WriteLine($"There were {failures.Count} failures:"); + } + failures.ForEach(t => { WriteTestFailure(t.method, t.exception, t.time); WriteLine(); }); + WriteLine(); + WriteLine($"Test run finished: Total: {passed + failures.Count} Passed: {passed} Failed: {failures.Count} Duration: {(int)duration.TotalMilliseconds} ms"); } } diff --git a/test/Intent/Runner.cs b/test/Intent/Runner.cs index 3bab3db2c7..4832cbcb1f 100644 --- a/test/Intent/Runner.cs +++ b/test/Intent/Runner.cs @@ -3,12 +3,15 @@ namespace Intent; +using System.Diagnostics; +using System.Linq; using System.Reflection; public class Runner { public static void Run(IEnumerable paths, IRunLogger logger) { + var assemblies = new List(); foreach (var path in paths) { try @@ -17,44 +20,71 @@ public static void Run(IEnumerable paths, IRunLogger logger) if (assembly.IsExcluded()) continue; - var types = assembly.GetTypes().SkipNonPublic().SkipExcluded(); - foreach (var type in types) - { - var methods = type.GetMethods().SkipExcluded(); + assemblies.Add(assembly); + } + catch (Exception ex) + { + logger.WriteFrameworkError(ex); + } + } + + var types = assemblies.SelectMany(assembly => assembly.GetTypes().SkipNonPublic().SkipExcluded()).ToList(); + var typesWithOnly = types.Where(type => type.GetCustomAttribute() != null).ToList(); + + var methods = types.SelectMany(type => type.GetMethods().SkipExcluded()).ToList(); + var methodsWithOnly = methods.Where(m => m.GetCustomAttribute() != null).ToList(); - // TODO: This chooses the Only tests only for single assembly and single class, - // to support this full we would have to enumerate all classes and methods first, - // it is easy, I just don't need it right now. - var methodsWithOnly = methods.Where(m => m.GetCustomAttribute() != null).ToList(); - if (methodsWithOnly.Count > 0) - methods = methodsWithOnly; + var methodsToRun = new List(); + if (typesWithOnly.Count > 0 || methodsWithOnly.Count > 0) + { + // Some types or methods are decorated with Only. Putting Only on a type should run all methods in + // that type. Putting Only on a method should run that method. + // + // So we need a list of all distinct methods that match that rule. + + var onlyMethodsFromTypes = typesWithOnly.SelectMany(type => type.GetMethods().SkipExcluded()).ToList(); + methodsToRun = onlyMethodsFromTypes.Concat(methodsWithOnly).Distinct().ToList(); + } + else + { + methodsToRun = methods; + } - foreach (var method in methods) + var failures = new List<(MethodInfo method, Exception exception, TimeSpan duration)>(); + var passed = 0; + var runStopwatch = Stopwatch.StartNew(); + foreach (var method in methodsToRun) + { + try + { + var testStopwatch = Stopwatch.StartNew(); + try + { + // Declaring type cannot be really null for types you define in C# + // without doing any reflection magic. + var instance = Activator.CreateInstance(method.DeclaringType!); + var testResult = method.Invoke(instance, Array.Empty()); + if (testResult is Task task) { - try - { - var instance = Activator.CreateInstance(type); - var testResult = method.Invoke(instance, Array.Empty()); - if (testResult is Task task) - { - // When the result is a task we need to await it. - // TODO: this can be improved with await, imho - task.GetAwaiter().GetResult(); - }; + // When the result is a task we need to await it. + // TODO: this can be improved with await, imho + task.GetAwaiter().GetResult(); + }; - logger.WriteTestPassed(method); - } - catch (Exception ex) - { - if (ex is TargetInvocationException tex && tex.InnerException != null) - { - logger.WriteTestFailure(method, tex.InnerException); - } - else - { - logger.WriteTestFailure(method, ex); - } - } + passed++; + logger.WriteTestPassed(method, testStopwatch.Elapsed); + } + catch (Exception ex) + { + if (ex is TargetInvocationException tex && tex.InnerException != null) + { + failures.Add((method, tex.InnerException, testStopwatch.Elapsed)); + logger.WriteTestFailure(method, tex.InnerException, testStopwatch.Elapsed); + } + else + { + failures.Add((method, ex, testStopwatch.Elapsed)); + logger.WriteTestFailure(method, ex, testStopwatch.Elapsed); } } } @@ -63,5 +93,7 @@ public static void Run(IEnumerable paths, IRunLogger logger) logger.WriteFrameworkError(ex); } } + + logger.WriteSummary(passed, failures, runStopwatch.Elapsed); } } diff --git a/test/vstest.ProgrammerTests/BasicRunAndDiscovery.cs b/test/vstest.ProgrammerTests/BasicRunAndDiscovery.cs new file mode 100644 index 0000000000..339872f12f --- /dev/null +++ b/test/vstest.ProgrammerTests/BasicRunAndDiscovery.cs @@ -0,0 +1,265 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +namespace vstest.ProgrammerTests; + +using System.Diagnostics; +using System.Runtime.Versioning; + +using FluentAssertions; +using FluentAssertions.Extensions; + +using Microsoft.VisualStudio.TestPlatform.Client; +using Microsoft.VisualStudio.TestPlatform.CommandLine; +using Microsoft.VisualStudio.TestPlatform.CommandLine.Publisher; +using Microsoft.VisualStudio.TestPlatform.CommandLine.TestPlatformHelpers; +using Microsoft.VisualStudio.TestPlatform.CommandLineUtilities; +using Microsoft.VisualStudio.TestPlatform.CommunicationUtilities.ObjectModel; +using Microsoft.VisualStudio.TestPlatform.CrossPlatEngine; +using Microsoft.VisualStudio.TestPlatform.CrossPlatEngine.TestRunAttachmentsProcessing; +using Microsoft.VisualStudio.TestPlatform.ObjectModel; +using Microsoft.VisualStudio.TestPlatform.ObjectModel.Client; + +# if DEBUG +using Microsoft.VisualStudio.TestPlatform.CommunicationUtilities.Interfaces; +#endif + +using vstest.ProgrammerTests.Fakes; +using Intent; + +// Tests are run by Intent library that is executed from our Program.Main. To debug press F5 in VS, and maybe mark just a single test with [Only]. +// To just run, press Ctrl+F5 to run without debugging. It will use short timeout for abort in case something is wrong with your test. + + +public class BasicRunAndDiscovery +{ + [Test(@" + Given a test assembly with 108 tests. + + When we run tests. + + Then all 108 tests are executed. + ")] + public async Task A() + { + // -- arrange + var fakeErrorAggregator = new FakeErrorAggregator(); + var commandLineOptions = new CommandLineOptions { IsDesignMode = true }; + + var fakeCurrentProcess = new FakeProcess(fakeErrorAggregator, @"X:\fake\vstest.console.exe"); + var fakeProcessHelper = new FakeProcessHelper(fakeErrorAggregator, fakeCurrentProcess); + + var fakeFileHelper = new FakeFileHelper(fakeErrorAggregator); + // TODO: Get framework name from constants + // TODO: have mstest1dll canned + var tests = new FakeTestBatchBuilder() + .WithTotalCount(108) + .WithDuration(100.Milliseconds()) + .WithBatchSize(10) + .Build(); + var mstest1Dll = new FakeTestDllFile(@"X:\fake\mstest1.dll", new FrameworkName(".NETCoreApp,Version=v5.0"), Architecture.X64, tests); + + List changeMessages = tests.Take(tests.Count - 1).Select(batch => // TODO: make the stats agree with the tests below + new FakeMessage(MessageType.TestRunStatsChange, + new TestRunChangedEventArgs(new TestRunStatistics(new Dictionary { [TestOutcome.Passed] = batch.Count }), batch, new List()) + )).ToList(); + FakeMessage completedMessage = new FakeMessage(MessageType.ExecutionComplete, new TestRunCompletePayload + { + // TODO: make the stats agree with the tests below + TestRunCompleteArgs = new TestRunCompleteEventArgs(new TestRunStatistics(new Dictionary { [TestOutcome.Passed] = 1 }), false, false, null, new System.Collections.ObjectModel.Collection(), TimeSpan.Zero), + LastRunTests = new TestRunChangedEventArgs(new TestRunStatistics(new Dictionary { [TestOutcome.Passed] = 1 }), tests.Last(), new List()), + }); + List messages = changeMessages.Concat(new[] { completedMessage }).ToList(); + var responses = new List> { + new RequestResponsePair(MessageType.VersionCheck, new FakeMessage(MessageType.VersionCheck, 5)), + new RequestResponsePair(MessageType.ExecutionInitialize, FakeMessage.NoResponse), + new RequestResponsePair(MessageType.StartTestExecutionWithSources, messages, false), + new RequestResponsePair(MessageType.SessionEnd, new [] { FakeMessage.NoResponse }, message => + { + // TODO: how do we associate this to the correct process? + var fp = fakeProcessHelper.Processes.Last(); + fakeProcessHelper.TerminateProcess(fp); + }), + new RequestResponsePair(MessageType.SessionEnd, FakeMessage.NoResponse ), + }; + + var fakeCommunicationChannel = new FakeCommunicationChannel(responses, fakeErrorAggregator, 1); + fakeCommunicationChannel.Start(new object()); + var fakeCommunicationEndpoint = new FakeCommunicationEndpoint(fakeCommunicationChannel, fakeErrorAggregator); +#if DEBUG + TestServiceLocator.Clear(); + TestServiceLocator.Register(fakeCommunicationEndpoint.TestHostConnectionInfo.Endpoint, fakeCommunicationEndpoint); +#else + // This fools compiler into not being able to tell that the the rest of the code is unreachable. + var a = true; + if (a) + { + throw new InvalidOperationException("Tests cannot run in Release mode, because TestServiceLocator is compiled only for Debug, and so the tests will fail to setup channel and will hang."); + } +#endif + var fakeTestHostProcess = new FakeProcess(fakeErrorAggregator, @"C:\temp\testhost.exe"); + var fakeTestRuntimeProvider = new FakeTestRuntimeProvider(fakeProcessHelper, fakeTestHostProcess, fakeFileHelper, mstest1Dll.AsList(), fakeCommunicationEndpoint, fakeErrorAggregator); + var fakeTestRuntimeProviderManager = new FakeTestRuntimeProviderManager(fakeErrorAggregator); + fakeTestRuntimeProviderManager.AddTestRuntimeProviders(fakeTestRuntimeProvider); + var testEngine = new TestEngine(fakeTestRuntimeProviderManager, fakeProcessHelper); + + var testPlatform = new TestPlatform(testEngine, fakeFileHelper, fakeTestRuntimeProviderManager); + + var testRunResultAggregator = new TestRunResultAggregator(); + var fakeTestPlatformEventSource = new FakeTestPlatformEventSource(fakeErrorAggregator); + + var fakeAssemblyMetadataProvider = new FakeAssemblyMetadataProvider(fakeFileHelper, fakeErrorAggregator); + var inferHelper = new InferHelper(fakeAssemblyMetadataProvider); + + // This is most likely not the correctl place where to cut this off, plugin cache is probably the better place, + // but it is not injected, and I don't want to investigate this now. + var fakeDataCollectorAttachmentsProcessorsFactory = new FakeDataCollectorAttachmentsProcessorsFactory(fakeErrorAggregator); + var testRunAttachmentsProcessingManager = new TestRunAttachmentsProcessingManager(fakeTestPlatformEventSource, fakeDataCollectorAttachmentsProcessorsFactory); + + Task fakeMetricsPublisherTask = Task.FromResult(new FakeMetricsPublisher(fakeErrorAggregator)); + TestRequestManager testRequestManager = new( + commandLineOptions, + testPlatform, + testRunResultAggregator, + fakeTestPlatformEventSource, + inferHelper, + fakeMetricsPublisherTask, + fakeProcessHelper, + testRunAttachmentsProcessingManager + ); + + // -- act + + // TODO: this gives me run configuration that is way too complete, do we a way to generate "bare" runsettings? if not we should add them. Would be also useful to get + // runsettings from parameter set so people can use it + // TODO: TestSessionTimeout gives me way to abort the run without having to cancel it externally, but could probably still lead to hangs if that funtionality is broken + // TODO: few tries later, that is exactly the case when we abort, it still hangs on waiting to complete request, because test run complete was not sent + // var runConfiguration = new Microsoft.VisualStudio.TestPlatform.ObjectModel.RunConfiguration { TestSessionTimeout = 40_000 }.ToXml().OuterXml; + var runConfiguration = string.Empty; + var testRunRequestPayload = new TestRunRequestPayload + { + // TODO: passing null sources and null testcases does not fail fast + Sources = mstest1Dll.Path.AsList(), + // TODO: passing null runsettings does not fail fast, instead it fails in Fakes settings code + // TODO: passing empty string fails in the xml parser code + RunSettings = $"{runConfiguration}" + }; + + // var fakeTestHostLauncher = new FakeTestHostLauncher(); + var fakeTestRunEventsRegistrar = new FakeTestRunEventsRegistrar(fakeErrorAggregator); + var protocolConfig = new ProtocolConfig(); + + // TODO: we make sure the test is running 10 minutes at max and then we try to abort + // if we aborted we write the error to aggregator, this needs to be made into a pattern + // so we can avoid hanging if the run does not complete correctly + var cancelAbort = new CancellationTokenSource(); + var task = Task.Run(async () => + { + await Task.Delay(TimeSpan.FromSeconds(Debugger.IsAttached ? 100 : 10), cancelAbort.Token); + if (Debugger.IsAttached) + { + // we will abort because we are hanging, look at stacks to see what the problem is + Debugger.Break(); + } + fakeErrorAggregator.Add(new Exception("errr we aborted")); + testRequestManager.AbortTestRun(); + + }); + testRequestManager.RunTests(testRunRequestPayload, testHostLauncher: null, fakeTestRunEventsRegistrar, protocolConfig); + cancelAbort.Cancel(); + if (!task.IsCanceled) + { + await task; + } + // pattern end + + // -- assert + fakeErrorAggregator.Errors.Should().BeEmpty(); + fakeTestRunEventsRegistrar.RunChangedEvents.SelectMany(er => er.Data.NewTestResults).Should().HaveCount(108); + } + + [Test(@" + Given multple test assemblies that use the same target framework. + + When we run tests. + + Then all tests from all assemblies are run.. + ")] + public async Task B() + { + // -- arrange + using var fixture = new Fixture(); + + var mstest1Dll = new FakeTestDllBuilder() + .WithPath(@"X:\fake\mstest1.dll") + .WithFramework(KnownFrameworkNames.Net5) + .WithArchitecture(Architecture.X64) + .WithTestCount(108, 10) + .Build(); + + var testhost1Process = new FakeProcess(fixture.ErrorAggregator, @"X:\fake\testhost1.exe"); + + var runTests1 = new FakeTestHostResponsesBuilder() + .VersionCheck(5) + .ExecutionInitialize(FakeMessage.NoResponse) + .StartTestExecutionWithSources(mstest1Dll.TestResultBatches) + .SessionEnd(FakeMessage.NoResponse, _ => testhost1Process.Exit()) + .SessionEnd(FakeMessage.NoResponse) + .Build(); + + var testhost1 = new FakeTestHostFixtureBuilder(fixture) + .WithTestDll(mstest1Dll) + .WithProcess(testhost1Process) + .WithResponses(runTests1) + .Build(); + + var mstest2Dll = new FakeTestDllBuilder() + .WithPath(@"X:\fake\mstest2.dll") + .WithFramework(KnownFrameworkNames.Net5) + .WithArchitecture(Architecture.X64) + .WithTestCount(50, 8) + .Build(); + + var testhost2Process = new FakeProcess(fixture.ErrorAggregator, @"X:\fake\testhost2.exe"); + + var runTests2 = new FakeTestHostResponsesBuilder() + .VersionCheck(5) + .ExecutionInitialize(FakeMessage.NoResponse) + .StartTestExecutionWithSources(mstest2Dll.TestResultBatches) + .SessionEnd(FakeMessage.NoResponse, f => f.Process.Exit()) + .SessionEnd(FakeMessage.NoResponse) + .Build(); + + var testhost2 = new FakeTestHostFixtureBuilder(fixture) + .WithTestDll(mstest2Dll) + .WithProcess(testhost2Process) + .WithResponses(runTests2) + .Build(); + + fixture.AddTestHostFixtures(testhost1, testhost2); + + var testRequestManager = fixture.BuildTestRequestManager(); + + // -- act + var runConfiguration = string.Empty; + var testRunRequestPayload = new TestRunRequestPayload + { + Sources = new List { mstest1Dll.Path, mstest2Dll.Path }, + + RunSettings = $"{runConfiguration}" + }; + + await testRequestManager.ExecuteWithAbort(tm => tm.RunTests(testRunRequestPayload, testHostLauncher: null, fixture.TestRunEventsRegistrar, fixture.ProtocolConfig)); + + // -- assert + fixture.AssertNoErrors(); + fixture.ExecutedTests.Should().HaveCount(mstest1Dll.TestCount + mstest2Dll.TestCount); + } +} + +// Test and improvmement ideas: +// TODO: passing null runsettings does not fail fast, instead it fails in Fakes settings code +// TODO: passing empty string fails in the xml parser code +// TODO: passing null sources and null testcases does not fail fast +// TODO: Just calling Exit, Close won't stop the run, we will keep waiting for test run to complete, I think in real life when we exit then Disconnected will be called on the vstest.console side, leading to abort flow. +//.StartTestExecutionWithSources(new FakeMessage(MessageType.TestMessage, new TestMessagePayload { MessageLevel = TestMessageLevel.Error, Message = "Loading type failed." }), afterAction: f => { /*f.Process.Exit();*/ f.FakeCommunicationEndpoint.Disconnect(); }) diff --git a/test/vstest.ProgrammerTests/Fakes/FakeCommunicationChannel.cs b/test/vstest.ProgrammerTests/Fakes/FakeCommunicationChannel.cs index 6965bde793..53bf0d681f 100644 --- a/test/vstest.ProgrammerTests/Fakes/FakeCommunicationChannel.cs +++ b/test/vstest.ProgrammerTests/Fakes/FakeCommunicationChannel.cs @@ -20,7 +20,6 @@ public FakeCommunicationChannel(int id) public int Id { get; } public CancellationTokenSource CancellationTokenSource { get; } = new(); - public BlockingCollection InQueue { get; } = new(); public BlockingCollection OutQueue { get; } = new(); @@ -47,6 +46,7 @@ public void Dispose() { CancellationTokenSource.Cancel(); InQueue.CompleteAdding(); + OutQueue.CompleteAdding(); } public Task NotifyDataAvailable() @@ -104,11 +104,12 @@ private void ProcessOutgoingMessages() { try { - // TODO: better name? this is message that we are currently trying to send - OutgoingMessage = OutQueue.Take(); + // TODO: better name for the property? This is message that we are currently trying to send. + OutgoingMessage = OutQueue.Take(token); OnMessageReceived(this, new MessageReceivedEventArgs { Data = OutgoingMessage.SerializedMessage }); OutgoingMessage = null; } + catch (OperationCanceledException) { } catch (Exception ex) { FakeErrorAggregator.Add(ex); @@ -198,6 +199,7 @@ private void ProcessIncomingMessages(TContext context) responsePair.AfterAction?.Invoke(context); } } + catch (OperationCanceledException) { } catch (Exception ex) { FakeErrorAggregator.Add(ex); diff --git a/test/vstest.ProgrammerTests/Fakes/FakeTestHost.cs b/test/vstest.ProgrammerTests/Fakes/FakeTestHost.cs index ea425831b4..3a1cb11ab3 100644 --- a/test/vstest.ProgrammerTests/Fakes/FakeTestHost.cs +++ b/test/vstest.ProgrammerTests/Fakes/FakeTestHost.cs @@ -2,7 +2,7 @@ // Licensed under the MIT license. See LICENSE file in the project root for full license information. namespace vstest.ProgrammerTests.Fakes; -internal class FakeTestHostFixture +internal class FakeTestHostFixture : IDisposable { public int Id { get; } public List Dlls { get; } @@ -33,4 +33,9 @@ public FakeTestHostFixture( // testhost before or after answering. fakeCommunicationChannel.Start(this); } + + public void Dispose() + { + try { FakeCommunicationChannel.Dispose(); } catch (ObjectDisposedException) { } + } } diff --git a/test/vstest.ProgrammerTests/Fakes/Fixture.cs b/test/vstest.ProgrammerTests/Fakes/Fixture.cs index bbf6dd7d9d..563568a0b5 100644 --- a/test/vstest.ProgrammerTests/Fakes/Fixture.cs +++ b/test/vstest.ProgrammerTests/Fakes/Fixture.cs @@ -18,8 +18,11 @@ namespace vstest.ProgrammerTests.Fakes; internal class Fixture : IDisposable { + private readonly List _disposables = new(); + public FakeErrorAggregator ErrorAggregator { get; } = new(); public FakeProcessHelper ProcessHelper { get; } + public string LogName { get; } public FakeProcess CurrentProcess { get; } public FakeFileHelper FileHelper { get; } public FakeTestRuntimeProviderManager TestRuntimeProviderManager { get; } @@ -34,10 +37,11 @@ internal class Fixture : IDisposable public FakeDataCollectorAttachmentsProcessorsFactory? DataCollectorAttachmentsProcessorsFactory { get; private set; } public TestRunAttachmentsProcessingManager? TestRunAttachmentsProcessingManager { get; private set; } public TestRequestManager? TestRequestManager { get; private set; } - public List ExecutedTests => TestRunEventsRegistrar.RunChangedEvents.SelectMany(er => er.Data.NewTestResults).ToList(); - public ProtocolConfig ProtocolConfig { get; internal set; } + public List ExecutedTests => TestRunEventsRegistrar.RunChangedEvents.SelectMany(er => er.Data.NewTestResults).ToList(); + public List DiscoveredTests => TestDiscoveryEventsRegistrar.DiscoveredTestsEvents.SelectMany(er => er.Data.DiscoveredTestCases).ToList(); + public Fixture(FixtureOptions? fixtureOptions = null) { // This type is compiled only in DEBUG, and won't exist otherwise. @@ -58,22 +62,31 @@ public Fixture(FixtureOptions? fixtureOptions = null) fixtureOptions?.FeatureFlags?.ToList().ForEach(flag => FeatureFlag.SetFlag(flag.Key, flag.Value)); #pragma warning restore CS0618 // Type or member is obsolete + // This makes the run a bit slower, but at least we get info in the output window. We probably should add a mode where we don't + // use a file to write the output. Just trace listener. That would also be useful for UWP I think. + LogName = Path.GetTempPath() + $"/log_{Guid.NewGuid()}.txt"; + //EqtTrace.InitializeVerboseTrace(LogName); CurrentProcess = new FakeProcess(ErrorAggregator, @"X:\fake\vstest.console.exe", string.Empty, null, null, null, null, null); ProcessHelper = new FakeProcessHelper(ErrorAggregator, CurrentProcess); FileHelper = new FakeFileHelper(ErrorAggregator); TestRuntimeProviderManager = new FakeTestRuntimeProviderManager(ErrorAggregator); TestRunEventsRegistrar = new FakeTestRunEventsRegistrar(ErrorAggregator); - TestRunEventsRegistrar = new FakeTestRunEventsRegistrar(ErrorAggregator); + TestDiscoveryEventsRegistrar = new FakeTestDiscoveryEventsRegistrar(ErrorAggregator); ProtocolConfig = new ProtocolConfig(); } + public void Dispose() { - + foreach (var disposable in _disposables) + { + try { disposable.Dispose(); } catch (ObjectDisposedException) { } + } } internal void AddTestHostFixtures(params FakeTestHostFixture[] testhosts) { + _disposables.AddRange(testhosts); var providers = testhosts.Select(t => t.FakeTestRuntimeProvider).ToArray(); TestRuntimeProviderManager.AddTestRuntimeProviders(providers); } @@ -103,7 +116,14 @@ internal TestRequestManagerTestHelper BuildTestRequestManager( Task fakeMetricsPublisherTask = Task.FromResult(new FakeMetricsPublisher(ErrorAggregator)); - var commandLineOptions = CommandLineOptions.Instance; + var commandLineOptions = new CommandLineOptions + { + // We are acting like we are running under IDE. This is done because some settings are trying to grab the + // value from the pre-parsed settings in command line options. And some are looking at the actual run settings. + // Ultimately we should have a single point of truth for both scenarios, but now it is easier to just provide + // runsettings to the request. + IsDesignMode = true, + }; TestRequestManager testRequestManager = new( commandLineOptions, TestPlatform, diff --git a/test/vstest.ProgrammerTests/Fakes/TestRequestManagerHelper.cs b/test/vstest.ProgrammerTests/Fakes/TestRequestManagerHelper.cs index bf83c8599e..9eab05998d 100644 --- a/test/vstest.ProgrammerTests/Fakes/TestRequestManagerHelper.cs +++ b/test/vstest.ProgrammerTests/Fakes/TestRequestManagerHelper.cs @@ -36,15 +36,22 @@ public async Task ExecuteWithAbort(Action testRequsestManage var abortOnTimeout = Task.Run(async () => { // Wait until timeout or until we are cancelled. - await Task.Delay(TimeSpan.FromSeconds(Debugger.IsAttached ? _debugOptions.DebugTimeout : _debugOptions.Timeout), cancelAbort.Token); - if (Debugger.IsAttached && _debugOptions.BreakOnAbort) + try + { + await Task.Delay(TimeSpan.FromSeconds(Debugger.IsAttached ? _debugOptions.DebugTimeout : _debugOptions.Timeout), cancelAbort.Token); + + if (Debugger.IsAttached && _debugOptions.BreakOnAbort) + { + var errors = _errorAggregator.Errors; + // we will abort because we are hanging, look at errors and at concurrent stacks to see where we are hanging. + Debugger.Break(); + } + _errorAggregator.Add(new Exception("errr we aborted")); + _testRequestManager.AbortTestRun(); + } + catch (TaskCanceledException) { - var errors = _errorAggregator.Errors; - // we will abort because we are hanging, look at errors and at concurrent stacks to see where we are hanging. - Debugger.Break(); } - _errorAggregator.Add(new Exception("errr we aborted")); - _testRequestManager.AbortTestRun(); }); testRequsestManagerAction(_testRequestManager); diff --git a/test/vstest.ProgrammerTests/UnitTest1.cs b/test/vstest.ProgrammerTests/MultiTFMRunAndDiscovery.cs similarity index 59% rename from test/vstest.ProgrammerTests/UnitTest1.cs rename to test/vstest.ProgrammerTests/MultiTFMRunAndDiscovery.cs index afc4109c54..2b93b3e141 100644 --- a/test/vstest.ProgrammerTests/UnitTest1.cs +++ b/test/vstest.ProgrammerTests/MultiTFMRunAndDiscovery.cs @@ -1,194 +1,167 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. +// Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT license. See LICENSE file in the project root for full license information. namespace vstest.ProgrammerTests; -using System.Diagnostics; -using System.Runtime.Versioning; - using FluentAssertions; -using FluentAssertions.Extensions; - -using Microsoft.VisualStudio.TestPlatform.Client; -using Microsoft.VisualStudio.TestPlatform.CommandLine; -using Microsoft.VisualStudio.TestPlatform.CommandLine.Publisher; -using Microsoft.VisualStudio.TestPlatform.CommandLine.TestPlatformHelpers; -using Microsoft.VisualStudio.TestPlatform.CommandLineUtilities; using Microsoft.VisualStudio.TestPlatform.CommunicationUtilities.ObjectModel; -using Microsoft.VisualStudio.TestPlatform.CrossPlatEngine; -using Microsoft.VisualStudio.TestPlatform.CrossPlatEngine.TestRunAttachmentsProcessing; using Microsoft.VisualStudio.TestPlatform.ObjectModel; using Microsoft.VisualStudio.TestPlatform.ObjectModel.Client; # if DEBUG -using Microsoft.VisualStudio.TestPlatform.CommunicationUtilities.Interfaces; #endif using vstest.ProgrammerTests.Fakes; using Microsoft.VisualStudio.TestPlatform.Utilities; using Intent; -// Tests are run by Intent library that is executed from our Program.Main. To debug press F5 in VS, and maybe mark just a single test with [Only]. -// To just run, press Ctrl+F5 to run without debugging. It will use short timeout for abort in case something is wrong with your test. -public class TestDiscoveryTests +public class MultiTFMRunAndDiscovery { - public async Task GivenAnMSTestAssemblyWith108Tests_WhenTestsAreRun_Then108TestsAreExecuted() + [Test(@" + Given two test assemblies that have the same architecture + but have different target frameworks. + + When we run test discovery. + + Then two testhosts should be started that target the same framework as each assembly. + ")] + public async Task A() { // -- arrange - var fakeErrorAggregator = new FakeErrorAggregator(); - var commandLineOptions = CommandLineOptions.Instance; - - var fakeCurrentProcess = new FakeProcess(fakeErrorAggregator, @"X:\fake\vstest.console.exe"); - var fakeProcessHelper = new FakeProcessHelper(fakeErrorAggregator, fakeCurrentProcess); - - var fakeFileHelper = new FakeFileHelper(fakeErrorAggregator); - // TODO: Get framework name from constants - // TODO: have mstest1dll canned - var tests = new FakeTestBatchBuilder() - .WithTotalCount(108) - .WithDuration(100.Milliseconds()) - .WithBatchSize(10) + using var fixture = new Fixture( + new FixtureOptions + { + FeatureFlags = new Dictionary + { + [FeatureFlag.MULTI_TFM_RUN] = true + } + } + ); + + var mstest1Dll = new FakeTestDllBuilder() + .WithPath(@"X:\fake\mstest1.dll") + .WithFramework(KnownFrameworkNames.Net5) // <--- + .WithArchitecture(Architecture.X64) + .WithTestCount(11, 5) .Build(); - var mstest1Dll = new FakeTestDllFile(@"X:\fake\mstest1.dll", new FrameworkName(".NETCoreApp,Version=v5.0"), Architecture.X64, tests); - List changeMessages = tests.Take(tests.Count - 1).Select(batch => // TODO: make the stats agree with the tests below - new FakeMessage(MessageType.TestRunStatsChange, - new TestRunChangedEventArgs(new TestRunStatistics(new Dictionary { [TestOutcome.Passed] = batch.Count }), batch, new List()) - )).ToList(); - FakeMessage completedMessage = new FakeMessage(MessageType.ExecutionComplete, new TestRunCompletePayload - { - // TODO: make the stats agree with the tests below - TestRunCompleteArgs = new TestRunCompleteEventArgs(new TestRunStatistics(new Dictionary { [TestOutcome.Passed] = 1 }), false, false, null, new System.Collections.ObjectModel.Collection(), TimeSpan.Zero), - LastRunTests = new TestRunChangedEventArgs(new TestRunStatistics(new Dictionary { [TestOutcome.Passed] = 1 }), tests.Last(), new List()), - }); - List messages = changeMessages.Concat(new[] { completedMessage }).ToList(); - var responses = new List> { - new RequestResponsePair(MessageType.VersionCheck, new FakeMessage(MessageType.VersionCheck, 5)), - new RequestResponsePair(MessageType.ExecutionInitialize, FakeMessage.NoResponse), - new RequestResponsePair(MessageType.StartTestExecutionWithSources, messages, false), - new RequestResponsePair(MessageType.SessionEnd, new [] { FakeMessage.NoResponse }, message => - { - // TODO: how do we associate this to the correct process? - var fp = fakeProcessHelper.Processes.Last(); - fakeProcessHelper.TerminateProcess(fp); - }), - }; + var testhost1Process = new FakeProcess(fixture.ErrorAggregator, @"X:\fake\testhost1.exe"); - var fakeCommunicationChannel = new FakeCommunicationChannel(responses, fakeErrorAggregator, 1); - fakeCommunicationChannel.Start(new object()); - var fakeCommunicationEndpoint = new FakeCommunicationEndpoint(fakeCommunicationChannel, fakeErrorAggregator); -#if DEBUG - TestServiceLocator.Clear(); - TestServiceLocator.Register(fakeCommunicationEndpoint.TestHostConnectionInfo.Endpoint, fakeCommunicationEndpoint); -#else - // This fools compiler into not being able to tell that the the rest of the code is unreachable. - var a = true; - if (a) - { - throw new InvalidOperationException("Tests cannot run in Release mode, because TestServiceLocator is compiled only for Debug, and so the tests will fail to setup channel and will hang."); - } -#endif - var fakeTestHostProcess = new FakeProcess(fakeErrorAggregator, @"C:\temp\testhost.exe"); - var fakeTestRuntimeProvider = new FakeTestRuntimeProvider(fakeProcessHelper, fakeTestHostProcess, fakeFileHelper, mstest1Dll.AsList(), fakeCommunicationEndpoint, fakeErrorAggregator); - var fakeTestRuntimeProviderManager = new FakeTestRuntimeProviderManager(fakeErrorAggregator); - fakeTestRuntimeProviderManager.AddTestRuntimeProviders(fakeTestRuntimeProvider); - var testEngine = new TestEngine(fakeTestRuntimeProviderManager, fakeProcessHelper); - - var testPlatform = new TestPlatform(testEngine, fakeFileHelper, fakeTestRuntimeProviderManager); - - var testRunResultAggregator = new TestRunResultAggregator(); - var fakeTestPlatformEventSource = new FakeTestPlatformEventSource(fakeErrorAggregator); - - var fakeAssemblyMetadataProvider = new FakeAssemblyMetadataProvider(fakeFileHelper, fakeErrorAggregator); - var inferHelper = new InferHelper(fakeAssemblyMetadataProvider); - - // This is most likely not the correctl place where to cut this off, plugin cache is probably the better place, - // but it is not injected, and I don't want to investigate this now. - var fakeDataCollectorAttachmentsProcessorsFactory = new FakeDataCollectorAttachmentsProcessorsFactory(fakeErrorAggregator); - var testRunAttachmentsProcessingManager = new TestRunAttachmentsProcessingManager(fakeTestPlatformEventSource, fakeDataCollectorAttachmentsProcessorsFactory); - - Task fakeMetricsPublisherTask = Task.FromResult(new FakeMetricsPublisher(fakeErrorAggregator)); - TestRequestManager testRequestManager = new( - commandLineOptions, - testPlatform, - testRunResultAggregator, - fakeTestPlatformEventSource, - inferHelper, - fakeMetricsPublisherTask, - fakeProcessHelper, - testRunAttachmentsProcessingManager - ); + var runTests1 = new FakeTestHostResponsesBuilder() + .VersionCheck(5) + .DiscoveryInitialize(FakeMessage.NoResponse) + .StartDiscovery(mstest1Dll.TestResultBatches) + .SessionEnd(FakeMessage.NoResponse, afterAction: _ => testhost1Process.Exit()) + .Build(); - // -- act + var testhost1 = new FakeTestHostFixtureBuilder(fixture) + .WithTestDll(mstest1Dll) + .WithProcess(testhost1Process) + .WithResponses(runTests1) + .Build(); - // TODO: this gives me run configuration that is way too complete, do we a way to generate "bare" runsettings? if not we should add them. Would be also useful to get - // runsettings from parameter set so people can use it - // TODO: TestSessionTimeout gives me way to abort the run without having to cancel it externally, but could probably still lead to hangs if that funtionality is broken - // TODO: few tries later, that is exactly the case when we abort, it still hangs on waiting to complete request, because test run complete was not sent - // var runConfiguration = new Microsoft.VisualStudio.TestPlatform.ObjectModel.RunConfiguration { TestSessionTimeout = 40_000 }.ToXml().OuterXml; - var runConfiguration = string.Empty; - var testRunRequestPayload = new TestRunRequestPayload - { - // TODO: passing null sources and null testcases does not fail fast - Sources = mstest1Dll.Path.AsList(), - // TODO: passing null runsettings does not fail fast, instead it fails in Fakes settings code - // TODO: passing empty string fails in the xml parser code - RunSettings = $"{runConfiguration}" - }; + // -- - // var fakeTestHostLauncher = new FakeTestHostLauncher(); - var fakeTestRunEventsRegistrar = new FakeTestRunEventsRegistrar(fakeErrorAggregator); - var protocolConfig = new ProtocolConfig(); + var mstest2Dll = new FakeTestDllBuilder() + .WithPath(@"X:\fake\mstest2.dll") + .WithFramework(KnownFrameworkNames.Net48) // <--- + .WithArchitecture(Architecture.X64) + .WithTestCount(21, 5) + .Build(); - // TODO: we make sure the test is running 10 minutes at max and then we try to abort - // if we aborted we write the error to aggregator, this needs to be made into a pattern - // so we can avoid hanging if the run does not complete correctly - var cancelAbort = new CancellationTokenSource(); - var task = Task.Run(async () => - { - await Task.Delay(TimeSpan.FromSeconds(Debugger.IsAttached ? 100 : 10), cancelAbort.Token); - if (Debugger.IsAttached) - { - // we will abort because we are hanging, look at stacks to see what the problem is - Debugger.Break(); - } - fakeErrorAggregator.Add(new Exception("errr we aborted")); - testRequestManager.AbortTestRun(); + var testhost2Process = new FakeProcess(fixture.ErrorAggregator, @"X:\fake\testhost2.exe"); - }); - testRequestManager.RunTests(testRunRequestPayload, testHostLauncher: null, fakeTestRunEventsRegistrar, protocolConfig); - cancelAbort.Cancel(); - if (!task.IsCanceled) + var runTests2 = new FakeTestHostResponsesBuilder() + .VersionCheck(5) + .DiscoveryInitialize(FakeMessage.NoResponse) + .StartDiscovery(mstest2Dll.TestResultBatches) + .SessionEnd(FakeMessage.NoResponse, _ => testhost2Process.Exit()) + // We actually do get asked to terminate multiple times. In the second host only. + .SessionEnd(FakeMessage.NoResponse) + .Build(); + + var testhost2 = new FakeTestHostFixtureBuilder(fixture) + .WithTestDll(mstest2Dll) + .WithProcess(testhost2Process) + .WithResponses(runTests2) + .Build(); + + fixture.AddTestHostFixtures(testhost1, testhost2); + + var testRequestManager = fixture.BuildTestRequestManager(); + + mstest1Dll.FrameworkName.Should().NotBe(mstest2Dll.FrameworkName); + + // -- act + var testDiscoveryPayload = new DiscoveryRequestPayload { - await task; - } - // pattern end + Sources = new List { mstest1Dll.Path, mstest2Dll.Path }, + RunSettings = $"" + }; + + await testRequestManager.ExecuteWithAbort(tm => tm.DiscoverTests(testDiscoveryPayload, fixture.TestDiscoveryEventsRegistrar, fixture.ProtocolConfig)); // -- assert - fakeErrorAggregator.Errors.Should().BeEmpty(); - fakeTestRunEventsRegistrar.RunChangedEvents.SelectMany(er => er.Data.NewTestResults).Should().HaveCount(108); + fixture.AssertNoErrors(); + // We figure out the framework for each assembly so there should be no incompatibility warnings + // REVIEW: This should be uncommented. Commenting it now, because it is helpful to see those warnings. + // fixture.TestRunEventsRegistrar.LoggedWarnings.Should().NotContainMatch("Test run detected DLL(s) which were built for different framework*"); + + fixture.ProcessHelper.Processes.Where(p => p.Started).Should().HaveCount(2); + var startWithSources1 = testhost1.FakeCommunicationChannel.ProcessedMessages.Single(m => m.Request.MessageType == MessageType.StartDiscovery); + var startWithSources1Text = startWithSources1.Request.Payload.Select(t => t.ToString()).JoinBySpace(); + // We sent mstest1.dll. + startWithSources1Text.Should().Contain("mstest1.dll"); + // And we sent net5 as the target framework, because that is the framework of mstest1.dll. + startWithSources1Text.Should().Contain(KnownFrameworkStrings.Net5); + + var startWithSources2 = testhost2.FakeCommunicationChannel.ProcessedMessages.Single(m => m.Request.MessageType == MessageType.StartDiscovery); + var startWithSources2Text = startWithSources2.Request.Payload.Select(t => t.ToString()).JoinBySpace(); + // We sent mstest2.dll. + startWithSources2Text.Should().Contain("mstest2.dll"); + // And we sent net48 as the target framework, because that is the framework of mstest2.dll. + startWithSources2Text.Should().Contain(mstest2Dll.FrameworkName.ToString()); + + fixture.DiscoveredTests.Should().HaveCount(mstest1Dll.TestCount + mstest2Dll.TestCount); } - public async Task GivenMultipleMsTestAssembliesThatUseTheSameTargetFrameworkAndArchitecture_WhenTestsAreRun_ThenAllTestsFromAllAssembliesAreRun() + [Test(@" + Given two test assemblies that have the same architecture + but have different target frameworks. + + When we run test discovery + and provide runsettings that define the desired target framework. + + Then two testhosts should be started that target the framework chosen by runsettings. + ")] + public async Task B() { // -- arrange - using var fixture = new Fixture(); + using var fixture = new Fixture( + new FixtureOptions + { + FeatureFlags = new Dictionary + { + [FeatureFlag.MULTI_TFM_RUN] = true + } + } + ); var mstest1Dll = new FakeTestDllBuilder() .WithPath(@"X:\fake\mstest1.dll") - .WithFramework(KnownFrameworkNames.Net5) + .WithFramework(KnownFrameworkNames.Net5) // <--- .WithArchitecture(Architecture.X64) - .WithTestCount(108, 10) + .WithTestCount(11, 5) .Build(); var testhost1Process = new FakeProcess(fixture.ErrorAggregator, @"X:\fake\testhost1.exe"); var runTests1 = new FakeTestHostResponsesBuilder() .VersionCheck(5) - .ExecutionInitialize(FakeMessage.NoResponse) - .StartTestExecutionWithSources(mstest1Dll.TestResultBatches) - .SessionEnd(FakeMessage.NoResponse, _ => testhost1Process.Exit()) + .DiscoveryInitialize(FakeMessage.NoResponse) + .StartDiscovery(mstest1Dll.TestResultBatches) + .SessionEnd(FakeMessage.NoResponse, afterAction: _ => testhost1Process.Exit()) .Build(); var testhost1 = new FakeTestHostFixtureBuilder(fixture) @@ -197,20 +170,24 @@ public async Task GivenMultipleMsTestAssembliesThatUseTheSameTargetFrameworkAndA .WithResponses(runTests1) .Build(); + // -- + var mstest2Dll = new FakeTestDllBuilder() .WithPath(@"X:\fake\mstest2.dll") - .WithFramework(KnownFrameworkNames.Net5) + .WithFramework(KnownFrameworkNames.Net6) // <--- .WithArchitecture(Architecture.X64) - .WithTestCount(50, 8) + .WithTestCount(21, 5) .Build(); var testhost2Process = new FakeProcess(fixture.ErrorAggregator, @"X:\fake\testhost2.exe"); var runTests2 = new FakeTestHostResponsesBuilder() .VersionCheck(5) - .ExecutionInitialize(FakeMessage.NoResponse) - .StartTestExecutionWithSources(mstest2Dll.TestResultBatches) - .SessionEnd(FakeMessage.NoResponse, f => f.Process.Exit()) + .DiscoveryInitialize(FakeMessage.NoResponse) + .StartDiscovery(mstest2Dll.TestResultBatches) + .SessionEnd(FakeMessage.NoResponse, _ => testhost2Process.Exit()) + // We actually do get asked to terminate multiple times. In the second host only. + .SessionEnd(FakeMessage.NoResponse) .Build(); var testhost2 = new FakeTestHostFixtureBuilder(fixture) @@ -223,23 +200,48 @@ public async Task GivenMultipleMsTestAssembliesThatUseTheSameTargetFrameworkAndA var testRequestManager = fixture.BuildTestRequestManager(); + mstest1Dll.FrameworkName.Should().NotBe(mstest2Dll.FrameworkName); + // -- act - var runConfiguration = string.Empty; - var testRunRequestPayload = new TestRunRequestPayload + var testDiscoveryPayload = new DiscoveryRequestPayload { Sources = new List { mstest1Dll.Path, mstest2Dll.Path }, - - RunSettings = $"{runConfiguration}" + RunSettings = $"{KnownFrameworkStrings.Net7}" }; - await testRequestManager.ExecuteWithAbort(tm => tm.RunTests(testRunRequestPayload, testHostLauncher: null, fixture.TestRunEventsRegistrar, fixture.ProtocolConfig)); + await testRequestManager.ExecuteWithAbort(tm => tm.DiscoverTests(testDiscoveryPayload, fixture.TestDiscoveryEventsRegistrar, fixture.ProtocolConfig)); // -- assert fixture.AssertNoErrors(); - fixture.ExecutedTests.Should().HaveCount(mstest1Dll.TestCount + mstest2Dll.TestCount); + // We figure out the framework for each assembly so there should be no incompatibility warnings + // REVIEW: This should be uncommented. Commenting it now, because it is helpful to see those warnings. + // fixture.TestRunEventsRegistrar.LoggedWarnings.Should().NotContainMatch("Test run detected DLL(s) which were built for different framework*"); + + fixture.ProcessHelper.Processes.Where(p => p.Started).Should().HaveCount(2); + var startWithSources1 = testhost1.FakeCommunicationChannel.ProcessedMessages.Single(m => m.Request.MessageType == MessageType.StartDiscovery); + var startWithSources1Text = startWithSources1.Request.Payload.Select(t => t.ToString()).JoinBySpace(); + // We sent mstest1.dll and net7 because that is what we have in settings. + startWithSources1Text.Should().Contain("mstest1.dll"); + startWithSources1Text.Should().Contain(KnownFrameworkStrings.Net7); + + var startWithSources2 = testhost2.FakeCommunicationChannel.ProcessedMessages.Single(m => m.Request.MessageType == MessageType.StartDiscovery); + var startWithSources2Text = startWithSources2.Request.Payload.Select(t => t.ToString()).JoinBySpace(); + // We sent mstest2.dll and net7 because that is what we have in settings. + startWithSources2Text.Should().Contain("mstest2.dll"); + startWithSources2Text.Should().Contain(KnownFrameworkStrings.Net7); + + fixture.DiscoveredTests.Should().HaveCount(mstest1Dll.TestCount + mstest2Dll.TestCount); } - public async Task GivenMultipleMsTestAssembliesThatHaveTheSameArchitecture_AndHaveDifferentTargetFrameworks_AndMULTI_TFM_RUNFeatureFlagIsDisabled_WhenTestsAreRun_ThenTwoTesthostsAreStartedBothForTheSameTFM() + [Test(@" + Given two test assemblies that have the same architecture + but have different target frameworks. + + When we execute tests. + + Then two testhosts should be started that target the same framework as each assembly. + ")] + public async Task C() { // -- arrange using var fixture = new Fixture( @@ -247,7 +249,7 @@ public async Task GivenMultipleMsTestAssembliesThatHaveTheSameArchitecture_AndHa { FeatureFlags = new Dictionary { - [FeatureFlag.MULTI_TFM_RUN] = false + [FeatureFlag.MULTI_TFM_RUN] = true } } ); @@ -256,7 +258,7 @@ public async Task GivenMultipleMsTestAssembliesThatHaveTheSameArchitecture_AndHa .WithPath(@"X:\fake\mstest1.dll") .WithFramework(KnownFrameworkNames.Net5) // <--- .WithArchitecture(Architecture.X64) - .WithTestCount(2) + .WithTestCount(11, 5) .Build(); var testhost1Process = new FakeProcess(fixture.ErrorAggregator, @"X:\fake\testhost1.exe"); @@ -278,11 +280,9 @@ public async Task GivenMultipleMsTestAssembliesThatHaveTheSameArchitecture_AndHa var mstest2Dll = new FakeTestDllBuilder() .WithPath(@"X:\fake\mstest2.dll") - .WithFramework(KnownFrameworkNames.Net48) // <--- + .WithFramework(KnownFrameworkNames.Net6) // <--- .WithArchitecture(Architecture.X64) - // In reality, the dll would fail to load, and no tests would run from this dll, - // we simulate that by making it have 0 tests. - .WithTestCount(0) + .WithTestCount(21, 5) .Build(); var testhost2Process = new FakeProcess(fixture.ErrorAggregator, @"X:\fake\testhost2.exe"); @@ -292,6 +292,8 @@ public async Task GivenMultipleMsTestAssembliesThatHaveTheSameArchitecture_AndHa .ExecutionInitialize(FakeMessage.NoResponse) .StartTestExecutionWithSources(mstest2Dll.TestResultBatches) .SessionEnd(FakeMessage.NoResponse, _ => testhost2Process.Exit()) + // We actually do get asked to terminate multiple times. In the second host only. + .SessionEnd(FakeMessage.NoResponse) .Build(); var testhost2 = new FakeTestHostFixtureBuilder(fixture) @@ -307,44 +309,48 @@ public async Task GivenMultipleMsTestAssembliesThatHaveTheSameArchitecture_AndHa mstest1Dll.FrameworkName.Should().NotBe(mstest2Dll.FrameworkName); // -- act - // TODO: Building whole default runconfiguration is needed here, because TestRequestManager does not ensure the basic settings are populated, - // and all methods that populate them just silently fail, so TestHostProvider does not get any useful settings. - var runConfiguration = new RunConfiguration().ToXml().OuterXml; var testRunRequestPayload = new TestRunRequestPayload { Sources = new List { mstest1Dll.Path, mstest2Dll.Path }, - - RunSettings = $"{runConfiguration}" + RunSettings = $"" }; await testRequestManager.ExecuteWithAbort(tm => tm.RunTests(testRunRequestPayload, testHostLauncher: null, fixture.TestRunEventsRegistrar, fixture.ProtocolConfig)); // -- assert fixture.AssertNoErrors(); - // We unify the frameworks to netcoreapp1.0 (because the vstest.console dll we are loading is built for netcoreapp and prefers netcoreapp), and because the - // behavior is to choose the common oldest framework. We then log warning about incompatible sources. - fixture.TestRunEventsRegistrar.LoggedWarnings.Should().ContainMatch($"Test run detected DLL(s) which were built for different framework and platform versions*{KnownFrameworkNames.Netcoreapp1}*"); + // We figure out the framework for each assembly so there should be no incompatibility warnings + // REVIEW: This should be uncommented. Commenting it now, because it is helpful to see those warnings. + // fixture.TestRunEventsRegistrar.LoggedWarnings.Should().NotContainMatch("Test run detected DLL(s) which were built for different framework*"); - // We started both testhosts, even thought we know one of them is incompatible. fixture.ProcessHelper.Processes.Where(p => p.Started).Should().HaveCount(2); var startWithSources1 = testhost1.FakeCommunicationChannel.ProcessedMessages.Single(m => m.Request.MessageType == MessageType.StartTestExecutionWithSources); var startWithSources1Text = startWithSources1.Request.Payload.Select(t => t.ToString()).JoinBySpace(); - // We sent mstest1.dll + // We sent mstest1.dll. startWithSources1Text.Should().Contain("mstest1.dll"); - // And we sent netcoreapp1.0 as the target framework - startWithSources1Text.Should().Contain(KnownFrameworkStrings.Netcoreapp1); + // And we sent net5 as the target framework, because that is the framework of mstest1.dll. + startWithSources1Text.Should().Contain(KnownFrameworkStrings.Net5); var startWithSources2 = testhost2.FakeCommunicationChannel.ProcessedMessages.Single(m => m.Request.MessageType == MessageType.StartTestExecutionWithSources); var startWithSources2Text = startWithSources2.Request.Payload.Select(t => t.ToString()).JoinBySpace(); - // We sent mstest2.dll + // We sent mstest2.dll. startWithSources2Text.Should().Contain("mstest2.dll"); - // And we sent netcoreapp1.0 as the target framework, even though it is incompatible - startWithSources2Text.Should().Contain(KnownFrameworkStrings.Netcoreapp1); + // And we sent net48 as the target framework, because that is the framework of mstest2.dll. + startWithSources2Text.Should().Contain(KnownFrameworkStrings.Net6); - fixture.ExecutedTests.Should().HaveCount(mstest1Dll.TestCount); + fixture.ExecutedTests.Should().HaveCount(mstest1Dll.TestCount + mstest2Dll.TestCount); } - public async Task GivenMultipleMsTestAssembliesThatHaveTheSameArchitecture_AndHaveDifferentTargetFrameworks_AndMULTI_TFM_RUNFeatureFlagIsEnabled_WhenTestsAreRun_ThenTwoTesthostsWithTheCorrectTFMForTheDllAreUsed() + [Test(@" + Given two test assemblies that have the same architecture + but have different target frameworks. + + When we execute tests + and provide runsettings that define the desired target framework. + + Then two testhosts should be started that target the framework chosen by runsettings. + ")] + public async Task D() { // -- arrange using var fixture = new Fixture( @@ -371,8 +377,6 @@ public async Task GivenMultipleMsTestAssembliesThatHaveTheSameArchitecture_AndHa .ExecutionInitialize(FakeMessage.NoResponse) .StartTestExecutionWithSources(mstest1Dll.TestResultBatches) .SessionEnd(FakeMessage.NoResponse, afterAction: _ => testhost1Process.Exit()) - // We actually do get asked to terminate multiple times. - .SessionEnd(FakeMessage.NoResponse) .Build(); var testhost1 = new FakeTestHostFixtureBuilder(fixture) @@ -385,7 +389,7 @@ public async Task GivenMultipleMsTestAssembliesThatHaveTheSameArchitecture_AndHa var mstest2Dll = new FakeTestDllBuilder() .WithPath(@"X:\fake\mstest2.dll") - .WithFramework(KnownFrameworkNames.Net48) // <--- + .WithFramework(KnownFrameworkNames.Net6) // <--- .WithArchitecture(Architecture.X64) .WithTestCount(21, 5) .Build(); @@ -397,7 +401,7 @@ public async Task GivenMultipleMsTestAssembliesThatHaveTheSameArchitecture_AndHa .ExecutionInitialize(FakeMessage.NoResponse) .StartTestExecutionWithSources(mstest2Dll.TestResultBatches) .SessionEnd(FakeMessage.NoResponse, _ => testhost2Process.Exit()) - // We actually do get asked to terminate multiple times. + // We actually do get asked to terminate multiple times. In the second host only. .SessionEnd(FakeMessage.NoResponse) .Build(); @@ -414,13 +418,10 @@ public async Task GivenMultipleMsTestAssembliesThatHaveTheSameArchitecture_AndHa mstest1Dll.FrameworkName.Should().NotBe(mstest2Dll.FrameworkName); // -- act - // TODO: Building whole default runconfiguration is needed here, because TestRequestManager does not ensure the basic settings are populated, - // and all methods that populate them just silently fail, so TestHostProvider does not get any useful settings. - var runConfiguration = new RunConfiguration().ToXml().OuterXml; var testRunRequestPayload = new TestRunRequestPayload { Sources = new List { mstest1Dll.Path, mstest2Dll.Path }, - RunSettings = $"{runConfiguration}" + RunSettings = $"{KnownFrameworkStrings.Net7}" }; await testRequestManager.ExecuteWithAbort(tm => tm.RunTests(testRunRequestPayload, testHostLauncher: null, fixture.TestRunEventsRegistrar, fixture.ProtocolConfig)); @@ -431,36 +432,35 @@ public async Task GivenMultipleMsTestAssembliesThatHaveTheSameArchitecture_AndHa // REVIEW: This should be uncommented. Commenting it now, because it is helpful to see those warnings. // fixture.TestRunEventsRegistrar.LoggedWarnings.Should().NotContainMatch("Test run detected DLL(s) which were built for different framework*"); - // We started both testhosts, even thought we know one of them is incompatible. fixture.ProcessHelper.Processes.Where(p => p.Started).Should().HaveCount(2); var startWithSources1 = testhost1.FakeCommunicationChannel.ProcessedMessages.Single(m => m.Request.MessageType == MessageType.StartTestExecutionWithSources); var startWithSources1Text = startWithSources1.Request.Payload.Select(t => t.ToString()).JoinBySpace(); // We sent mstest1.dll. startWithSources1Text.Should().Contain("mstest1.dll"); - // And we sent net5 as the target framework, because that is the framework of mstest1.dll. - startWithSources1Text.Should().Contain(KnownFrameworkStrings.Net5); + startWithSources1Text.Should().Contain(KnownFrameworkStrings.Net7); var startWithSources2 = testhost2.FakeCommunicationChannel.ProcessedMessages.Single(m => m.Request.MessageType == MessageType.StartTestExecutionWithSources); var startWithSources2Text = startWithSources2.Request.Payload.Select(t => t.ToString()).JoinBySpace(); // We sent mstest2.dll. startWithSources2Text.Should().Contain("mstest2.dll"); - // And we sent net48 as the target framework, because that is the framework of mstest2.dll. - startWithSources2Text.Should().Contain(mstest2Dll.FrameworkName.ToString()); + startWithSources2Text.Should().Contain(KnownFrameworkStrings.Net7); fixture.ExecutedTests.Should().HaveCount(mstest1Dll.TestCount + mstest2Dll.TestCount); } +} - [Only] - public async Task GivenMultipleMsTestAssembliesThatHaveTheSameArchitecture_AndHaveDifferentTargetFrameworks_AndMULTI_TFM_RUNFeatureFlagIsEnabled_WhenTestsAreDiscovered_ThenTwoTesthostsWithTheCorrectTFMForTheDllAreUsed() +public class MultiTFMRunAndDiscoveryCompatibilityMode +{ + [Exclude] + public async Task GivenMultipleMsTestAssembliesThatHaveTheSameArchitecture_AndHaveDifferentTargetFrameworks_AndMULTI_TFM_RUNFeatureFlagIsDisabled_WhenTestsAreRun_ThenTwoTesthostsAreStartedBothForTheSameTFM() { - EqtTrace.InitializeVerboseTrace(@"C:\temp\log.txt"); // -- arrange using var fixture = new Fixture( new FixtureOptions { FeatureFlags = new Dictionary { - [FeatureFlag.MULTI_TFM_RUN] = true + [FeatureFlag.MULTI_TFM_RUN] = false } } ); @@ -469,17 +469,16 @@ public async Task GivenMultipleMsTestAssembliesThatHaveTheSameArchitecture_AndHa .WithPath(@"X:\fake\mstest1.dll") .WithFramework(KnownFrameworkNames.Net5) // <--- .WithArchitecture(Architecture.X64) - .WithTestCount(11, 5) + .WithTestCount(2) .Build(); var testhost1Process = new FakeProcess(fixture.ErrorAggregator, @"X:\fake\testhost1.exe"); var runTests1 = new FakeTestHostResponsesBuilder() .VersionCheck(5) - .DiscoveryInitialize(FakeMessage.NoResponse) - .StartDiscovery(mstest1Dll.TestResultBatches) + .ExecutionInitialize(FakeMessage.NoResponse) + .StartTestExecutionWithSources(mstest1Dll.TestResultBatches) .SessionEnd(FakeMessage.NoResponse, afterAction: _ => testhost1Process.Exit()) - // We actually do get asked to terminate multiple times. .SessionEnd(FakeMessage.NoResponse) .Build(); @@ -495,17 +494,18 @@ public async Task GivenMultipleMsTestAssembliesThatHaveTheSameArchitecture_AndHa .WithPath(@"X:\fake\mstest2.dll") .WithFramework(KnownFrameworkNames.Net48) // <--- .WithArchitecture(Architecture.X64) - .WithTestCount(21, 5) + // In reality, the dll would fail to load, and no tests would run from this dll, + // we simulate that by making it have 0 tests. + .WithTestCount(0) .Build(); var testhost2Process = new FakeProcess(fixture.ErrorAggregator, @"X:\fake\testhost2.exe"); var runTests2 = new FakeTestHostResponsesBuilder() .VersionCheck(5) - .DiscoveryInitialize(FakeMessage.NoResponse) - .StartDiscovery(mstest2Dll.TestResultBatches) + .ExecutionInitialize(FakeMessage.NoResponse) + .StartTestExecutionWithSources(mstest2Dll.TestResultBatches) .SessionEnd(FakeMessage.NoResponse, _ => testhost2Process.Exit()) - // We actually do get asked to terminate multiple times. .SessionEnd(FakeMessage.NoResponse) .Build(); @@ -525,37 +525,38 @@ public async Task GivenMultipleMsTestAssembliesThatHaveTheSameArchitecture_AndHa // TODO: Building whole default runconfiguration is needed here, because TestRequestManager does not ensure the basic settings are populated, // and all methods that populate them just silently fail, so TestHostProvider does not get any useful settings. var runConfiguration = new RunConfiguration().ToXml().OuterXml; - var testDiscoveryPayload = new DiscoveryRequestPayload + var testRunRequestPayload = new TestRunRequestPayload { Sources = new List { mstest1Dll.Path, mstest2Dll.Path }, + RunSettings = $"{runConfiguration}" }; - await testRequestManager.ExecuteWithAbort(tm => tm.DiscoverTests(testDiscoveryPayload, fixture.TestDiscoveryEventsRegistrar, fixture.ProtocolConfig)); + await testRequestManager.ExecuteWithAbort(tm => tm.RunTests(testRunRequestPayload, testHostLauncher: null, fixture.TestRunEventsRegistrar, fixture.ProtocolConfig)); // -- assert fixture.AssertNoErrors(); - // We figure out the framework for each assembly so there should be no incompatibility warnings - // REVIEW: This should be uncommented. Commenting it now, because it is helpful to see those warnings. - // fixture.TestRunEventsRegistrar.LoggedWarnings.Should().NotContainMatch("Test run detected DLL(s) which were built for different framework*"); + // We unify the frameworks to netcoreapp1.0 (because the vstest.console dll we are loading is built for netcoreapp and prefers netcoreapp), and because the + // behavior is to choose the common oldest framework. We then log warning about incompatible sources. + fixture.TestRunEventsRegistrar.LoggedWarnings.Should().ContainMatch($"Test run detected DLL(s) which were built for different framework and platform versions*{KnownFrameworkNames.Netcoreapp1}*"); // We started both testhosts, even thought we know one of them is incompatible. fixture.ProcessHelper.Processes.Where(p => p.Started).Should().HaveCount(2); - var startWithSources1 = testhost1.FakeCommunicationChannel.ProcessedMessages.Single(m => m.Request.MessageType == MessageType.StartDiscovery); + var startWithSources1 = testhost1.FakeCommunicationChannel.ProcessedMessages.Single(m => m.Request.MessageType == MessageType.StartTestExecutionWithSources); var startWithSources1Text = startWithSources1.Request.Payload.Select(t => t.ToString()).JoinBySpace(); - // We sent mstest1.dll. + // We sent mstest1.dll startWithSources1Text.Should().Contain("mstest1.dll"); - // And we sent net5 as the target framework, because that is the framework of mstest1.dll. - startWithSources1Text.Should().Contain(KnownFrameworkStrings.Net5); + // And we sent netcoreapp1.0 as the target framework + startWithSources1Text.Should().Contain(KnownFrameworkStrings.Netcoreapp1); - var startWithSources2 = testhost2.FakeCommunicationChannel.ProcessedMessages.Single(m => m.Request.MessageType == MessageType.StartDiscovery); + var startWithSources2 = testhost2.FakeCommunicationChannel.ProcessedMessages.Single(m => m.Request.MessageType == MessageType.StartTestExecutionWithSources); var startWithSources2Text = startWithSources2.Request.Payload.Select(t => t.ToString()).JoinBySpace(); - // We sent mstest2.dll. + // We sent mstest2.dll startWithSources2Text.Should().Contain("mstest2.dll"); - // And we sent net48 as the target framework, because that is the framework of mstest2.dll. - startWithSources2Text.Should().Contain(mstest2Dll.FrameworkName.ToString()); + // And we sent netcoreapp1.0 as the target framework, even though it is incompatible + startWithSources2Text.Should().Contain(KnownFrameworkStrings.Netcoreapp1); - fixture.ExecutedTests.Should().HaveCount(mstest1Dll.TestCount + mstest2Dll.TestCount); + fixture.ExecutedTests.Should().HaveCount(mstest1Dll.TestCount); } } From 8ce64aefdb85173d4401dc73f13a177c5b556ad6 Mon Sep 17 00:00:00 2001 From: nohwnd Date: Thu, 24 Feb 2022 16:13:15 +0100 Subject: [PATCH 005/112] First test rewrite. --- .../BasicRunAndDiscovery.cs | 167 ++++-------------- 1 file changed, 30 insertions(+), 137 deletions(-) diff --git a/test/vstest.ProgrammerTests/BasicRunAndDiscovery.cs b/test/vstest.ProgrammerTests/BasicRunAndDiscovery.cs index 339872f12f..8fe5239e91 100644 --- a/test/vstest.ProgrammerTests/BasicRunAndDiscovery.cs +++ b/test/vstest.ProgrammerTests/BasicRunAndDiscovery.cs @@ -3,27 +3,10 @@ namespace vstest.ProgrammerTests; -using System.Diagnostics; -using System.Runtime.Versioning; - using FluentAssertions; -using FluentAssertions.Extensions; - -using Microsoft.VisualStudio.TestPlatform.Client; -using Microsoft.VisualStudio.TestPlatform.CommandLine; -using Microsoft.VisualStudio.TestPlatform.CommandLine.Publisher; -using Microsoft.VisualStudio.TestPlatform.CommandLine.TestPlatformHelpers; -using Microsoft.VisualStudio.TestPlatform.CommandLineUtilities; -using Microsoft.VisualStudio.TestPlatform.CommunicationUtilities.ObjectModel; -using Microsoft.VisualStudio.TestPlatform.CrossPlatEngine; -using Microsoft.VisualStudio.TestPlatform.CrossPlatEngine.TestRunAttachmentsProcessing; using Microsoft.VisualStudio.TestPlatform.ObjectModel; using Microsoft.VisualStudio.TestPlatform.ObjectModel.Client; -# if DEBUG -using Microsoft.VisualStudio.TestPlatform.CommunicationUtilities.Interfaces; -#endif - using vstest.ProgrammerTests.Fakes; using Intent; @@ -43,139 +26,49 @@ Then all 108 tests are executed. public async Task A() { // -- arrange - var fakeErrorAggregator = new FakeErrorAggregator(); - var commandLineOptions = new CommandLineOptions { IsDesignMode = true }; - - var fakeCurrentProcess = new FakeProcess(fakeErrorAggregator, @"X:\fake\vstest.console.exe"); - var fakeProcessHelper = new FakeProcessHelper(fakeErrorAggregator, fakeCurrentProcess); - - var fakeFileHelper = new FakeFileHelper(fakeErrorAggregator); - // TODO: Get framework name from constants - // TODO: have mstest1dll canned - var tests = new FakeTestBatchBuilder() - .WithTotalCount(108) - .WithDuration(100.Milliseconds()) - .WithBatchSize(10) + using var fixture = new Fixture(); + + var mstest1Dll = new FakeTestDllBuilder() + .WithPath(@"X:\fake\mstest1.dll") + .WithFramework(KnownFrameworkNames.Net5) + .WithArchitecture(Architecture.X64) + .WithTestCount(108, 10) .Build(); - var mstest1Dll = new FakeTestDllFile(@"X:\fake\mstest1.dll", new FrameworkName(".NETCoreApp,Version=v5.0"), Architecture.X64, tests); - List changeMessages = tests.Take(tests.Count - 1).Select(batch => // TODO: make the stats agree with the tests below - new FakeMessage(MessageType.TestRunStatsChange, - new TestRunChangedEventArgs(new TestRunStatistics(new Dictionary { [TestOutcome.Passed] = batch.Count }), batch, new List()) - )).ToList(); - FakeMessage completedMessage = new FakeMessage(MessageType.ExecutionComplete, new TestRunCompletePayload - { - // TODO: make the stats agree with the tests below - TestRunCompleteArgs = new TestRunCompleteEventArgs(new TestRunStatistics(new Dictionary { [TestOutcome.Passed] = 1 }), false, false, null, new System.Collections.ObjectModel.Collection(), TimeSpan.Zero), - LastRunTests = new TestRunChangedEventArgs(new TestRunStatistics(new Dictionary { [TestOutcome.Passed] = 1 }), tests.Last(), new List()), - }); - List messages = changeMessages.Concat(new[] { completedMessage }).ToList(); - var responses = new List> { - new RequestResponsePair(MessageType.VersionCheck, new FakeMessage(MessageType.VersionCheck, 5)), - new RequestResponsePair(MessageType.ExecutionInitialize, FakeMessage.NoResponse), - new RequestResponsePair(MessageType.StartTestExecutionWithSources, messages, false), - new RequestResponsePair(MessageType.SessionEnd, new [] { FakeMessage.NoResponse }, message => - { - // TODO: how do we associate this to the correct process? - var fp = fakeProcessHelper.Processes.Last(); - fakeProcessHelper.TerminateProcess(fp); - }), - new RequestResponsePair(MessageType.SessionEnd, FakeMessage.NoResponse ), - }; + var testhost1Process = new FakeProcess(fixture.ErrorAggregator, @"X:\fake\testhost1.exe"); - var fakeCommunicationChannel = new FakeCommunicationChannel(responses, fakeErrorAggregator, 1); - fakeCommunicationChannel.Start(new object()); - var fakeCommunicationEndpoint = new FakeCommunicationEndpoint(fakeCommunicationChannel, fakeErrorAggregator); -#if DEBUG - TestServiceLocator.Clear(); - TestServiceLocator.Register(fakeCommunicationEndpoint.TestHostConnectionInfo.Endpoint, fakeCommunicationEndpoint); -#else - // This fools compiler into not being able to tell that the the rest of the code is unreachable. - var a = true; - if (a) - { - throw new InvalidOperationException("Tests cannot run in Release mode, because TestServiceLocator is compiled only for Debug, and so the tests will fail to setup channel and will hang."); - } -#endif - var fakeTestHostProcess = new FakeProcess(fakeErrorAggregator, @"C:\temp\testhost.exe"); - var fakeTestRuntimeProvider = new FakeTestRuntimeProvider(fakeProcessHelper, fakeTestHostProcess, fakeFileHelper, mstest1Dll.AsList(), fakeCommunicationEndpoint, fakeErrorAggregator); - var fakeTestRuntimeProviderManager = new FakeTestRuntimeProviderManager(fakeErrorAggregator); - fakeTestRuntimeProviderManager.AddTestRuntimeProviders(fakeTestRuntimeProvider); - var testEngine = new TestEngine(fakeTestRuntimeProviderManager, fakeProcessHelper); - - var testPlatform = new TestPlatform(testEngine, fakeFileHelper, fakeTestRuntimeProviderManager); - - var testRunResultAggregator = new TestRunResultAggregator(); - var fakeTestPlatformEventSource = new FakeTestPlatformEventSource(fakeErrorAggregator); - - var fakeAssemblyMetadataProvider = new FakeAssemblyMetadataProvider(fakeFileHelper, fakeErrorAggregator); - var inferHelper = new InferHelper(fakeAssemblyMetadataProvider); - - // This is most likely not the correctl place where to cut this off, plugin cache is probably the better place, - // but it is not injected, and I don't want to investigate this now. - var fakeDataCollectorAttachmentsProcessorsFactory = new FakeDataCollectorAttachmentsProcessorsFactory(fakeErrorAggregator); - var testRunAttachmentsProcessingManager = new TestRunAttachmentsProcessingManager(fakeTestPlatformEventSource, fakeDataCollectorAttachmentsProcessorsFactory); - - Task fakeMetricsPublisherTask = Task.FromResult(new FakeMetricsPublisher(fakeErrorAggregator)); - TestRequestManager testRequestManager = new( - commandLineOptions, - testPlatform, - testRunResultAggregator, - fakeTestPlatformEventSource, - inferHelper, - fakeMetricsPublisherTask, - fakeProcessHelper, - testRunAttachmentsProcessingManager - ); + var runTests1 = new FakeTestHostResponsesBuilder() + .VersionCheck(5) + .ExecutionInitialize(FakeMessage.NoResponse) + .StartTestExecutionWithSources(mstest1Dll.TestResultBatches) + .SessionEnd(FakeMessage.NoResponse, _ => testhost1Process.Exit()) + .SessionEnd(FakeMessage.NoResponse) + .Build(); - // -- act + var testhost1 = new FakeTestHostFixtureBuilder(fixture) + .WithTestDll(mstest1Dll) + .WithProcess(testhost1Process) + .WithResponses(runTests1) + .Build(); + + fixture.AddTestHostFixtures(testhost1); - // TODO: this gives me run configuration that is way too complete, do we a way to generate "bare" runsettings? if not we should add them. Would be also useful to get - // runsettings from parameter set so people can use it - // TODO: TestSessionTimeout gives me way to abort the run without having to cancel it externally, but could probably still lead to hangs if that funtionality is broken - // TODO: few tries later, that is exactly the case when we abort, it still hangs on waiting to complete request, because test run complete was not sent - // var runConfiguration = new Microsoft.VisualStudio.TestPlatform.ObjectModel.RunConfiguration { TestSessionTimeout = 40_000 }.ToXml().OuterXml; + var testRequestManager = fixture.BuildTestRequestManager(); + + // -- act var runConfiguration = string.Empty; var testRunRequestPayload = new TestRunRequestPayload { - // TODO: passing null sources and null testcases does not fail fast - Sources = mstest1Dll.Path.AsList(), - // TODO: passing null runsettings does not fail fast, instead it fails in Fakes settings code - // TODO: passing empty string fails in the xml parser code + Sources = new List { mstest1Dll.Path }, + RunSettings = $"{runConfiguration}" }; - // var fakeTestHostLauncher = new FakeTestHostLauncher(); - var fakeTestRunEventsRegistrar = new FakeTestRunEventsRegistrar(fakeErrorAggregator); - var protocolConfig = new ProtocolConfig(); - - // TODO: we make sure the test is running 10 minutes at max and then we try to abort - // if we aborted we write the error to aggregator, this needs to be made into a pattern - // so we can avoid hanging if the run does not complete correctly - var cancelAbort = new CancellationTokenSource(); - var task = Task.Run(async () => - { - await Task.Delay(TimeSpan.FromSeconds(Debugger.IsAttached ? 100 : 10), cancelAbort.Token); - if (Debugger.IsAttached) - { - // we will abort because we are hanging, look at stacks to see what the problem is - Debugger.Break(); - } - fakeErrorAggregator.Add(new Exception("errr we aborted")); - testRequestManager.AbortTestRun(); - - }); - testRequestManager.RunTests(testRunRequestPayload, testHostLauncher: null, fakeTestRunEventsRegistrar, protocolConfig); - cancelAbort.Cancel(); - if (!task.IsCanceled) - { - await task; - } - // pattern end + await testRequestManager.ExecuteWithAbort(tm => tm.RunTests(testRunRequestPayload, testHostLauncher: null, fixture.TestRunEventsRegistrar, fixture.ProtocolConfig)); // -- assert - fakeErrorAggregator.Errors.Should().BeEmpty(); - fakeTestRunEventsRegistrar.RunChangedEvents.SelectMany(er => er.Data.NewTestResults).Should().HaveCount(108); + fixture.AssertNoErrors(); + fixture.ExecutedTests.Should().HaveCount(mstest1Dll.TestCount); } [Test(@" From 5d2f4ee37481a77ceeffefed12022b33e49b0bc2 Mon Sep 17 00:00:00 2001 From: nohwnd Date: Thu, 24 Feb 2022 17:02:59 +0100 Subject: [PATCH 006/112] test sessions start --- .../Fakes/FakeTestSessionEventsHandler.cs | 50 ++++++++ test/vstest.ProgrammerTests/Fakes/Fixture.cs | 5 +- .../Fakes/TestMessage.cs | 18 +++ .../Fakes/TestRequestManagerHelper.cs | 4 +- .../MultiTFMRunAndDiscovery.cs | 118 +++++++++++++++++- 5 files changed, 191 insertions(+), 4 deletions(-) create mode 100644 test/vstest.ProgrammerTests/Fakes/FakeTestSessionEventsHandler.cs create mode 100644 test/vstest.ProgrammerTests/Fakes/TestMessage.cs diff --git a/test/vstest.ProgrammerTests/Fakes/FakeTestSessionEventsHandler.cs b/test/vstest.ProgrammerTests/Fakes/FakeTestSessionEventsHandler.cs new file mode 100644 index 0000000000..142e057ebb --- /dev/null +++ b/test/vstest.ProgrammerTests/Fakes/FakeTestSessionEventsHandler.cs @@ -0,0 +1,50 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +namespace vstest.ProgrammerTests.Fakes; + +using System.Diagnostics.Contracts; + +using Microsoft.VisualStudio.TestPlatform.ObjectModel.Client; +using Microsoft.VisualStudio.TestPlatform.ObjectModel.Logging; + +internal class FakeTestSessionEventsHandler : ITestSessionEventsHandler +{ + private readonly FakeErrorAggregator _fakeErrorAggregator; + + public FakeTestSessionEventsHandler (FakeErrorAggregator fakeErrorAggregator) + { + _fakeErrorAggregator = fakeErrorAggregator; + } + + public List AllEvents { get; } = new(); + public List LoggedMessages { get; } = new(); + public List RawMessages { get; } = new(); + public List StartTestSessionCompleteEvents { get; } = new(); + public List StopTestSessionCompleteEvents { get; } = new(); + + public void HandleLogMessage(TestMessageLevel level, string message) + { + var msg = new TestMessage(level, message); + AllEvents.Add(msg); + LoggedMessages.Add(msg); + } + + public void HandleRawMessage(string rawMessage) + { + AllEvents.Add(rawMessage); + RawMessages.Add(rawMessage); + } + + public void HandleStartTestSessionComplete(StartTestSessionCompleteEventArgs eventArgs) + { + AllEvents.Add(eventArgs); + StartTestSessionCompleteEvents.Add(eventArgs); + } + + public void HandleStopTestSessionComplete(StopTestSessionCompleteEventArgs eventArgs) + { + AllEvents.Add(eventArgs); + StopTestSessionCompleteEvents.Add(eventArgs); + } +} diff --git a/test/vstest.ProgrammerTests/Fakes/Fixture.cs b/test/vstest.ProgrammerTests/Fakes/Fixture.cs index 563568a0b5..835a4a903a 100644 --- a/test/vstest.ProgrammerTests/Fakes/Fixture.cs +++ b/test/vstest.ProgrammerTests/Fakes/Fixture.cs @@ -42,6 +42,8 @@ internal class Fixture : IDisposable public List ExecutedTests => TestRunEventsRegistrar.RunChangedEvents.SelectMany(er => er.Data.NewTestResults).ToList(); public List DiscoveredTests => TestDiscoveryEventsRegistrar.DiscoveredTestsEvents.SelectMany(er => er.Data.DiscoveredTestCases).ToList(); + public ITestSessionEventsHandler TestSessionEventsHandler { get; } + public Fixture(FixtureOptions? fixtureOptions = null) { // This type is compiled only in DEBUG, and won't exist otherwise. @@ -67,12 +69,13 @@ public Fixture(FixtureOptions? fixtureOptions = null) LogName = Path.GetTempPath() + $"/log_{Guid.NewGuid()}.txt"; //EqtTrace.InitializeVerboseTrace(LogName); - CurrentProcess = new FakeProcess(ErrorAggregator, @"X:\fake\vstest.console.exe", string.Empty, null, null, null, null, null); + CurrentProcess = new FakeProcess(ErrorAggregator, @"X:\fake\vstest.console.exe"); ProcessHelper = new FakeProcessHelper(ErrorAggregator, CurrentProcess); FileHelper = new FakeFileHelper(ErrorAggregator); TestRuntimeProviderManager = new FakeTestRuntimeProviderManager(ErrorAggregator); TestRunEventsRegistrar = new FakeTestRunEventsRegistrar(ErrorAggregator); TestDiscoveryEventsRegistrar = new FakeTestDiscoveryEventsRegistrar(ErrorAggregator); + TestSessionEventsHandler = new FakeTestSessionEventsHandler(ErrorAggregator); ProtocolConfig = new ProtocolConfig(); } diff --git a/test/vstest.ProgrammerTests/Fakes/TestMessage.cs b/test/vstest.ProgrammerTests/Fakes/TestMessage.cs new file mode 100644 index 0000000000..7cf02e33ac --- /dev/null +++ b/test/vstest.ProgrammerTests/Fakes/TestMessage.cs @@ -0,0 +1,18 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +namespace vstest.ProgrammerTests.Fakes; + +using Microsoft.VisualStudio.TestPlatform.ObjectModel.Logging; + +internal class TestMessage +{ + public TestMessage(TestMessageLevel level, string message) + { + Level = level; + Message = message; + } + + public TestMessageLevel Level { get; } + public string Message { get; } +} \ No newline at end of file diff --git a/test/vstest.ProgrammerTests/Fakes/TestRequestManagerHelper.cs b/test/vstest.ProgrammerTests/Fakes/TestRequestManagerHelper.cs index 9eab05998d..0b79d1702a 100644 --- a/test/vstest.ProgrammerTests/Fakes/TestRequestManagerHelper.cs +++ b/test/vstest.ProgrammerTests/Fakes/TestRequestManagerHelper.cs @@ -20,7 +20,7 @@ public TestRequestManagerTestHelper(FakeErrorAggregator errorAggregator, TestReq _debugOptions = debugOptions; } - public async Task ExecuteWithAbort(Action testRequsestManagerAction) + public async Task ExecuteWithAbort(Action testRequestManagerAction) { // We make sure the test is running for the timeout time at max and then we try to abort // if we aborted we write the error to aggregator @@ -54,7 +54,7 @@ public async Task ExecuteWithAbort(Action testRequsestManage } }); - testRequsestManagerAction(_testRequestManager); + testRequestManagerAction(_testRequestManager); cancelAbort.Cancel(); if (!abortOnTimeout.IsCanceled) diff --git a/test/vstest.ProgrammerTests/MultiTFMRunAndDiscovery.cs b/test/vstest.ProgrammerTests/MultiTFMRunAndDiscovery.cs index 2b93b3e141..d2dd9f1b1d 100644 --- a/test/vstest.ProgrammerTests/MultiTFMRunAndDiscovery.cs +++ b/test/vstest.ProgrammerTests/MultiTFMRunAndDiscovery.cs @@ -14,7 +14,7 @@ namespace vstest.ProgrammerTests; using vstest.ProgrammerTests.Fakes; using Microsoft.VisualStudio.TestPlatform.Utilities; using Intent; - +using Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.Payloads; public class MultiTFMRunAndDiscovery { @@ -447,6 +447,122 @@ public async Task D() fixture.ExecutedTests.Should().HaveCount(mstest1Dll.TestCount + mstest2Dll.TestCount); } + + [Test(@" + Given two test assemblies that have the same architecture + but have different target frameworks. + + When we execute tests + and provide runsettings that define the desired target framework. + + Then two testhosts should be started that target the framework chosen by runsettings. + ")][Only] + public async Task E() + { + // -- arrange + using var fixture = new Fixture( + new FixtureOptions + { + FeatureFlags = new Dictionary + { + [FeatureFlag.MULTI_TFM_RUN] = true + } + } + ); + + var mstest1Dll = new FakeTestDllBuilder() + .WithPath(@"X:\fake\mstest1.dll") + .WithFramework(KnownFrameworkNames.Net5) // <--- + .WithArchitecture(Architecture.X64) + .WithTestCount(11, 5) + .Build(); + + var testhost1Process = new FakeProcess(fixture.ErrorAggregator, @"X:\fake\testhost1.exe"); + + var runTests1 = new FakeTestHostResponsesBuilder() + .VersionCheck(5) + .ExecutionInitialize(FakeMessage.NoResponse) + .StartTestExecutionWithSources(mstest1Dll.TestResultBatches) + .SessionEnd(FakeMessage.NoResponse, afterAction: _ => testhost1Process.Exit()) + .Build(); + + var testhost1 = new FakeTestHostFixtureBuilder(fixture) + .WithTestDll(mstest1Dll) + .WithProcess(testhost1Process) + .WithResponses(runTests1) + .Build(); + + // -- + + var mstest2Dll = new FakeTestDllBuilder() + .WithPath(@"X:\fake\mstest2.dll") + .WithFramework(KnownFrameworkNames.Net6) // <--- + .WithArchitecture(Architecture.X64) + .WithTestCount(21, 5) + .Build(); + + var testhost2Process = new FakeProcess(fixture.ErrorAggregator, @"X:\fake\testhost2.exe"); + + var runTests2 = new FakeTestHostResponsesBuilder() + .VersionCheck(5) + .ExecutionInitialize(FakeMessage.NoResponse) + .StartTestExecutionWithSources(mstest2Dll.TestResultBatches) + .SessionEnd(FakeMessage.NoResponse, _ => testhost2Process.Exit()) + // We actually do get asked to terminate multiple times. In the second host only. + .SessionEnd(FakeMessage.NoResponse) + .Build(); + + var testhost2 = new FakeTestHostFixtureBuilder(fixture) + .WithTestDll(mstest2Dll) + .WithProcess(testhost2Process) + .WithResponses(runTests2) + .Build(); + + fixture.AddTestHostFixtures(testhost1, testhost2); + + var testRequestManager = fixture.BuildTestRequestManager(); + + mstest1Dll.FrameworkName.Should().NotBe(mstest2Dll.FrameworkName); + + // -- act + + var startTestSessionPayload = new StartTestSessionPayload + { + RunSettings = "", + Sources = new []{ mstest1Dll.Path, mstest2Dll.Path } + }; + + await testRequestManager.ExecuteWithAbort(tm => tm.StartTestSession(startTestSessionPayload, testHostLauncher: null, fixture.TestSessionEventsHandler, fixture.ProtocolConfig)); + + var testRunRequestPayload = new TestRunRequestPayload + { + Sources = new List { mstest1Dll.Path, mstest2Dll.Path }, + RunSettings = $"{KnownFrameworkStrings.Net7}" + }; + + await testRequestManager.ExecuteWithAbort(tm => tm.RunTests(testRunRequestPayload, testHostLauncher: null, fixture.TestRunEventsRegistrar, fixture.ProtocolConfig)); + + // -- assert + fixture.AssertNoErrors(); + // We figure out the framework for each assembly so there should be no incompatibility warnings + // REVIEW: This should be uncommented. Commenting it now, because it is helpful to see those warnings. + // fixture.TestRunEventsRegistrar.LoggedWarnings.Should().NotContainMatch("Test run detected DLL(s) which were built for different framework*"); + + fixture.ProcessHelper.Processes.Where(p => p.Started).Should().HaveCount(2); + var startWithSources1 = testhost1.FakeCommunicationChannel.ProcessedMessages.Single(m => m.Request.MessageType == MessageType.StartTestExecutionWithSources); + var startWithSources1Text = startWithSources1.Request.Payload.Select(t => t.ToString()).JoinBySpace(); + // We sent mstest1.dll. + startWithSources1Text.Should().Contain("mstest1.dll"); + startWithSources1Text.Should().Contain(KnownFrameworkStrings.Net7); + + var startWithSources2 = testhost2.FakeCommunicationChannel.ProcessedMessages.Single(m => m.Request.MessageType == MessageType.StartTestExecutionWithSources); + var startWithSources2Text = startWithSources2.Request.Payload.Select(t => t.ToString()).JoinBySpace(); + // We sent mstest2.dll. + startWithSources2Text.Should().Contain("mstest2.dll"); + startWithSources2Text.Should().Contain(KnownFrameworkStrings.Net7); + + fixture.ExecutedTests.Should().HaveCount(mstest1Dll.TestCount + mstest2Dll.TestCount); + } } public class MultiTFMRunAndDiscoveryCompatibilityMode From 8ec7a913b1e057d6a71c815ef2c010e882ff4f93 Mon Sep 17 00:00:00 2001 From: nohwnd Date: Fri, 25 Feb 2022 15:08:06 +0100 Subject: [PATCH 007/112] Manually test console mode a little bit --- playground/MSTest1/MSTest1.csproj | 3 ++- playground/MSTest1/UnitTest1.cs | 3 +++ playground/TestPlatform.Playground/Program.cs | 25 ++++++++++++++++--- .../Properties/launchSettings.json | 2 +- .../TestPlatform.Playground.csproj | 2 +- src/AttachVS/AttachVs.cs | 8 ++++++ .../TestPlatform.cs | 4 ++- .../Engine/ClientProtocol/ITestEngine.cs | 3 ++- .../Parallel/ParallelProxyDiscoveryManager.cs | 2 +- .../Parallel/ParallelProxyExecutionManager.cs | 5 ++-- .../TestEngine.cs | 24 ++++++++++++++---- .../DebuggerBreakpoint.cs | 4 +-- 12 files changed, 67 insertions(+), 18 deletions(-) diff --git a/playground/MSTest1/MSTest1.csproj b/playground/MSTest1/MSTest1.csproj index e952c0fa11..012d7dcf44 100644 --- a/playground/MSTest1/MSTest1.csproj +++ b/playground/MSTest1/MSTest1.csproj @@ -6,9 +6,10 @@ - $(TargetFrameworks);net472 + $(TargetFrameworks);net472;net5.0 $(TargetFrameworks);net451 + false false diff --git a/playground/MSTest1/UnitTest1.cs b/playground/MSTest1/UnitTest1.cs index f24dc1cf10..9b303150c6 100644 --- a/playground/MSTest1/UnitTest1.cs +++ b/playground/MSTest1/UnitTest1.cs @@ -5,6 +5,8 @@ namespace MSTest1; +using System.Threading; + using Microsoft.VisualStudio.TestTools.UnitTesting; [TestClass] @@ -13,5 +15,6 @@ public class UnitTest1 [TestMethod] public void TestMethod1() { + // Thread.Sleep(1000); } } diff --git a/playground/TestPlatform.Playground/Program.cs b/playground/TestPlatform.Playground/Program.cs index 328d97e44d..d856b26151 100644 --- a/playground/TestPlatform.Playground/Program.cs +++ b/playground/TestPlatform.Playground/Program.cs @@ -51,15 +51,34 @@ static void Main(string[] args) true + 4 "; + var sources = new[] { - Path.Combine(playground, "MSTest1", "bin", "Debug", "net472", "MSTest1.dll") + Path.Combine(playground, "MSTest1", "bin", "Debug", "net472", "MSTest1.dll"), + Path.Combine(playground, "MSTest1", "bin", "Debug", "net5.0", "MSTest1.dll"), + @"C:\Users\jajares\source\repos\TestProject48\TestProject48\bin\Debug\net48\TestProject48.dll", + @"C:\Users\jajares\source\repos\TestProject48\TestProject1\bin\x64\Debug\net48\win10-x64\TestProject1.dll" }; - var options = new TestPlatformOptions(); - r.RunTestsWithCustomTestHost(sources, sourceSettings, options, new TestRunHandler(), new DebuggerTestHostLauncher()); + // console mode + var settingsFile = Path.GetTempFileName(); + try + { + + File.WriteAllText(settingsFile, sourceSettings); + Process.Start(console, string.Join(" ", sources) + " --settings:" + settingsFile).WaitForExit(); + } + finally + { + try { File.Delete(settingsFile); } catch { } + } + + // design mode + // var options = new TestPlatformOptions(); + // r.RunTestsWithCustomTestHost(sources, sourceSettings, options, new TestRunHandler(), new DebuggerTestHostLauncher()); } public class TestRunHandler : ITestRunEventsHandler diff --git a/playground/TestPlatform.Playground/Properties/launchSettings.json b/playground/TestPlatform.Playground/Properties/launchSettings.json index c688670d9c..8f51ff3b0e 100644 --- a/playground/TestPlatform.Playground/Properties/launchSettings.json +++ b/playground/TestPlatform.Playground/Properties/launchSettings.json @@ -6,7 +6,7 @@ "VSTEST_CONNECTION_TIMEOUT": "999", "VSTEST_DEBUG_NOBP": "1", "VSTEST_RUNNER_DEBUG_ATTACHVS": "1", - "VSTEST_HOST_DEBUG_ATTACHVS": "1", + "VSTEST_HOST_DEBUG_ATTACHVS": "0", "VSTEST_DATACOLLECTOR_DEBUG_ATTACHVS": "1" } } diff --git a/playground/TestPlatform.Playground/TestPlatform.Playground.csproj b/playground/TestPlatform.Playground/TestPlatform.Playground.csproj index d73746f16e..3d54d46a81 100644 --- a/playground/TestPlatform.Playground/TestPlatform.Playground.csproj +++ b/playground/TestPlatform.Playground/TestPlatform.Playground.csproj @@ -39,7 +39,7 @@ - + diff --git a/src/AttachVS/AttachVs.cs b/src/AttachVS/AttachVs.cs index f7a4f6d0d3..5d9419d083 100644 --- a/src/AttachVS/AttachVs.cs +++ b/src/AttachVS/AttachVs.cs @@ -143,6 +143,14 @@ private static bool AttachVs(Process vs, int pid) Trace($"ComException: Retrying in 250ms.\n{ex}"); Thread.Sleep(250); } + catch (TargetInvocationException ex) + { + if (ex.InnerException is not COMException) + throw; + + Trace($"ComException: Retrying in 250ms.\n{ex}"); + Thread.Sleep(250); + } } Marshal.ReleaseComObject(moniker[0]); diff --git a/src/Microsoft.TestPlatform.Client/TestPlatform.cs b/src/Microsoft.TestPlatform.Client/TestPlatform.cs index 70a8452dbb..29173b77bf 100644 --- a/src/Microsoft.TestPlatform.Client/TestPlatform.cs +++ b/src/Microsoft.TestPlatform.Client/TestPlatform.cs @@ -136,7 +136,9 @@ public bool StartTestSession( return false; } - IProxyTestSessionManager testSessionManager = TestEngine.GetTestSessionManager(requestData, testSessionCriteria); + // REVIEW: TODO: NOMERGE: figure out the details of each source + Dictionary sourceToSourceDetailMap = new(); + IProxyTestSessionManager testSessionManager = TestEngine.GetTestSessionManager(requestData, testSessionCriteria, sourceToSourceDetailMap); if (testSessionManager == null) { // The test session manager is null because the combination of runsettings and diff --git a/src/Microsoft.TestPlatform.Common/Interfaces/Engine/ClientProtocol/ITestEngine.cs b/src/Microsoft.TestPlatform.Common/Interfaces/Engine/ClientProtocol/ITestEngine.cs index e7fc4e340d..af9e719f3b 100644 --- a/src/Microsoft.TestPlatform.Common/Interfaces/Engine/ClientProtocol/ITestEngine.cs +++ b/src/Microsoft.TestPlatform.Common/Interfaces/Engine/ClientProtocol/ITestEngine.cs @@ -65,7 +65,8 @@ IProxyExecutionManager GetExecutionManager( /// An IProxyTestSessionManager object that can manage test sessions. IProxyTestSessionManager GetTestSessionManager( IRequestData requestData, - StartTestSessionCriteria testSessionCriteria); + StartTestSessionCriteria testSessionCriteria, + Dictionary sourceToSourceDetailMap); /// /// Fetches the extension manager for this engine. This manager would provide extensibility diff --git a/src/Microsoft.TestPlatform.CrossPlatEngine/Client/Parallel/ParallelProxyDiscoveryManager.cs b/src/Microsoft.TestPlatform.CrossPlatEngine/Client/Parallel/ParallelProxyDiscoveryManager.cs index 0d4198f334..e68df9e95f 100644 --- a/src/Microsoft.TestPlatform.CrossPlatEngine/Client/Parallel/ParallelProxyDiscoveryManager.cs +++ b/src/Microsoft.TestPlatform.CrossPlatEngine/Client/Parallel/ParallelProxyDiscoveryManager.cs @@ -202,7 +202,7 @@ private void DiscoverTestsPrivate(ITestDiscoveryEventsHandler2 discoveryEventsHa // ERRR: GetConcurrentManagerInstances() was called before that touches proxyOperationManager, not CreateNewConcurrentManager directly // is this still compatible? I guess I am skipping the testhost pool? Or am I getting the manager from the pool via a creator? var parallel = 0; - while (parallel <= MaxParallelLevel) + while (parallel < MaxParallelLevel) { parallel++; if (!TryFetchNextSource(_sourceEnumerator, out string source)) diff --git a/src/Microsoft.TestPlatform.CrossPlatEngine/Client/Parallel/ParallelProxyExecutionManager.cs b/src/Microsoft.TestPlatform.CrossPlatEngine/Client/Parallel/ParallelProxyExecutionManager.cs index 3f6695f862..25c6a89cee 100644 --- a/src/Microsoft.TestPlatform.CrossPlatEngine/Client/Parallel/ParallelProxyExecutionManager.cs +++ b/src/Microsoft.TestPlatform.CrossPlatEngine/Client/Parallel/ParallelProxyExecutionManager.cs @@ -203,7 +203,8 @@ public bool HandlePartialRunComplete( var SharedHosts = false; EqtTrace.Verbose("ParallelProxyExecutionManager: HandlePartialRunComplete: Replace execution manager. Shared: {0}, Aborted: {1}.", SharedHosts, testRunCompleteArgs.IsAborted); - // TODO: this should not always happen, this should happen only if we are going to the next non-shared host + // TODO: this should not always happen, this should happen only if we are going to the next non-shared host, but is has been like this forever, so be cautions when changing it. + // Maybe it is not as safe to re-use host for execution. RemoveManager(proxyExecutionManager); TestRunCriteria testRunCriteria = null; @@ -267,7 +268,7 @@ private int StartTestRunPrivate(ITestRunEventsHandler runEventsHandler) // preference. Originally this was done in the UpdateParallelLevel in base constructor, but we did not know which source will be the next there. So we could not // start the manager specifically for the source with the correct tfm and platform. var parallel = 0; - while (parallel <= MaxParallelLevel) + while (parallel < MaxParallelLevel) { parallel++; TestRunCriteria testRunCriteria = null; diff --git a/src/Microsoft.TestPlatform.CrossPlatEngine/TestEngine.cs b/src/Microsoft.TestPlatform.CrossPlatEngine/TestEngine.cs index 83b1e83fb5..af2d1960b4 100644 --- a/src/Microsoft.TestPlatform.CrossPlatEngine/TestEngine.cs +++ b/src/Microsoft.TestPlatform.CrossPlatEngine/TestEngine.cs @@ -82,7 +82,7 @@ public IProxyDiscoveryManager GetDiscoveryManager( // This is a big if that figures out if we can run in process. In process run is very restricted, it is non-parallel run // that has the same target framework as the current process, and it also must not be running in DesignMode (server mode / under IDE) // and more conditions. In all other cases we run in a separate testhost process. - if (ShouldRunInProcess(discoveryCriteria.RunSettings, isParallelRun, isDataCollectorEnabled: false)) + if (ShouldRunInProcess(discoveryCriteria.RunSettings, isParallelRun, isDataCollectorEnabled: false, sourceToSourceDetailMap)) { // Get testhost manager by configuration, and either use it for in-process run. or for single source run. // If there are more sources throw it away. We cannot avoid creating this manager because we don't know if it will be shared or not @@ -190,7 +190,8 @@ public IProxyExecutionManager GetExecutionManager( if (ShouldRunInProcess( testRunCriteria.TestRunSettings, isParallelRun, - isDataCollectorEnabled || isInProcDataCollectorEnabled)) + isDataCollectorEnabled || isInProcDataCollectorEnabled, + sourceToSourceDetailMap)) { // Not updating runsettings from source detail on purpose here. We are running in process, so whatever the settings we figured out at the start. They must be compatible // with the current process, otherwise we would not be able to run inside of the current process. @@ -298,7 +299,8 @@ public IProxyExecutionManager GetExecutionManager( /// public IProxyTestSessionManager GetTestSessionManager( IRequestData requestData, - StartTestSessionCriteria testSessionCriteria) + StartTestSessionCriteria testSessionCriteria, + Dictionary sourceToSourceDetailMap) { var parallelLevel = VerifyParallelSettingAndCalculateParallelLevel( testSessionCriteria.Sources.Count, @@ -314,7 +316,8 @@ public IProxyTestSessionManager GetTestSessionManager( if (ShouldRunInProcess( testSessionCriteria.RunSettings, parallelLevel > 1, - isDataCollectorEnabled || isInProcDataCollectorEnabled)) + isDataCollectorEnabled || isInProcDataCollectorEnabled, + sourceToSourceDetailMap)) { // This condition is the equivalent of the in-process proxy execution manager case. // In this case all tests will be run in the vstest.console process, so there's no @@ -467,8 +470,19 @@ private int VerifyParallelSettingAndCalculateParallelLevel( private bool ShouldRunInProcess( string runsettings, bool isParallelEnabled, - bool isDataCollectorEnabled) + bool isDataCollectorEnabled, + Dictionary sourceToSourceDetailMap) { + var allUseSameFramework = sourceToSourceDetailMap.Values.Select(detail => detail.Framework).Distinct().Count() == 1; + var allUseSameArchitecture = sourceToSourceDetailMap.Values.Select(detail => detail.Architecture).Distinct().Count() == 1; + var isMixedArchitectureOrFrameworkRun = !allUseSameFramework || !allUseSameArchitecture; + + if (isMixedArchitectureOrFrameworkRun) + { + EqtTrace.Info("TestEngine.ShouldRunInNoIsolation: This is a run that mixes sources that have different architectures or frameworks, running in isolation (in a separate testhost proces)."); + return false; + } + var runConfiguration = XmlRunSettingsUtilities.GetRunConfigurationNode(runsettings); if (runConfiguration.InIsolation) diff --git a/src/Microsoft.TestPlatform.Execution.Shared/DebuggerBreakpoint.cs b/src/Microsoft.TestPlatform.Execution.Shared/DebuggerBreakpoint.cs index db33a4c3de..ad3031f0a5 100644 --- a/src/Microsoft.TestPlatform.Execution.Shared/DebuggerBreakpoint.cs +++ b/src/Microsoft.TestPlatform.Execution.Shared/DebuggerBreakpoint.cs @@ -46,11 +46,11 @@ internal static void AttachVisualStudioDebugger(string environmentVariable) if (vsPid == null) { - ConsoleOutput.Instance.WriteLine("Attaching Visual Studio, either a parent or the one that was started first... To specify a VS instance to use, use the PID in the option, instead of 1. No breakpoints are automatically set.", OutputLevel.Information); + ConsoleOutput.Instance.WriteLine("Attaching Visual Studio, either a parent or the one that was started first... To specify a VS instance to use, use the PID in the option, instead of 1.", OutputLevel.Information); } else { - ConsoleOutput.Instance.WriteLine($"Attaching Visual Studio with PID {vsPid} to the process '{Process.GetCurrentProcess().ProcessName}({Process.GetCurrentProcess().Id})'... No breakpoints are automatically set.", OutputLevel.Information); + ConsoleOutput.Instance.WriteLine($"Attaching Visual Studio with PID {vsPid} to the process '{Process.GetCurrentProcess().ProcessName}({Process.GetCurrentProcess().Id})'...", OutputLevel.Information); } AttachVs(Process.GetCurrentProcess(), vsPid); From 21eee179368c2c8b7314afa01accd290a0f64652 Mon Sep 17 00:00:00 2001 From: nohwnd Date: Fri, 25 Feb 2022 15:25:36 +0100 Subject: [PATCH 008/112] Add nested types and split up --- test/Intent/Extensions.cs | 2 +- .../BasicRunAndDiscovery.cs | 6 +- .../MultiTFMRunAndDiscovery.cs | 989 +++++++++--------- 3 files changed, 503 insertions(+), 494 deletions(-) diff --git a/test/Intent/Extensions.cs b/test/Intent/Extensions.cs index 78a9bef318..b06cb5e259 100644 --- a/test/Intent/Extensions.cs +++ b/test/Intent/Extensions.cs @@ -19,7 +19,7 @@ public static List SkipExcluded(this IEnumerable e) public static List SkipNonPublic(this IEnumerable e) { - return e.Where(i => i.IsPublic).ToList(); + return e.Where(i => i.IsPublic || i.IsNestedPublic).ToList(); } public static List SkipExcluded(this IEnumerable e) diff --git a/test/vstest.ProgrammerTests/BasicRunAndDiscovery.cs b/test/vstest.ProgrammerTests/BasicRunAndDiscovery.cs index 8fe5239e91..c6ea10ecf8 100644 --- a/test/vstest.ProgrammerTests/BasicRunAndDiscovery.cs +++ b/test/vstest.ProgrammerTests/BasicRunAndDiscovery.cs @@ -56,12 +56,11 @@ public async Task A() var testRequestManager = fixture.BuildTestRequestManager(); // -- act - var runConfiguration = string.Empty; var testRunRequestPayload = new TestRunRequestPayload { Sources = new List { mstest1Dll.Path }, - RunSettings = $"{runConfiguration}" + RunSettings = $"" }; await testRequestManager.ExecuteWithAbort(tm => tm.RunTests(testRunRequestPayload, testHostLauncher: null, fixture.TestRunEventsRegistrar, fixture.ProtocolConfig)); @@ -134,12 +133,11 @@ public async Task B() var testRequestManager = fixture.BuildTestRequestManager(); // -- act - var runConfiguration = string.Empty; var testRunRequestPayload = new TestRunRequestPayload { Sources = new List { mstest1Dll.Path, mstest2Dll.Path }, - RunSettings = $"{runConfiguration}" + RunSettings = $"" }; await testRequestManager.ExecuteWithAbort(tm => tm.RunTests(testRunRequestPayload, testHostLauncher: null, fixture.TestRunEventsRegistrar, fixture.ProtocolConfig)); diff --git a/test/vstest.ProgrammerTests/MultiTFMRunAndDiscovery.cs b/test/vstest.ProgrammerTests/MultiTFMRunAndDiscovery.cs index d2dd9f1b1d..8a5b817b47 100644 --- a/test/vstest.ProgrammerTests/MultiTFMRunAndDiscovery.cs +++ b/test/vstest.ProgrammerTests/MultiTFMRunAndDiscovery.cs @@ -16,9 +16,12 @@ namespace vstest.ProgrammerTests; using Intent; using Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.Payloads; -public class MultiTFMRunAndDiscovery +public class MultiTFM { - [Test(@" + public class MultiTFMDiscovery + { + + [Test(@" Given two test assemblies that have the same architecture but have different target frameworks. @@ -26,107 +29,107 @@ When we run test discovery. Then two testhosts should be started that target the same framework as each assembly. ")] - public async Task A() - { - // -- arrange - using var fixture = new Fixture( - new FixtureOptions - { - FeatureFlags = new Dictionary + public async Task A() + { + // -- arrange + using var fixture = new Fixture( + new FixtureOptions { - [FeatureFlag.MULTI_TFM_RUN] = true + FeatureFlags = new Dictionary + { + [FeatureFlag.MULTI_TFM_RUN] = true + } } - } - ); - - var mstest1Dll = new FakeTestDllBuilder() - .WithPath(@"X:\fake\mstest1.dll") - .WithFramework(KnownFrameworkNames.Net5) // <--- - .WithArchitecture(Architecture.X64) - .WithTestCount(11, 5) - .Build(); - - var testhost1Process = new FakeProcess(fixture.ErrorAggregator, @"X:\fake\testhost1.exe"); - - var runTests1 = new FakeTestHostResponsesBuilder() - .VersionCheck(5) - .DiscoveryInitialize(FakeMessage.NoResponse) - .StartDiscovery(mstest1Dll.TestResultBatches) - .SessionEnd(FakeMessage.NoResponse, afterAction: _ => testhost1Process.Exit()) - .Build(); - - var testhost1 = new FakeTestHostFixtureBuilder(fixture) - .WithTestDll(mstest1Dll) - .WithProcess(testhost1Process) - .WithResponses(runTests1) - .Build(); - - // -- - - var mstest2Dll = new FakeTestDllBuilder() - .WithPath(@"X:\fake\mstest2.dll") - .WithFramework(KnownFrameworkNames.Net48) // <--- - .WithArchitecture(Architecture.X64) - .WithTestCount(21, 5) - .Build(); - - var testhost2Process = new FakeProcess(fixture.ErrorAggregator, @"X:\fake\testhost2.exe"); - - var runTests2 = new FakeTestHostResponsesBuilder() - .VersionCheck(5) - .DiscoveryInitialize(FakeMessage.NoResponse) - .StartDiscovery(mstest2Dll.TestResultBatches) - .SessionEnd(FakeMessage.NoResponse, _ => testhost2Process.Exit()) - // We actually do get asked to terminate multiple times. In the second host only. - .SessionEnd(FakeMessage.NoResponse) - .Build(); - - var testhost2 = new FakeTestHostFixtureBuilder(fixture) - .WithTestDll(mstest2Dll) - .WithProcess(testhost2Process) - .WithResponses(runTests2) - .Build(); - - fixture.AddTestHostFixtures(testhost1, testhost2); - - var testRequestManager = fixture.BuildTestRequestManager(); - - mstest1Dll.FrameworkName.Should().NotBe(mstest2Dll.FrameworkName); - - // -- act - var testDiscoveryPayload = new DiscoveryRequestPayload - { - Sources = new List { mstest1Dll.Path, mstest2Dll.Path }, - RunSettings = $"" - }; - - await testRequestManager.ExecuteWithAbort(tm => tm.DiscoverTests(testDiscoveryPayload, fixture.TestDiscoveryEventsRegistrar, fixture.ProtocolConfig)); - - // -- assert - fixture.AssertNoErrors(); - // We figure out the framework for each assembly so there should be no incompatibility warnings - // REVIEW: This should be uncommented. Commenting it now, because it is helpful to see those warnings. - // fixture.TestRunEventsRegistrar.LoggedWarnings.Should().NotContainMatch("Test run detected DLL(s) which were built for different framework*"); - - fixture.ProcessHelper.Processes.Where(p => p.Started).Should().HaveCount(2); - var startWithSources1 = testhost1.FakeCommunicationChannel.ProcessedMessages.Single(m => m.Request.MessageType == MessageType.StartDiscovery); - var startWithSources1Text = startWithSources1.Request.Payload.Select(t => t.ToString()).JoinBySpace(); - // We sent mstest1.dll. - startWithSources1Text.Should().Contain("mstest1.dll"); - // And we sent net5 as the target framework, because that is the framework of mstest1.dll. - startWithSources1Text.Should().Contain(KnownFrameworkStrings.Net5); - - var startWithSources2 = testhost2.FakeCommunicationChannel.ProcessedMessages.Single(m => m.Request.MessageType == MessageType.StartDiscovery); - var startWithSources2Text = startWithSources2.Request.Payload.Select(t => t.ToString()).JoinBySpace(); - // We sent mstest2.dll. - startWithSources2Text.Should().Contain("mstest2.dll"); - // And we sent net48 as the target framework, because that is the framework of mstest2.dll. - startWithSources2Text.Should().Contain(mstest2Dll.FrameworkName.ToString()); - - fixture.DiscoveredTests.Should().HaveCount(mstest1Dll.TestCount + mstest2Dll.TestCount); - } - - [Test(@" + ); + + var mstest1Dll = new FakeTestDllBuilder() + .WithPath(@"X:\fake\mstest1.dll") + .WithFramework(KnownFrameworkNames.Net5) // <--- + .WithArchitecture(Architecture.X64) + .WithTestCount(11, 5) + .Build(); + + var testhost1Process = new FakeProcess(fixture.ErrorAggregator, @"X:\fake\testhost1.exe"); + + var runTests1 = new FakeTestHostResponsesBuilder() + .VersionCheck(5) + .DiscoveryInitialize(FakeMessage.NoResponse) + .StartDiscovery(mstest1Dll.TestResultBatches) + .SessionEnd(FakeMessage.NoResponse, afterAction: _ => testhost1Process.Exit()) + .Build(); + + var testhost1 = new FakeTestHostFixtureBuilder(fixture) + .WithTestDll(mstest1Dll) + .WithProcess(testhost1Process) + .WithResponses(runTests1) + .Build(); + + // -- + + var mstest2Dll = new FakeTestDllBuilder() + .WithPath(@"X:\fake\mstest2.dll") + .WithFramework(KnownFrameworkNames.Net48) // <--- + .WithArchitecture(Architecture.X64) + .WithTestCount(21, 5) + .Build(); + + var testhost2Process = new FakeProcess(fixture.ErrorAggregator, @"X:\fake\testhost2.exe"); + + var runTests2 = new FakeTestHostResponsesBuilder() + .VersionCheck(5) + .DiscoveryInitialize(FakeMessage.NoResponse) + .StartDiscovery(mstest2Dll.TestResultBatches) + .SessionEnd(FakeMessage.NoResponse, _ => testhost2Process.Exit()) + // We actually do get asked to terminate multiple times. In the second host only. + .SessionEnd(FakeMessage.NoResponse) + .Build(); + + var testhost2 = new FakeTestHostFixtureBuilder(fixture) + .WithTestDll(mstest2Dll) + .WithProcess(testhost2Process) + .WithResponses(runTests2) + .Build(); + + fixture.AddTestHostFixtures(testhost1, testhost2); + + var testRequestManager = fixture.BuildTestRequestManager(); + + mstest1Dll.FrameworkName.Should().NotBe(mstest2Dll.FrameworkName); + + // -- act + var testDiscoveryPayload = new DiscoveryRequestPayload + { + Sources = new List { mstest1Dll.Path, mstest2Dll.Path }, + RunSettings = $"" + }; + + await testRequestManager.ExecuteWithAbort(tm => tm.DiscoverTests(testDiscoveryPayload, fixture.TestDiscoveryEventsRegistrar, fixture.ProtocolConfig)); + + // -- assert + fixture.AssertNoErrors(); + // We figure out the framework for each assembly so there should be no incompatibility warnings + // REVIEW: This should be uncommented. Commenting it now, because it is helpful to see those warnings. + // fixture.TestRunEventsRegistrar.LoggedWarnings.Should().NotContainMatch("Test run detected DLL(s) which were built for different framework*"); + + fixture.ProcessHelper.Processes.Where(p => p.Started).Should().HaveCount(2); + var startWithSources1 = testhost1.FakeCommunicationChannel.ProcessedMessages.Single(m => m.Request.MessageType == MessageType.StartDiscovery); + var startWithSources1Text = startWithSources1.Request.Payload.Select(t => t.ToString()).JoinBySpace(); + // We sent mstest1.dll. + startWithSources1Text.Should().Contain("mstest1.dll"); + // And we sent net5 as the target framework, because that is the framework of mstest1.dll. + startWithSources1Text.Should().Contain(KnownFrameworkStrings.Net5); + + var startWithSources2 = testhost2.FakeCommunicationChannel.ProcessedMessages.Single(m => m.Request.MessageType == MessageType.StartDiscovery); + var startWithSources2Text = startWithSources2.Request.Payload.Select(t => t.ToString()).JoinBySpace(); + // We sent mstest2.dll. + startWithSources2Text.Should().Contain("mstest2.dll"); + // And we sent net48 as the target framework, because that is the framework of mstest2.dll. + startWithSources2Text.Should().Contain(mstest2Dll.FrameworkName.ToString()); + + fixture.DiscoveredTests.Should().HaveCount(mstest1Dll.TestCount + mstest2Dll.TestCount); + } + + [Test(@" Given two test assemblies that have the same architecture but have different target frameworks. @@ -135,320 +138,327 @@ and provide runsettings that define the desired target framework. Then two testhosts should be started that target the framework chosen by runsettings. ")] - public async Task B() - { - // -- arrange - using var fixture = new Fixture( - new FixtureOptions - { - FeatureFlags = new Dictionary + public async Task B() + { + // -- arrange + using var fixture = new Fixture( + new FixtureOptions { - [FeatureFlag.MULTI_TFM_RUN] = true + FeatureFlags = new Dictionary + { + [FeatureFlag.MULTI_TFM_RUN] = true + } } - } - ); - - var mstest1Dll = new FakeTestDllBuilder() - .WithPath(@"X:\fake\mstest1.dll") - .WithFramework(KnownFrameworkNames.Net5) // <--- - .WithArchitecture(Architecture.X64) - .WithTestCount(11, 5) - .Build(); - - var testhost1Process = new FakeProcess(fixture.ErrorAggregator, @"X:\fake\testhost1.exe"); - - var runTests1 = new FakeTestHostResponsesBuilder() - .VersionCheck(5) - .DiscoveryInitialize(FakeMessage.NoResponse) - .StartDiscovery(mstest1Dll.TestResultBatches) - .SessionEnd(FakeMessage.NoResponse, afterAction: _ => testhost1Process.Exit()) - .Build(); - - var testhost1 = new FakeTestHostFixtureBuilder(fixture) - .WithTestDll(mstest1Dll) - .WithProcess(testhost1Process) - .WithResponses(runTests1) - .Build(); - - // -- - - var mstest2Dll = new FakeTestDllBuilder() - .WithPath(@"X:\fake\mstest2.dll") - .WithFramework(KnownFrameworkNames.Net6) // <--- - .WithArchitecture(Architecture.X64) - .WithTestCount(21, 5) - .Build(); - - var testhost2Process = new FakeProcess(fixture.ErrorAggregator, @"X:\fake\testhost2.exe"); - - var runTests2 = new FakeTestHostResponsesBuilder() - .VersionCheck(5) - .DiscoveryInitialize(FakeMessage.NoResponse) - .StartDiscovery(mstest2Dll.TestResultBatches) - .SessionEnd(FakeMessage.NoResponse, _ => testhost2Process.Exit()) - // We actually do get asked to terminate multiple times. In the second host only. - .SessionEnd(FakeMessage.NoResponse) - .Build(); - - var testhost2 = new FakeTestHostFixtureBuilder(fixture) - .WithTestDll(mstest2Dll) - .WithProcess(testhost2Process) - .WithResponses(runTests2) - .Build(); - - fixture.AddTestHostFixtures(testhost1, testhost2); - - var testRequestManager = fixture.BuildTestRequestManager(); - - mstest1Dll.FrameworkName.Should().NotBe(mstest2Dll.FrameworkName); - - // -- act - var testDiscoveryPayload = new DiscoveryRequestPayload - { - Sources = new List { mstest1Dll.Path, mstest2Dll.Path }, - RunSettings = $"{KnownFrameworkStrings.Net7}" - }; - - await testRequestManager.ExecuteWithAbort(tm => tm.DiscoverTests(testDiscoveryPayload, fixture.TestDiscoveryEventsRegistrar, fixture.ProtocolConfig)); - - // -- assert - fixture.AssertNoErrors(); - // We figure out the framework for each assembly so there should be no incompatibility warnings - // REVIEW: This should be uncommented. Commenting it now, because it is helpful to see those warnings. - // fixture.TestRunEventsRegistrar.LoggedWarnings.Should().NotContainMatch("Test run detected DLL(s) which were built for different framework*"); - - fixture.ProcessHelper.Processes.Where(p => p.Started).Should().HaveCount(2); - var startWithSources1 = testhost1.FakeCommunicationChannel.ProcessedMessages.Single(m => m.Request.MessageType == MessageType.StartDiscovery); - var startWithSources1Text = startWithSources1.Request.Payload.Select(t => t.ToString()).JoinBySpace(); - // We sent mstest1.dll and net7 because that is what we have in settings. - startWithSources1Text.Should().Contain("mstest1.dll"); - startWithSources1Text.Should().Contain(KnownFrameworkStrings.Net7); - - var startWithSources2 = testhost2.FakeCommunicationChannel.ProcessedMessages.Single(m => m.Request.MessageType == MessageType.StartDiscovery); - var startWithSources2Text = startWithSources2.Request.Payload.Select(t => t.ToString()).JoinBySpace(); - // We sent mstest2.dll and net7 because that is what we have in settings. - startWithSources2Text.Should().Contain("mstest2.dll"); - startWithSources2Text.Should().Contain(KnownFrameworkStrings.Net7); - - fixture.DiscoveredTests.Should().HaveCount(mstest1Dll.TestCount + mstest2Dll.TestCount); + ); + + var mstest1Dll = new FakeTestDllBuilder() + .WithPath(@"X:\fake\mstest1.dll") + .WithFramework(KnownFrameworkNames.Net5) // <--- + .WithArchitecture(Architecture.X64) + .WithTestCount(11, 5) + .Build(); + + var testhost1Process = new FakeProcess(fixture.ErrorAggregator, @"X:\fake\testhost1.exe"); + + var runTests1 = new FakeTestHostResponsesBuilder() + .VersionCheck(5) + .DiscoveryInitialize(FakeMessage.NoResponse) + .StartDiscovery(mstest1Dll.TestResultBatches) + .SessionEnd(FakeMessage.NoResponse, afterAction: _ => testhost1Process.Exit()) + .Build(); + + var testhost1 = new FakeTestHostFixtureBuilder(fixture) + .WithTestDll(mstest1Dll) + .WithProcess(testhost1Process) + .WithResponses(runTests1) + .Build(); + + // -- + + var mstest2Dll = new FakeTestDllBuilder() + .WithPath(@"X:\fake\mstest2.dll") + .WithFramework(KnownFrameworkNames.Net6) // <--- + .WithArchitecture(Architecture.X64) + .WithTestCount(21, 5) + .Build(); + + var testhost2Process = new FakeProcess(fixture.ErrorAggregator, @"X:\fake\testhost2.exe"); + + var runTests2 = new FakeTestHostResponsesBuilder() + .VersionCheck(5) + .DiscoveryInitialize(FakeMessage.NoResponse) + .StartDiscovery(mstest2Dll.TestResultBatches) + .SessionEnd(FakeMessage.NoResponse, _ => testhost2Process.Exit()) + // We actually do get asked to terminate multiple times. In the second host only. + .SessionEnd(FakeMessage.NoResponse) + .Build(); + + var testhost2 = new FakeTestHostFixtureBuilder(fixture) + .WithTestDll(mstest2Dll) + .WithProcess(testhost2Process) + .WithResponses(runTests2) + .Build(); + + fixture.AddTestHostFixtures(testhost1, testhost2); + + var testRequestManager = fixture.BuildTestRequestManager(); + + mstest1Dll.FrameworkName.Should().NotBe(mstest2Dll.FrameworkName); + + // -- act + var testDiscoveryPayload = new DiscoveryRequestPayload + { + Sources = new List { mstest1Dll.Path, mstest2Dll.Path }, + RunSettings = $"{KnownFrameworkStrings.Net7}" + }; + + await testRequestManager.ExecuteWithAbort(tm => tm.DiscoverTests(testDiscoveryPayload, fixture.TestDiscoveryEventsRegistrar, fixture.ProtocolConfig)); + + // -- assert + fixture.AssertNoErrors(); + // We figure out the framework for each assembly so there should be no incompatibility warnings + // REVIEW: This should be uncommented. Commenting it now, because it is helpful to see those warnings. + // fixture.TestRunEventsRegistrar.LoggedWarnings.Should().NotContainMatch("Test run detected DLL(s) which were built for different framework*"); + + fixture.ProcessHelper.Processes.Where(p => p.Started).Should().HaveCount(2); + var startWithSources1 = testhost1.FakeCommunicationChannel.ProcessedMessages.Single(m => m.Request.MessageType == MessageType.StartDiscovery); + var startWithSources1Text = startWithSources1.Request.Payload.Select(t => t.ToString()).JoinBySpace(); + // We sent mstest1.dll and net7 because that is what we have in settings. + startWithSources1Text.Should().Contain("mstest1.dll"); + startWithSources1Text.Should().Contain(KnownFrameworkStrings.Net7); + + var startWithSources2 = testhost2.FakeCommunicationChannel.ProcessedMessages.Single(m => m.Request.MessageType == MessageType.StartDiscovery); + var startWithSources2Text = startWithSources2.Request.Payload.Select(t => t.ToString()).JoinBySpace(); + // We sent mstest2.dll and net7 because that is what we have in settings. + startWithSources2Text.Should().Contain("mstest2.dll"); + startWithSources2Text.Should().Contain(KnownFrameworkStrings.Net7); + + fixture.DiscoveredTests.Should().HaveCount(mstest1Dll.TestCount + mstest2Dll.TestCount); + } } - [Test(@" - Given two test assemblies that have the same architecture - but have different target frameworks. - - When we execute tests. - - Then two testhosts should be started that target the same framework as each assembly. - ")] - public async Task C() + public class MultiTFMExecution { - // -- arrange - using var fixture = new Fixture( - new FixtureOptions - { - FeatureFlags = new Dictionary - { - [FeatureFlag.MULTI_TFM_RUN] = true - } - } - ); - - var mstest1Dll = new FakeTestDllBuilder() - .WithPath(@"X:\fake\mstest1.dll") - .WithFramework(KnownFrameworkNames.Net5) // <--- - .WithArchitecture(Architecture.X64) - .WithTestCount(11, 5) - .Build(); - - var testhost1Process = new FakeProcess(fixture.ErrorAggregator, @"X:\fake\testhost1.exe"); - - var runTests1 = new FakeTestHostResponsesBuilder() - .VersionCheck(5) - .ExecutionInitialize(FakeMessage.NoResponse) - .StartTestExecutionWithSources(mstest1Dll.TestResultBatches) - .SessionEnd(FakeMessage.NoResponse, afterAction: _ => testhost1Process.Exit()) - .Build(); + [Test(@" + Given two test assemblies that have the same architecture + but have different target frameworks. - var testhost1 = new FakeTestHostFixtureBuilder(fixture) - .WithTestDll(mstest1Dll) - .WithProcess(testhost1Process) - .WithResponses(runTests1) - .Build(); - - // -- + When we execute tests. - var mstest2Dll = new FakeTestDllBuilder() - .WithPath(@"X:\fake\mstest2.dll") - .WithFramework(KnownFrameworkNames.Net6) // <--- - .WithArchitecture(Architecture.X64) - .WithTestCount(21, 5) - .Build(); - - var testhost2Process = new FakeProcess(fixture.ErrorAggregator, @"X:\fake\testhost2.exe"); - - var runTests2 = new FakeTestHostResponsesBuilder() - .VersionCheck(5) - .ExecutionInitialize(FakeMessage.NoResponse) - .StartTestExecutionWithSources(mstest2Dll.TestResultBatches) - .SessionEnd(FakeMessage.NoResponse, _ => testhost2Process.Exit()) - // We actually do get asked to terminate multiple times. In the second host only. - .SessionEnd(FakeMessage.NoResponse) - .Build(); - - var testhost2 = new FakeTestHostFixtureBuilder(fixture) - .WithTestDll(mstest2Dll) - .WithProcess(testhost2Process) - .WithResponses(runTests2) - .Build(); - - fixture.AddTestHostFixtures(testhost1, testhost2); - - var testRequestManager = fixture.BuildTestRequestManager(); - - mstest1Dll.FrameworkName.Should().NotBe(mstest2Dll.FrameworkName); - - // -- act - var testRunRequestPayload = new TestRunRequestPayload + Then two testhosts should be started that target the same framework as each assembly. + ")] + public async Task C() { - Sources = new List { mstest1Dll.Path, mstest2Dll.Path }, - RunSettings = $"" - }; - - await testRequestManager.ExecuteWithAbort(tm => tm.RunTests(testRunRequestPayload, testHostLauncher: null, fixture.TestRunEventsRegistrar, fixture.ProtocolConfig)); - - // -- assert - fixture.AssertNoErrors(); - // We figure out the framework for each assembly so there should be no incompatibility warnings - // REVIEW: This should be uncommented. Commenting it now, because it is helpful to see those warnings. - // fixture.TestRunEventsRegistrar.LoggedWarnings.Should().NotContainMatch("Test run detected DLL(s) which were built for different framework*"); - - fixture.ProcessHelper.Processes.Where(p => p.Started).Should().HaveCount(2); - var startWithSources1 = testhost1.FakeCommunicationChannel.ProcessedMessages.Single(m => m.Request.MessageType == MessageType.StartTestExecutionWithSources); - var startWithSources1Text = startWithSources1.Request.Payload.Select(t => t.ToString()).JoinBySpace(); - // We sent mstest1.dll. - startWithSources1Text.Should().Contain("mstest1.dll"); - // And we sent net5 as the target framework, because that is the framework of mstest1.dll. - startWithSources1Text.Should().Contain(KnownFrameworkStrings.Net5); - - var startWithSources2 = testhost2.FakeCommunicationChannel.ProcessedMessages.Single(m => m.Request.MessageType == MessageType.StartTestExecutionWithSources); - var startWithSources2Text = startWithSources2.Request.Payload.Select(t => t.ToString()).JoinBySpace(); - // We sent mstest2.dll. - startWithSources2Text.Should().Contain("mstest2.dll"); - // And we sent net48 as the target framework, because that is the framework of mstest2.dll. - startWithSources2Text.Should().Contain(KnownFrameworkStrings.Net6); - - fixture.ExecutedTests.Should().HaveCount(mstest1Dll.TestCount + mstest2Dll.TestCount); - } - - [Test(@" - Given two test assemblies that have the same architecture - but have different target frameworks. - - When we execute tests - and provide runsettings that define the desired target framework. - - Then two testhosts should be started that target the framework chosen by runsettings. - ")] - public async Task D() - { - // -- arrange - using var fixture = new Fixture( - new FixtureOptions - { - FeatureFlags = new Dictionary + // -- arrange + using var fixture = new Fixture( + new FixtureOptions { - [FeatureFlag.MULTI_TFM_RUN] = true + FeatureFlags = new Dictionary + { + [FeatureFlag.MULTI_TFM_RUN] = true + } } - } - ); - - var mstest1Dll = new FakeTestDllBuilder() - .WithPath(@"X:\fake\mstest1.dll") - .WithFramework(KnownFrameworkNames.Net5) // <--- - .WithArchitecture(Architecture.X64) - .WithTestCount(11, 5) - .Build(); - - var testhost1Process = new FakeProcess(fixture.ErrorAggregator, @"X:\fake\testhost1.exe"); - - var runTests1 = new FakeTestHostResponsesBuilder() - .VersionCheck(5) - .ExecutionInitialize(FakeMessage.NoResponse) - .StartTestExecutionWithSources(mstest1Dll.TestResultBatches) - .SessionEnd(FakeMessage.NoResponse, afterAction: _ => testhost1Process.Exit()) - .Build(); - - var testhost1 = new FakeTestHostFixtureBuilder(fixture) - .WithTestDll(mstest1Dll) - .WithProcess(testhost1Process) - .WithResponses(runTests1) - .Build(); - - // -- - - var mstest2Dll = new FakeTestDllBuilder() - .WithPath(@"X:\fake\mstest2.dll") - .WithFramework(KnownFrameworkNames.Net6) // <--- - .WithArchitecture(Architecture.X64) - .WithTestCount(21, 5) - .Build(); - - var testhost2Process = new FakeProcess(fixture.ErrorAggregator, @"X:\fake\testhost2.exe"); - - var runTests2 = new FakeTestHostResponsesBuilder() - .VersionCheck(5) - .ExecutionInitialize(FakeMessage.NoResponse) - .StartTestExecutionWithSources(mstest2Dll.TestResultBatches) - .SessionEnd(FakeMessage.NoResponse, _ => testhost2Process.Exit()) - // We actually do get asked to terminate multiple times. In the second host only. - .SessionEnd(FakeMessage.NoResponse) - .Build(); - - var testhost2 = new FakeTestHostFixtureBuilder(fixture) - .WithTestDll(mstest2Dll) - .WithProcess(testhost2Process) - .WithResponses(runTests2) - .Build(); - - fixture.AddTestHostFixtures(testhost1, testhost2); - - var testRequestManager = fixture.BuildTestRequestManager(); - - mstest1Dll.FrameworkName.Should().NotBe(mstest2Dll.FrameworkName); - - // -- act - var testRunRequestPayload = new TestRunRequestPayload + ); + + var mstest1Dll = new FakeTestDllBuilder() + .WithPath(@"X:\fake\mstest1.dll") + .WithFramework(KnownFrameworkNames.Net5) // <--- + .WithArchitecture(Architecture.X64) + .WithTestCount(11, 5) + .Build(); + + var testhost1Process = new FakeProcess(fixture.ErrorAggregator, @"X:\fake\testhost1.exe"); + + var runTests1 = new FakeTestHostResponsesBuilder() + .VersionCheck(5) + .ExecutionInitialize(FakeMessage.NoResponse) + .StartTestExecutionWithSources(mstest1Dll.TestResultBatches) + .SessionEnd(FakeMessage.NoResponse, afterAction: _ => testhost1Process.Exit()) + .Build(); + + var testhost1 = new FakeTestHostFixtureBuilder(fixture) + .WithTestDll(mstest1Dll) + .WithProcess(testhost1Process) + .WithResponses(runTests1) + .Build(); + + // -- + + var mstest2Dll = new FakeTestDllBuilder() + .WithPath(@"X:\fake\mstest2.dll") + .WithFramework(KnownFrameworkNames.Net6) // <--- + .WithArchitecture(Architecture.X64) + .WithTestCount(21, 5) + .Build(); + + var testhost2Process = new FakeProcess(fixture.ErrorAggregator, @"X:\fake\testhost2.exe"); + + var runTests2 = new FakeTestHostResponsesBuilder() + .VersionCheck(5) + .ExecutionInitialize(FakeMessage.NoResponse) + .StartTestExecutionWithSources(mstest2Dll.TestResultBatches) + .SessionEnd(FakeMessage.NoResponse, _ => testhost2Process.Exit()) + // We actually do get asked to terminate multiple times. In the second host only. + .SessionEnd(FakeMessage.NoResponse) + .Build(); + + var testhost2 = new FakeTestHostFixtureBuilder(fixture) + .WithTestDll(mstest2Dll) + .WithProcess(testhost2Process) + .WithResponses(runTests2) + .Build(); + + fixture.AddTestHostFixtures(testhost1, testhost2); + + var testRequestManager = fixture.BuildTestRequestManager(); + + mstest1Dll.FrameworkName.Should().NotBe(mstest2Dll.FrameworkName); + + // -- act + var testRunRequestPayload = new TestRunRequestPayload + { + Sources = new List { mstest1Dll.Path, mstest2Dll.Path }, + RunSettings = $"" + }; + + await testRequestManager.ExecuteWithAbort(tm => tm.RunTests(testRunRequestPayload, testHostLauncher: null, fixture.TestRunEventsRegistrar, fixture.ProtocolConfig)); + + // -- assert + fixture.AssertNoErrors(); + // We figure out the framework for each assembly so there should be no incompatibility warnings + // REVIEW: This should be uncommented. Commenting it now, because it is helpful to see those warnings. + // fixture.TestRunEventsRegistrar.LoggedWarnings.Should().NotContainMatch("Test run detected DLL(s) which were built for different framework*"); + + fixture.ProcessHelper.Processes.Where(p => p.Started).Should().HaveCount(2); + var startWithSources1 = testhost1.FakeCommunicationChannel.ProcessedMessages.Single(m => m.Request.MessageType == MessageType.StartTestExecutionWithSources); + var startWithSources1Text = startWithSources1.Request.Payload.Select(t => t.ToString()).JoinBySpace(); + // We sent mstest1.dll. + startWithSources1Text.Should().Contain("mstest1.dll"); + // And we sent net5 as the target framework, because that is the framework of mstest1.dll. + startWithSources1Text.Should().Contain(KnownFrameworkStrings.Net5); + + var startWithSources2 = testhost2.FakeCommunicationChannel.ProcessedMessages.Single(m => m.Request.MessageType == MessageType.StartTestExecutionWithSources); + var startWithSources2Text = startWithSources2.Request.Payload.Select(t => t.ToString()).JoinBySpace(); + // We sent mstest2.dll. + startWithSources2Text.Should().Contain("mstest2.dll"); + // And we sent net48 as the target framework, because that is the framework of mstest2.dll. + startWithSources2Text.Should().Contain(KnownFrameworkStrings.Net6); + + fixture.ExecutedTests.Should().HaveCount(mstest1Dll.TestCount + mstest2Dll.TestCount); + } + + [Test(@" + Given two test assemblies that have the same architecture + but have different target frameworks. + + When we execute tests + and provide runsettings that define the desired target framework. + + Then two testhosts should be started that target the framework chosen by runsettings. + ")] + public async Task D() { - Sources = new List { mstest1Dll.Path, mstest2Dll.Path }, - RunSettings = $"{KnownFrameworkStrings.Net7}" - }; - - await testRequestManager.ExecuteWithAbort(tm => tm.RunTests(testRunRequestPayload, testHostLauncher: null, fixture.TestRunEventsRegistrar, fixture.ProtocolConfig)); - - // -- assert - fixture.AssertNoErrors(); - // We figure out the framework for each assembly so there should be no incompatibility warnings - // REVIEW: This should be uncommented. Commenting it now, because it is helpful to see those warnings. - // fixture.TestRunEventsRegistrar.LoggedWarnings.Should().NotContainMatch("Test run detected DLL(s) which were built for different framework*"); - - fixture.ProcessHelper.Processes.Where(p => p.Started).Should().HaveCount(2); - var startWithSources1 = testhost1.FakeCommunicationChannel.ProcessedMessages.Single(m => m.Request.MessageType == MessageType.StartTestExecutionWithSources); - var startWithSources1Text = startWithSources1.Request.Payload.Select(t => t.ToString()).JoinBySpace(); - // We sent mstest1.dll. - startWithSources1Text.Should().Contain("mstest1.dll"); - startWithSources1Text.Should().Contain(KnownFrameworkStrings.Net7); - - var startWithSources2 = testhost2.FakeCommunicationChannel.ProcessedMessages.Single(m => m.Request.MessageType == MessageType.StartTestExecutionWithSources); - var startWithSources2Text = startWithSources2.Request.Payload.Select(t => t.ToString()).JoinBySpace(); - // We sent mstest2.dll. - startWithSources2Text.Should().Contain("mstest2.dll"); - startWithSources2Text.Should().Contain(KnownFrameworkStrings.Net7); - - fixture.ExecutedTests.Should().HaveCount(mstest1Dll.TestCount + mstest2Dll.TestCount); + // -- arrange + using var fixture = new Fixture( + new FixtureOptions + { + FeatureFlags = new Dictionary + { + [FeatureFlag.MULTI_TFM_RUN] = true + } + } + ); + + var mstest1Dll = new FakeTestDllBuilder() + .WithPath(@"X:\fake\mstest1.dll") + .WithFramework(KnownFrameworkNames.Net5) // <--- + .WithArchitecture(Architecture.X64) + .WithTestCount(11, 5) + .Build(); + + var testhost1Process = new FakeProcess(fixture.ErrorAggregator, @"X:\fake\testhost1.exe"); + + var runTests1 = new FakeTestHostResponsesBuilder() + .VersionCheck(5) + .ExecutionInitialize(FakeMessage.NoResponse) + .StartTestExecutionWithSources(mstest1Dll.TestResultBatches) + .SessionEnd(FakeMessage.NoResponse, afterAction: _ => testhost1Process.Exit()) + .Build(); + + var testhost1 = new FakeTestHostFixtureBuilder(fixture) + .WithTestDll(mstest1Dll) + .WithProcess(testhost1Process) + .WithResponses(runTests1) + .Build(); + + // -- + + var mstest2Dll = new FakeTestDllBuilder() + .WithPath(@"X:\fake\mstest2.dll") + .WithFramework(KnownFrameworkNames.Net6) // <--- + .WithArchitecture(Architecture.X64) + .WithTestCount(21, 5) + .Build(); + + var testhost2Process = new FakeProcess(fixture.ErrorAggregator, @"X:\fake\testhost2.exe"); + + var runTests2 = new FakeTestHostResponsesBuilder() + .VersionCheck(5) + .ExecutionInitialize(FakeMessage.NoResponse) + .StartTestExecutionWithSources(mstest2Dll.TestResultBatches) + .SessionEnd(FakeMessage.NoResponse, _ => testhost2Process.Exit()) + // We actually do get asked to terminate multiple times. In the second host only. + .SessionEnd(FakeMessage.NoResponse) + .Build(); + + var testhost2 = new FakeTestHostFixtureBuilder(fixture) + .WithTestDll(mstest2Dll) + .WithProcess(testhost2Process) + .WithResponses(runTests2) + .Build(); + + fixture.AddTestHostFixtures(testhost1, testhost2); + + var testRequestManager = fixture.BuildTestRequestManager(); + + mstest1Dll.FrameworkName.Should().NotBe(mstest2Dll.FrameworkName); + + // -- act + var testRunRequestPayload = new TestRunRequestPayload + { + Sources = new List { mstest1Dll.Path, mstest2Dll.Path }, + RunSettings = $"{KnownFrameworkStrings.Net7}" + }; + + await testRequestManager.ExecuteWithAbort(tm => tm.RunTests(testRunRequestPayload, testHostLauncher: null, fixture.TestRunEventsRegistrar, fixture.ProtocolConfig)); + + // -- assert + fixture.AssertNoErrors(); + // We figure out the framework for each assembly so there should be no incompatibility warnings + // REVIEW: This should be uncommented. Commenting it now, because it is helpful to see those warnings. + // fixture.TestRunEventsRegistrar.LoggedWarnings.Should().NotContainMatch("Test run detected DLL(s) which were built for different framework*"); + + fixture.ProcessHelper.Processes.Where(p => p.Started).Should().HaveCount(2); + var startWithSources1 = testhost1.FakeCommunicationChannel.ProcessedMessages.Single(m => m.Request.MessageType == MessageType.StartTestExecutionWithSources); + var startWithSources1Text = startWithSources1.Request.Payload.Select(t => t.ToString()).JoinBySpace(); + // We sent mstest1.dll. + startWithSources1Text.Should().Contain("mstest1.dll"); + startWithSources1Text.Should().Contain(KnownFrameworkStrings.Net7); + + var startWithSources2 = testhost2.FakeCommunicationChannel.ProcessedMessages.Single(m => m.Request.MessageType == MessageType.StartTestExecutionWithSources); + var startWithSources2Text = startWithSources2.Request.Payload.Select(t => t.ToString()).JoinBySpace(); + // We sent mstest2.dll. + startWithSources2Text.Should().Contain("mstest2.dll"); + startWithSources2Text.Should().Contain(KnownFrameworkStrings.Net7); + + fixture.ExecutedTests.Should().HaveCount(mstest1Dll.TestCount + mstest2Dll.TestCount); + } } - [Test(@" + public class MultiTFMTestSessions + { + + [Test(@" Given two test assemblies that have the same architecture but have different target frameworks. @@ -457,111 +467,112 @@ and provide runsettings that define the desired target framework. Then two testhosts should be started that target the framework chosen by runsettings. ")][Only] - public async Task E() - { - // -- arrange - using var fixture = new Fixture( - new FixtureOptions - { - FeatureFlags = new Dictionary + public async Task E() + { + // -- arrange + using var fixture = new Fixture( + new FixtureOptions { - [FeatureFlag.MULTI_TFM_RUN] = true + FeatureFlags = new Dictionary + { + [FeatureFlag.MULTI_TFM_RUN] = true + } } - } - ); + ); - var mstest1Dll = new FakeTestDllBuilder() - .WithPath(@"X:\fake\mstest1.dll") - .WithFramework(KnownFrameworkNames.Net5) // <--- - .WithArchitecture(Architecture.X64) - .WithTestCount(11, 5) - .Build(); + var mstest1Dll = new FakeTestDllBuilder() + .WithPath(@"X:\fake\mstest1.dll") + .WithFramework(KnownFrameworkNames.Net5) // <--- + .WithArchitecture(Architecture.X64) + .WithTestCount(11, 5) + .Build(); - var testhost1Process = new FakeProcess(fixture.ErrorAggregator, @"X:\fake\testhost1.exe"); + var testhost1Process = new FakeProcess(fixture.ErrorAggregator, @"X:\fake\testhost1.exe"); - var runTests1 = new FakeTestHostResponsesBuilder() - .VersionCheck(5) - .ExecutionInitialize(FakeMessage.NoResponse) - .StartTestExecutionWithSources(mstest1Dll.TestResultBatches) - .SessionEnd(FakeMessage.NoResponse, afterAction: _ => testhost1Process.Exit()) - .Build(); + var runTests1 = new FakeTestHostResponsesBuilder() + .VersionCheck(5) + .ExecutionInitialize(FakeMessage.NoResponse) + .StartTestExecutionWithSources(mstest1Dll.TestResultBatches) + .SessionEnd(FakeMessage.NoResponse, afterAction: _ => testhost1Process.Exit()) + .Build(); - var testhost1 = new FakeTestHostFixtureBuilder(fixture) - .WithTestDll(mstest1Dll) - .WithProcess(testhost1Process) - .WithResponses(runTests1) - .Build(); + var testhost1 = new FakeTestHostFixtureBuilder(fixture) + .WithTestDll(mstest1Dll) + .WithProcess(testhost1Process) + .WithResponses(runTests1) + .Build(); - // -- + // -- - var mstest2Dll = new FakeTestDllBuilder() - .WithPath(@"X:\fake\mstest2.dll") - .WithFramework(KnownFrameworkNames.Net6) // <--- - .WithArchitecture(Architecture.X64) - .WithTestCount(21, 5) - .Build(); + var mstest2Dll = new FakeTestDllBuilder() + .WithPath(@"X:\fake\mstest2.dll") + .WithFramework(KnownFrameworkNames.Net6) // <--- + .WithArchitecture(Architecture.X64) + .WithTestCount(21, 5) + .Build(); - var testhost2Process = new FakeProcess(fixture.ErrorAggregator, @"X:\fake\testhost2.exe"); + var testhost2Process = new FakeProcess(fixture.ErrorAggregator, @"X:\fake\testhost2.exe"); - var runTests2 = new FakeTestHostResponsesBuilder() - .VersionCheck(5) - .ExecutionInitialize(FakeMessage.NoResponse) - .StartTestExecutionWithSources(mstest2Dll.TestResultBatches) - .SessionEnd(FakeMessage.NoResponse, _ => testhost2Process.Exit()) - // We actually do get asked to terminate multiple times. In the second host only. - .SessionEnd(FakeMessage.NoResponse) - .Build(); + var runTests2 = new FakeTestHostResponsesBuilder() + .VersionCheck(5) + .ExecutionInitialize(FakeMessage.NoResponse) + .StartTestExecutionWithSources(mstest2Dll.TestResultBatches) + .SessionEnd(FakeMessage.NoResponse, _ => testhost2Process.Exit()) + // We actually do get asked to terminate multiple times. In the second host only. + .SessionEnd(FakeMessage.NoResponse) + .Build(); - var testhost2 = new FakeTestHostFixtureBuilder(fixture) - .WithTestDll(mstest2Dll) - .WithProcess(testhost2Process) - .WithResponses(runTests2) - .Build(); + var testhost2 = new FakeTestHostFixtureBuilder(fixture) + .WithTestDll(mstest2Dll) + .WithProcess(testhost2Process) + .WithResponses(runTests2) + .Build(); - fixture.AddTestHostFixtures(testhost1, testhost2); + fixture.AddTestHostFixtures(testhost1, testhost2); - var testRequestManager = fixture.BuildTestRequestManager(); + var testRequestManager = fixture.BuildTestRequestManager(); - mstest1Dll.FrameworkName.Should().NotBe(mstest2Dll.FrameworkName); + mstest1Dll.FrameworkName.Should().NotBe(mstest2Dll.FrameworkName); - // -- act - - var startTestSessionPayload = new StartTestSessionPayload - { - RunSettings = "", - Sources = new []{ mstest1Dll.Path, mstest2Dll.Path } - }; - - await testRequestManager.ExecuteWithAbort(tm => tm.StartTestSession(startTestSessionPayload, testHostLauncher: null, fixture.TestSessionEventsHandler, fixture.ProtocolConfig)); - - var testRunRequestPayload = new TestRunRequestPayload - { - Sources = new List { mstest1Dll.Path, mstest2Dll.Path }, - RunSettings = $"{KnownFrameworkStrings.Net7}" - }; - - await testRequestManager.ExecuteWithAbort(tm => tm.RunTests(testRunRequestPayload, testHostLauncher: null, fixture.TestRunEventsRegistrar, fixture.ProtocolConfig)); + // -- act - // -- assert - fixture.AssertNoErrors(); - // We figure out the framework for each assembly so there should be no incompatibility warnings - // REVIEW: This should be uncommented. Commenting it now, because it is helpful to see those warnings. - // fixture.TestRunEventsRegistrar.LoggedWarnings.Should().NotContainMatch("Test run detected DLL(s) which were built for different framework*"); - - fixture.ProcessHelper.Processes.Where(p => p.Started).Should().HaveCount(2); - var startWithSources1 = testhost1.FakeCommunicationChannel.ProcessedMessages.Single(m => m.Request.MessageType == MessageType.StartTestExecutionWithSources); - var startWithSources1Text = startWithSources1.Request.Payload.Select(t => t.ToString()).JoinBySpace(); - // We sent mstest1.dll. - startWithSources1Text.Should().Contain("mstest1.dll"); - startWithSources1Text.Should().Contain(KnownFrameworkStrings.Net7); + var startTestSessionPayload = new StartTestSessionPayload + { + RunSettings = "", + Sources = new[] { mstest1Dll.Path, mstest2Dll.Path } + }; - var startWithSources2 = testhost2.FakeCommunicationChannel.ProcessedMessages.Single(m => m.Request.MessageType == MessageType.StartTestExecutionWithSources); - var startWithSources2Text = startWithSources2.Request.Payload.Select(t => t.ToString()).JoinBySpace(); - // We sent mstest2.dll. - startWithSources2Text.Should().Contain("mstest2.dll"); - startWithSources2Text.Should().Contain(KnownFrameworkStrings.Net7); + await testRequestManager.ExecuteWithAbort(tm => tm.StartTestSession(startTestSessionPayload, testHostLauncher: null, fixture.TestSessionEventsHandler, fixture.ProtocolConfig)); - fixture.ExecutedTests.Should().HaveCount(mstest1Dll.TestCount + mstest2Dll.TestCount); + var testRunRequestPayload = new TestRunRequestPayload + { + Sources = new List { mstest1Dll.Path, mstest2Dll.Path }, + RunSettings = $"{KnownFrameworkStrings.Net7}" + }; + + await testRequestManager.ExecuteWithAbort(tm => tm.RunTests(testRunRequestPayload, testHostLauncher: null, fixture.TestRunEventsRegistrar, fixture.ProtocolConfig)); + + // -- assert + fixture.AssertNoErrors(); + // We figure out the framework for each assembly so there should be no incompatibility warnings + // REVIEW: This should be uncommented. Commenting it now, because it is helpful to see those warnings. + // fixture.TestRunEventsRegistrar.LoggedWarnings.Should().NotContainMatch("Test run detected DLL(s) which were built for different framework*"); + + fixture.ProcessHelper.Processes.Where(p => p.Started).Should().HaveCount(2); + var startWithSources1 = testhost1.FakeCommunicationChannel.ProcessedMessages.Single(m => m.Request.MessageType == MessageType.StartTestExecutionWithSources); + var startWithSources1Text = startWithSources1.Request.Payload.Select(t => t.ToString()).JoinBySpace(); + // We sent mstest1.dll. + startWithSources1Text.Should().Contain("mstest1.dll"); + startWithSources1Text.Should().Contain(KnownFrameworkStrings.Net7); + + var startWithSources2 = testhost2.FakeCommunicationChannel.ProcessedMessages.Single(m => m.Request.MessageType == MessageType.StartTestExecutionWithSources); + var startWithSources2Text = startWithSources2.Request.Payload.Select(t => t.ToString()).JoinBySpace(); + // We sent mstest2.dll. + startWithSources2Text.Should().Contain("mstest2.dll"); + startWithSources2Text.Should().Contain(KnownFrameworkStrings.Net7); + + fixture.ExecutedTests.Should().HaveCount(mstest1Dll.TestCount + mstest2Dll.TestCount); + } } } From 6549db963c73afee38773c52b0f8ee400fd41923 Mon Sep 17 00:00:00 2001 From: nohwnd Date: Tue, 1 Mar 2022 11:12:30 +0100 Subject: [PATCH 009/112] Test sessions, and rework a bit --- .../TestPlatform.cs | 5 +- .../Hosting/ITestRuntimeProviderManager.cs | 4 +- .../Hosting/TestRunTimeProviderManager.cs | 4 +- .../IParallelOperationManager.cs | 18 - .../IParallelProxyDiscoveryManager.cs | 2 +- .../IParallelProxyExecutionManager.cs | 2 +- .../Parallel/ParallelOperationManager.cs | 293 +++++++------- .../Parallel/ParallelProxyDiscoveryManager.cs | 174 +++------ .../Parallel/ParallelProxyExecutionManager.cs | 364 ++++++++---------- .../Client/TestRuntimeProviderInfo.cs | 25 ++ .../TestEngine.cs | 162 +++++--- .../TestSession/ProxyTestSessionManager.cs | 82 ++-- .../Client/Interfaces/ITestPlatform.cs | 3 +- .../RunSettings/RunConfiguration.cs | 1 + .../TestPlatformHelpers/TestRequestManager.cs | 10 +- .../Parallel/ParallelOperationManagerTests.cs | 1 - .../Fakes/ActionRecord.cs | 19 + .../Fakes/FakeCommunicationChannel.cs | 5 + .../Fakes/FakeMessage.cs | 5 + .../Fakes/FakeProcess.cs | 10 + .../Fakes/FakeTestRuntimeProvider.cs | 7 + .../Fakes/FakeTestRuntimeProviderManager.cs | 29 +- test/vstest.ProgrammerTests/Fakes/Fixture.cs | 2 +- .../Fakes/RequestResponsePair.cs | 5 + .../MultiTFMRunAndDiscovery.cs | 18 +- 25 files changed, 622 insertions(+), 628 deletions(-) delete mode 100644 src/Microsoft.TestPlatform.Common/Interfaces/Engine/ClientProtocol/IParallelOperationManager.cs create mode 100644 src/Microsoft.TestPlatform.CrossPlatEngine/Client/TestRuntimeProviderInfo.cs create mode 100644 test/vstest.ProgrammerTests/Fakes/ActionRecord.cs diff --git a/src/Microsoft.TestPlatform.Client/TestPlatform.cs b/src/Microsoft.TestPlatform.Client/TestPlatform.cs index 29173b77bf..efe15a52fb 100644 --- a/src/Microsoft.TestPlatform.Client/TestPlatform.cs +++ b/src/Microsoft.TestPlatform.Client/TestPlatform.cs @@ -124,7 +124,8 @@ public ITestRunRequest CreateTestRunRequest( public bool StartTestSession( IRequestData requestData, StartTestSessionCriteria testSessionCriteria!!, - ITestSessionEventsHandler eventsHandler) + ITestSessionEventsHandler eventsHandler, + Dictionary sourceToSourceDetailMap) { RunConfiguration runConfiguration = XmlRunSettingsUtilities.GetRunConfigurationNode(testSessionCriteria.RunSettings); TestAdapterLoadingStrategy strategy = runConfiguration.TestAdapterLoadingStrategy; @@ -136,8 +137,6 @@ public bool StartTestSession( return false; } - // REVIEW: TODO: NOMERGE: figure out the details of each source - Dictionary sourceToSourceDetailMap = new(); IProxyTestSessionManager testSessionManager = TestEngine.GetTestSessionManager(requestData, testSessionCriteria, sourceToSourceDetailMap); if (testSessionManager == null) { diff --git a/src/Microsoft.TestPlatform.Common/Hosting/ITestRuntimeProviderManager.cs b/src/Microsoft.TestPlatform.Common/Hosting/ITestRuntimeProviderManager.cs index 57366c4669..5c2f285440 100644 --- a/src/Microsoft.TestPlatform.Common/Hosting/ITestRuntimeProviderManager.cs +++ b/src/Microsoft.TestPlatform.Common/Hosting/ITestRuntimeProviderManager.cs @@ -3,10 +3,12 @@ namespace Microsoft.VisualStudio.TestPlatform.Common.Hosting; +using System.Collections.Generic; + using Microsoft.VisualStudio.TestPlatform.ObjectModel.Host; internal interface ITestRuntimeProviderManager { - ITestRuntimeProvider GetTestHostManagerByRunConfiguration(string runConfiguration); + ITestRuntimeProvider GetTestHostManagerByRunConfiguration(string runConfiguration, List sources); ITestRuntimeProvider GetTestHostManagerByUri(string hostUri); } diff --git a/src/Microsoft.TestPlatform.Common/Hosting/TestRunTimeProviderManager.cs b/src/Microsoft.TestPlatform.Common/Hosting/TestRunTimeProviderManager.cs index 9752ec6f57..c637e5e2e0 100644 --- a/src/Microsoft.TestPlatform.Common/Hosting/TestRunTimeProviderManager.cs +++ b/src/Microsoft.TestPlatform.Common/Hosting/TestRunTimeProviderManager.cs @@ -10,6 +10,8 @@ namespace Microsoft.VisualStudio.TestPlatform.Common.Hosting; using Logging; using ObjectModel.Host; using Microsoft.VisualStudio.TestPlatform.ObjectModel.Logging; +using Microsoft.VisualStudio.TestPlatform.Common.Interfaces; +using System.Collections.Generic; /// /// Responsible for managing TestRuntimeProviderManager extensions @@ -44,7 +46,7 @@ public ITestRuntimeProvider GetTestHostManagerByUri(string hostUri) return host?.Value; } - public virtual ITestRuntimeProvider GetTestHostManagerByRunConfiguration(string runConfiguration) + public virtual ITestRuntimeProvider GetTestHostManagerByRunConfiguration(string runConfiguration, List _) { foreach (var testExtension in _testHostExtensionManager.TestExtensions) { diff --git a/src/Microsoft.TestPlatform.Common/Interfaces/Engine/ClientProtocol/IParallelOperationManager.cs b/src/Microsoft.TestPlatform.Common/Interfaces/Engine/ClientProtocol/IParallelOperationManager.cs deleted file mode 100644 index f0300abddd..0000000000 --- a/src/Microsoft.TestPlatform.Common/Interfaces/Engine/ClientProtocol/IParallelOperationManager.cs +++ /dev/null @@ -1,18 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT license. See LICENSE file in the project root for full license information. - -#nullable disable - -namespace Microsoft.VisualStudio.TestPlatform.ObjectModel.Engine; - -/// -/// Interface defining the parallel operation manager -/// -public interface IParallelOperationManager -{ - /// - /// Update the parallelism level of the manager - /// - /// Parallelism level - void UpdateParallelLevel(int parallelLevel); -} diff --git a/src/Microsoft.TestPlatform.Common/Interfaces/Engine/ClientProtocol/IParallelProxyDiscoveryManager.cs b/src/Microsoft.TestPlatform.Common/Interfaces/Engine/ClientProtocol/IParallelProxyDiscoveryManager.cs index cbd40b93a0..8f4ff0e14f 100644 --- a/src/Microsoft.TestPlatform.Common/Interfaces/Engine/ClientProtocol/IParallelProxyDiscoveryManager.cs +++ b/src/Microsoft.TestPlatform.Common/Interfaces/Engine/ClientProtocol/IParallelProxyDiscoveryManager.cs @@ -10,7 +10,7 @@ namespace Microsoft.VisualStudio.TestPlatform.ObjectModel.Engine; /// /// Interface defining the parallel discovery manager /// -public interface IParallelProxyDiscoveryManager : IParallelOperationManager, IProxyDiscoveryManager +public interface IParallelProxyDiscoveryManager : IProxyDiscoveryManager { /// /// Indicates if user requested an abortion diff --git a/src/Microsoft.TestPlatform.Common/Interfaces/Engine/ClientProtocol/IParallelProxyExecutionManager.cs b/src/Microsoft.TestPlatform.Common/Interfaces/Engine/ClientProtocol/IParallelProxyExecutionManager.cs index 14c9c1a36d..26a57849fb 100644 --- a/src/Microsoft.TestPlatform.Common/Interfaces/Engine/ClientProtocol/IParallelProxyExecutionManager.cs +++ b/src/Microsoft.TestPlatform.Common/Interfaces/Engine/ClientProtocol/IParallelProxyExecutionManager.cs @@ -12,7 +12,7 @@ namespace Microsoft.VisualStudio.TestPlatform.ObjectModel.Engine; /// /// Interface defining the parallel execution manager /// -public interface IParallelProxyExecutionManager : IParallelOperationManager, IProxyExecutionManager +public interface IParallelProxyExecutionManager : IProxyExecutionManager { /// /// Handles Partial Run Complete event coming from a specific concurrent proxy execution manager diff --git a/src/Microsoft.TestPlatform.CrossPlatEngine/Client/Parallel/ParallelOperationManager.cs b/src/Microsoft.TestPlatform.CrossPlatEngine/Client/Parallel/ParallelOperationManager.cs index 7d8df891ec..294d5356a4 100644 --- a/src/Microsoft.TestPlatform.CrossPlatEngine/Client/Parallel/ParallelOperationManager.cs +++ b/src/Microsoft.TestPlatform.CrossPlatEngine/Client/Parallel/ParallelOperationManager.cs @@ -1,213 +1,183 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT license. See LICENSE file in the project root for full license information. -#nullable disable - namespace Microsoft.VisualStudio.TestPlatform.CrossPlatEngine.Client; using System; -using System.Collections; using System.Collections.Concurrent; using System.Collections.Generic; using System.Linq; using System.Threading.Tasks; +using Microsoft.VisualStudio.TestPlatform.CrossPlatEngine.Client.Parallel; + using ObjectModel; -using ObjectModel.Engine; /// -/// Common parallel manager functionality. +/// Manages work that is done on multiple testhosts in parallel. /// -internal abstract class ParallelOperationManager : IParallelOperationManager, IDisposable +internal sealed class ParallelOperationManager : IDisposable { - #region ConcurrentManagerInstanceData - - protected Func CreateNewConcurrentManager { get; } + private readonly Func _createNewManager; + private readonly int _maxParallelLevel; /// /// Holds all active managers, so we can do actions on all of them, like initialize, run, cancel or close. /// // TODO: make this ConcurrentDictionary and use it's concurrent api, if we have the need. - private readonly IDictionary _concurrentManagerHandlerMap = new ConcurrentDictionary(); - - /// - /// Singleton Instance of this class - /// - protected static TParallelManager s_instance; + private readonly IDictionary _managerToEventHandlerMap = new ConcurrentDictionary(); /// /// Default number of Processes /// - private int _currentParallelLevel; - - protected int MaxParallelLevel { get; private set; } - - #endregion - - #region Concurrency Keeper Objects - - /// - /// LockObject to iterate our sourceEnumerator in parallel - /// We can use the sourceEnumerator itself as lockObject, but since its a changing object - it's risky to use it as one - /// - protected object _sourceEnumeratorLockObject = new(); + private TEventHandler _eventHandler; + private Func _getEventHandler; + private Action _runWorkload; + private readonly List> _workloads = new(); + private readonly List _managerSlots = new(); - #endregion + private readonly object _lock = new(); - protected ParallelOperationManager(Func createNewManager, int parallelLevel) + public ParallelOperationManager(Func createNewManager, int parallelLevel) { - CreateNewConcurrentManager = createNewManager; - - // Update Parallel Level - // REVIEW: this "pre-starts" testhosts so we have a pool of them, this is the reason the number or - // parallel hosts is capped to the amount of sources so we don't "pre-start" too many of them - // instead we should take each source, look if it can be run by shared host, and if so try to - // grab a free host, run new one if we are below parallel level, or wait if we are at parallel - // level and everyone is busy if we have non-shared host we do just the two last options, run new - // one if current count is under parallel level, or wait till we can run new one. - // this.UpdateParallelLevel(parallelLevel); - - MaxParallelLevel = parallelLevel; + _createNewManager = createNewManager; + _maxParallelLevel = parallelLevel; + ClearSlots(); } - /// - /// Remove and dispose a manager from concurrent list of manager. - /// - /// Manager to remove - public void RemoveManager(TParallelManager manager) + private void ClearSlots() { - _concurrentManagerHandlerMap.Remove(manager); + lock (_lock) + { + _managerSlots.Clear(); + _managerSlots.AddRange(Enumerable.Range(0, _maxParallelLevel).Select(_ => new Slot())); + } } - /// - /// Add a manager in concurrent list of manager. - /// - /// Manager to add - /// eventHandler of the manager - public void AddManager(TParallelManager manager, TEventHandler handler) + public void StartWork( + List> workloads!!, + TEventHandler eventHandler!!, + Func getEventHandler!!, + Action runWorkload!!) { - _concurrentManagerHandlerMap.Add(manager, handler); - } + _eventHandler = eventHandler; + _getEventHandler = getEventHandler; + _runWorkload = runWorkload; - /// - /// Update event handler for the manager. - /// If it is a new manager, add this. - /// - /// Manager to update - /// event handler to update for manager - public void UpdateHandlerForManager(TParallelManager manager, TEventHandler handler) - { - if (_concurrentManagerHandlerMap.ContainsKey(manager)) - { - _concurrentManagerHandlerMap[manager] = handler; - } - else + _workloads.AddRange(workloads); + + lock (_lock) { - AddManager(manager, handler); + ClearSlots(); + RunWorkInParallel(); } } - /// - /// Get the event handler associated with the manager. - /// - /// Manager - public TEventHandler GetHandlerForGivenManager(TParallelManager manager) + private bool RunWorkInParallel() { - return _concurrentManagerHandlerMap[manager]; - } + // TODO: Right now we don't re-use shared hosts, but if we did, this is the place + // where we should find a workload that fits the manager if any of them is shared. + // Or tear it down, and start a new one. - /// - /// Get total number of active concurrent manager - /// - public int GetConcurrentManagersCount() - { - return _concurrentManagerHandlerMap.Count; - } - - /// - /// Get instances of all active concurrent manager - /// - public IEnumerable GetConcurrentManagerInstances() - { - return _concurrentManagerHandlerMap.Keys.ToList(); - } + List workToRun = new(); + lock (_lock) + { + if (_workloads.Count == 0) + return true; + var availableSlots = _managerSlots.Where(slot => slot.IsAvailable).ToList(); + var availableWorkloads = _workloads.Where(workload => workload != null).ToList(); + var amount = Math.Min(availableSlots.Count, availableWorkloads.Count); + var workloadsToRun = availableWorkloads.Take(amount).ToList(); - /// - /// Updates the Concurrent Executors according to new parallel setting - /// - /// Number of Parallel Executors allowed - public void UpdateParallelLevel(int newParallelLevel) - { - if (_concurrentManagerHandlerMap == null) - { - throw new Exception("ParallelOperationManager.UpdateParallelLevel: This should not be used anymore, to pre-start hosts."); + for (int i = 0; i < amount; i++) + { + var slot = availableSlots[i]; + slot.IsAvailable = false; + var workload = availableWorkloads[i]; + workToRun.Add(new SlotWorkloadPair(slot, workload)); + _workloads.Remove(workload); + } } - else if (_currentParallelLevel != newParallelLevel) + + foreach (var pair in workToRun) { - // If number of concurrent clients is less than the new level - // Create more concurrent clients and update the list - if (_currentParallelLevel < newParallelLevel) + try { - // This path does not even seem to be used anywhere. - - throw new Exception("ParallelOperationManager.UpdateParallelLevel: This should not be used anymore, to ensure we add more hosts."); + var manager = _createNewManager(pair.Workload.Provider); + var eventHandler = _getEventHandler(_eventHandler, manager); + pair.Slot.EventHandler = eventHandler; + pair.Slot.Manager = manager; + pair.Slot.ManagerInfo = pair.Workload.Provider; + pair.Slot.Work = pair.Workload.Work; + + _runWorkload(manager, eventHandler, pair.Workload.Work); } - else + finally { - // If number of concurrent clients is more than the new level - // Dispose off the extra ones - int managersCount = _currentParallelLevel - newParallelLevel; - - foreach (var concurrentManager in GetConcurrentManagerInstances()) - { - if (managersCount == 0) - { - break; - } - else - { - RemoveManager(concurrentManager); - managersCount--; - } - } + // clean the slot or something, to make sure we don't keep it reserved. } } - // Update current parallel setting to new one - _currentParallelLevel = newParallelLevel; + return false; } - public void Dispose() + public bool RunNextWork(TManager completedManager) { - if (_concurrentManagerHandlerMap != null) + lock (_lock) { - foreach (var managerInstance in GetConcurrentManagerInstances()) + var completedSlot = _managerSlots.Where(s => ReferenceEquals(completedManager, s.Manager)).ToList(); + if (!completedSlot.Any()) { - RemoveManager(managerInstance); + // yikes, we should have found it there } + + var slot = completedSlot.Single(); + slot.IsAvailable = true; + + return RunWorkInParallel(); } + } - s_instance = default; + /// + /// Remove and dispose a manager from concurrent list of manager. + /// + /// Manager to remove + public void RemoveManager(TManager manager) + { + _managerToEventHandlerMap.Remove(manager); + } + + /// + /// Add a manager in concurrent list of manager. + /// + /// Manager to add + /// eventHandler of the manager + public void AddManager(TManager manager, TEventHandler handler) + { + _managerToEventHandlerMap.Add(manager, handler); } - protected void DoActionOnAllManagers(Action action, bool doActionsInParallel = false) + public void DoActionOnAllManagers(Action action, bool doActionsInParallel = false) { - if (_concurrentManagerHandlerMap != null && _concurrentManagerHandlerMap.Count > 0) + lock (_lock) { int i = 0; - var actionTasks = new Task[_concurrentManagerHandlerMap.Count]; - foreach (var client in GetConcurrentManagerInstances()) + var actionTasks = new Task[_managerToEventHandlerMap.Count]; + foreach (var manager in _managerSlots.Select(s => s.Manager)) { + if (manager == null) + continue; + // Read the array before firing the task - beware of closures if (doActionsInParallel) { - actionTasks[i] = Task.Run(() => action(client)); + actionTasks[i] = Task.Run(() => action(manager)); i++; } else { - DoManagerAction(() => action(client)); + DoManagerAction(() => action(manager)); } } @@ -216,6 +186,7 @@ protected void DoActionOnAllManagers(Action action, bool doAct DoManagerAction(() => Task.WaitAll(actionTasks)); } } + } private void DoManagerAction(Action action) @@ -233,25 +204,37 @@ private void DoManagerAction(Action action) } } - /// - /// Fetches the next data object for the concurrent executor to work on - /// - /// source data to work on - source file or testCaseList - /// True, if data exists. False otherwise - protected bool TryFetchNextSource(IEnumerator enumerator, out TSource source) + internal void StopAllManagers() { - // TODO: If only something like a concurrent queue existed. - source = default; - var hasNext = false; - lock (_sourceEnumeratorLockObject) + ClearSlots(); + } + + public void Dispose() + { + ClearSlots(); + } + + private class Slot + { + public bool IsAvailable { get; set; } = true; + + public TManager? Manager { get; set; } + + public TestRuntimeProviderInfo? ManagerInfo { get; set; } + + public TEventHandler? EventHandler { get; set; } + + public TWorkload? Work { get; set; } + } + + private class SlotWorkloadPair + { + public SlotWorkloadPair(Slot slot, ProviderSpecificWorkload workload) { - if (enumerator != null && enumerator.MoveNext()) - { - source = (TSource)enumerator.Current; - hasNext = source != null; - } + Slot = slot; + Workload = workload; } - - return hasNext; + public Slot Slot { get; } + public ProviderSpecificWorkload Workload { get; } } } diff --git a/src/Microsoft.TestPlatform.CrossPlatEngine/Client/Parallel/ParallelProxyDiscoveryManager.cs b/src/Microsoft.TestPlatform.CrossPlatEngine/Client/Parallel/ParallelProxyDiscoveryManager.cs index e68df9e95f..c15cbc6946 100644 --- a/src/Microsoft.TestPlatform.CrossPlatEngine/Client/Parallel/ParallelProxyDiscoveryManager.cs +++ b/src/Microsoft.TestPlatform.CrossPlatEngine/Client/Parallel/ParallelProxyDiscoveryManager.cs @@ -17,24 +17,18 @@ namespace Microsoft.VisualStudio.TestPlatform.CrossPlatEngine.Client.Parallel; using Microsoft.VisualStudio.TestPlatform.ObjectModel.Client; using ObjectModel.Engine; using ObjectModel.Logging; -using Microsoft.VisualStudio.TestPlatform.CrossPlatEngine.Utilities; /// /// ParallelProxyDiscoveryManager that manages parallel discovery /// -internal class ParallelProxyDiscoveryManager : ParallelOperationManager, IParallelProxyDiscoveryManager +internal class ParallelProxyDiscoveryManager : IParallelProxyDiscoveryManager { private readonly IDataSerializer _dataSerializer; - private readonly Dictionary _sourceToSourceDetailMap; + private readonly ParallelOperationManager _parallelOperationManager; + private readonly Dictionary _sourceToTestHostProviderMap; private int _discoveryCompletedClients; private int _availableTestSources = -1; - private DiscoveryCriteria _actualDiscoveryCriteria; - - private IEnumerator _sourceEnumerator; - - private ITestDiscoveryEventsHandler2 _currentDiscoveryEventsHandler; - private ParallelDiscoveryDataAggregator _currentDiscoveryDataAggregator; private bool _skipDefaultAdapters; private readonly IRequestData _requestData; @@ -46,17 +40,28 @@ internal class ParallelProxyDiscoveryManager : ParallelOperationManager private readonly object _discoveryStatusLockObject = new(); - public ParallelProxyDiscoveryManager(IRequestData requestData, Func actualProxyManagerCreator, int parallelLevel, Dictionary sourceToSourceDetailMap) - : this(requestData, actualProxyManagerCreator, JsonDataSerializer.Instance, parallelLevel, sourceToSourceDetailMap) + public ParallelProxyDiscoveryManager( + IRequestData requestData, + Func actualProxyManagerCreator, + int parallelLevel, + List testHostProviders) + : this(requestData, actualProxyManagerCreator, JsonDataSerializer.Instance, parallelLevel, testHostProviders) { } - internal ParallelProxyDiscoveryManager(IRequestData requestData, Func actualProxyManagerCreator, IDataSerializer dataSerializer, int parallelLevel, Dictionary sourceToSourceDetailMap) - : base(actualProxyManagerCreator, parallelLevel) + internal ParallelProxyDiscoveryManager( + IRequestData requestData, + Func actualProxyManagerCreator, + IDataSerializer dataSerializer, + int parallelLevel, + List testHostProviders) { _requestData = requestData; _dataSerializer = dataSerializer; - _sourceToSourceDetailMap = sourceToSourceDetailMap; + _parallelOperationManager = new(actualProxyManagerCreator, parallelLevel); + _sourceToTestHostProviderMap = testHostProviders + .SelectMany(provider => provider.SourceDetails.Select(s => new KeyValuePair(s.Source, provider))) + .ToDictionary(pair => pair.Key, pair => pair.Value); } #region IProxyDiscoveryManager @@ -64,20 +69,14 @@ internal ParallelProxyDiscoveryManager(IRequestData requestData, Func public void Initialize(bool skipDefaultAdapters) { - // The parent ctor does not pre-create any managers, save the info for later. - // DoActionOnAllManagers((proxyManager) => proxyManager.Initialize(skipDefaultAdapters), doActionsInParallel: true); _skipDefaultAdapters = skipDefaultAdapters; } /// public void DiscoverTests(DiscoveryCriteria discoveryCriteria, ITestDiscoveryEventsHandler2 eventHandler) { - _actualDiscoveryCriteria = discoveryCriteria; - - // Set the enumerator for parallel yielding of sources - // Whenever a concurrent executor becomes free, it picks up the next source using this enumerator - _sourceEnumerator = discoveryCriteria.Sources.GetEnumerator(); - _availableTestSources = discoveryCriteria.Sources.Count(); + var workloads = SplitToWorkloads(discoveryCriteria, _sourceToTestHostProviderMap); + _availableTestSources = workloads.Count; EqtTrace.Verbose("ParallelProxyDiscoveryManager: Start discovery. Total sources: " + _availableTestSources); @@ -87,27 +86,37 @@ public void DiscoverTests(DiscoveryCriteria discoveryCriteria, ITestDiscoveryEve // Marking all sources as not discovered before starting actual discovery _currentDiscoveryDataAggregator.MarkSourcesWithStatus(discoveryCriteria.Sources.ToList(), DiscoveryStatus.NotDiscovered); - DiscoverTestsPrivate(eventHandler); + _parallelOperationManager.StartWork(workloads, eventHandler, GetParallelEventHandler, DiscoverTestsOnConcurrentManager); + } + + private ITestDiscoveryEventsHandler2 GetParallelEventHandler(ITestDiscoveryEventsHandler2 eventHandler, IProxyDiscoveryManager concurrentManager) + { + return new ParallelDiscoveryEventsHandler( + _requestData, + concurrentManager, + eventHandler, + this, + _currentDiscoveryDataAggregator); } /// public void Abort() { IsAbortRequested = true; - DoActionOnAllManagers((proxyManager) => proxyManager.Abort(), doActionsInParallel: true); + _parallelOperationManager.DoActionOnAllManagers((proxyManager) => proxyManager.Abort(), doActionsInParallel: true); } /// public void Abort(ITestDiscoveryEventsHandler2 eventHandler) { IsAbortRequested = true; - DoActionOnAllManagers((proxyManager) => proxyManager.Abort(eventHandler), doActionsInParallel: true); + _parallelOperationManager.DoActionOnAllManagers((proxyManager) => proxyManager.Abort(eventHandler), doActionsInParallel: true); } /// public void Close() { - DoActionOnAllManagers(proxyManager => proxyManager.Close(), doActionsInParallel: true); + _parallelOperationManager.DoActionOnAllManagers(proxyManager => proxyManager.Close(), doActionsInParallel: true); } #endregion @@ -139,90 +148,39 @@ when testhost crashed by itself and when user requested it (f.e. through TW) */ if (allDiscoverersCompleted || IsAbortRequested) { - // Reset enumerators - _sourceEnumerator = null; - - _currentDiscoveryDataAggregator = null; - _currentDiscoveryEventsHandler = null; - - // Dispose concurrent executors - UpdateParallelLevel(0); + _parallelOperationManager.StopAllManagers(); return true; } - // REVIEW: this was here before I did multi tfm work, this should be reviewed, because - // the comment builds on false premise, so maybe too much work is done here, because we should take shared hosts into account, and schedule - // the next source on the same manager if we have the possibility. - // and not kill testhost for every source. - /* Discovery is not complete. - Now when both.net framework and.net core projects can run in parallel - we should clear manager and create new one for both cases. - Otherwise `proxyDiscoveryManager` instance is alredy closed by now and it will give exception - when trying to do some operation on it. - */ - var SharedHosts = false; - EqtTrace.Verbose("ParallelProxyDiscoveryManager: HandlePartialDiscoveryComplete: Replace discovery manager. Shared: {0}, Aborted: {1}.", SharedHosts, isAborted); - - RemoveManager(proxyDiscoveryManager); - - // If we have more sources, create manager for that source. - // The source determines which type of host to create, because it can have a framework - // and architecture associated with it. - if (TryFetchNextSource(_sourceEnumerator, out string source)) - { - var sourceDetail = _sourceToSourceDetailMap[source]; - proxyDiscoveryManager = CreateNewConcurrentManager(sourceDetail); - var parallelEventsHandler = new ParallelDiscoveryEventsHandler( - _requestData, - proxyDiscoveryManager, - _currentDiscoveryEventsHandler, - this, - _currentDiscoveryDataAggregator); - AddManager(proxyDiscoveryManager, parallelEventsHandler); - } - - // REVIEW: is this really how it should be done? Proxy manager can be if we don't pass any and if we don't have more sources? - // Let's attempt to trigger discovery for the source. - DiscoverTestsOnConcurrentManager(source, proxyDiscoveryManager); + _parallelOperationManager.RunNextWork(proxyDiscoveryManager); return false; } #endregion - private void DiscoverTestsPrivate(ITestDiscoveryEventsHandler2 discoveryEventsHandler) + private List> SplitToWorkloads(DiscoveryCriteria discoveryCriteria, Dictionary sourceToTestHostProviderMap) { - _currentDiscoveryEventsHandler = discoveryEventsHandler; - - // Reset the discovery complete data - _discoveryCompletedClients = 0; + List> workloads = discoveryCriteria.Sources + .Select(source => new ProviderSpecificWorkload(NewDiscoveryCriteriaFromSource(source, discoveryCriteria), sourceToTestHostProviderMap[source])) + .ToList(); + return workloads; + } - // ERRR: GetConcurrentManagerInstances() was called before that touches proxyOperationManager, not CreateNewConcurrentManager directly - // is this still compatible? I guess I am skipping the testhost pool? Or am I getting the manager from the pool via a creator? - var parallel = 0; - while (parallel < MaxParallelLevel) - { - parallel++; - if (!TryFetchNextSource(_sourceEnumerator, out string source)) - { - break; - } - - var sourceDetail = _sourceToSourceDetailMap[source]; - var concurrentManager = CreateNewConcurrentManager(sourceDetail); + private DiscoveryCriteria NewDiscoveryCriteriaFromSource(string source, DiscoveryCriteria discoveryCriteria) + { + var criteria = new DiscoveryCriteria( + new[] { source }, + discoveryCriteria.FrequencyOfDiscoveredTestsEvent, + discoveryCriteria.DiscoveredTestEventTimeout, + discoveryCriteria.RunSettings + ); - var parallelEventsHandler = new ParallelDiscoveryEventsHandler( - _requestData, - concurrentManager, - discoveryEventsHandler, - this, - _currentDiscoveryDataAggregator); + criteria.TestCaseFilter = discoveryCriteria.TestCaseFilter; - UpdateHandlerForManager(concurrentManager, parallelEventsHandler); - DiscoverTestsOnConcurrentManager(source, concurrentManager); - } + return criteria; } /// @@ -230,35 +188,15 @@ private void DiscoverTestsPrivate(ITestDiscoveryEventsHandler2 discoveryEventsHa /// Each concurrent discoverer calls this method, once its completed working on previous data /// /// Proxy discovery manager instance. - private void DiscoverTestsOnConcurrentManager(string source, IProxyDiscoveryManager proxyDiscoveryManager) + private void DiscoverTestsOnConcurrentManager(IProxyDiscoveryManager proxyDiscoveryManager, ITestDiscoveryEventsHandler2 eventHandler, DiscoveryCriteria discoveryCriteria) { - if (source == null) - { - EqtTrace.Verbose("ProxyParallelDiscoveryManager: No sources available for discovery."); - return; - } - - EqtTrace.Verbose("ProxyParallelDiscoveryManager: Triggering test discovery for next source: {0}", source); - - var sourceDetail = _sourceToSourceDetailMap[source]; - var runsettingsXml = SourceDetailHelper.UpdateRunSettingsFromSourceDetail(_actualDiscoveryCriteria.RunSettings, sourceDetail); - - - var discoveryCriteria = new DiscoveryCriteria( - new[] { source }, - _actualDiscoveryCriteria.FrequencyOfDiscoveredTestsEvent, - _actualDiscoveryCriteria.DiscoveredTestEventTimeout, - runsettingsXml - ); - - discoveryCriteria.TestCaseFilter = _actualDiscoveryCriteria.TestCaseFilter; // Kick off another discovery task for the next source Task.Run(() => { EqtTrace.Verbose("ParallelProxyDiscoveryManager: Discovery started."); - proxyDiscoveryManager.DiscoverTests(discoveryCriteria, GetHandlerForGivenManager(proxyDiscoveryManager)); + proxyDiscoveryManager.DiscoverTests(discoveryCriteria, eventHandler); }) .ContinueWith(t => { @@ -267,7 +205,7 @@ private void DiscoverTestsOnConcurrentManager(string source, IProxyDiscoveryMana // discovery will not terminate EqtTrace.Error("ParallelProxyDiscoveryManager: Failed to trigger discovery. Exception: " + t.Exception); - var handler = GetHandlerForGivenManager(proxyDiscoveryManager); + var handler = eventHandler; var testMessagePayload = new TestMessagePayload { MessageLevel = TestMessageLevel.Error, Message = t.Exception.ToString() }; handler.HandleRawMessage(_dataSerializer.SerializePayload(MessageType.TestMessage, testMessagePayload)); handler.HandleLogMessage(TestMessageLevel.Error, t.Exception.ToString()); diff --git a/src/Microsoft.TestPlatform.CrossPlatEngine/Client/Parallel/ParallelProxyExecutionManager.cs b/src/Microsoft.TestPlatform.CrossPlatEngine/Client/Parallel/ParallelProxyExecutionManager.cs index 25c6a89cee..043c038885 100644 --- a/src/Microsoft.TestPlatform.CrossPlatEngine/Client/Parallel/ParallelProxyExecutionManager.cs +++ b/src/Microsoft.TestPlatform.CrossPlatEngine/Client/Parallel/ParallelProxyExecutionManager.cs @@ -6,7 +6,6 @@ namespace Microsoft.VisualStudio.TestPlatform.CrossPlatEngine.Client.Parallel; using System; -using System.Collections; using System.Collections.Generic; using System.Collections.ObjectModel; using System.Linq; @@ -23,15 +22,15 @@ namespace Microsoft.VisualStudio.TestPlatform.CrossPlatEngine.Client.Parallel; using Microsoft.VisualStudio.TestPlatform.ObjectModel.Client; using ObjectModel.Engine; using ObjectModel.Logging; -using Microsoft.VisualStudio.TestPlatform.CrossPlatEngine.Utilities; /// /// ParallelProxyExecutionManager that manages parallel execution /// -internal class ParallelProxyExecutionManager : ParallelOperationManager, IParallelProxyExecutionManager +internal class ParallelProxyExecutionManager : IParallelProxyExecutionManager { private readonly IDataSerializer _dataSerializer; - private readonly Dictionary _sourceToSourceDetailMap; + private readonly ParallelOperationManager _parallelOperationManager; + private readonly Dictionary _sourceToTestHostProviderMap; #region TestRunSpecificData @@ -40,17 +39,7 @@ internal class ParallelProxyExecutionManager : ParallelOperationManager _sourceEnumerator; - - private IEnumerator _testCaseListEnumerator; - - private bool _hasSpecificTestsRun; - - private ITestRunEventsHandler _currentRunEventsHandler; + private int _availableWorkloads = -1; private ParallelRunDataAggregator _currentRunDataAggregator; @@ -71,97 +60,73 @@ internal class ParallelProxyExecutionManager : ParallelOperationManager actualProxyManagerCreator, int parallelLevel, Dictionary sourceToSourceDetailMap) - : this(requestData, actualProxyManagerCreator, JsonDataSerializer.Instance, parallelLevel, sourceToSourceDetailMap) + public ParallelProxyExecutionManager( + IRequestData requestData, + Func actualProxyManagerCreator, + int parallelLevel, + List testHostProviders) + : this(requestData, actualProxyManagerCreator, JsonDataSerializer.Instance, parallelLevel, testHostProviders) { } - internal ParallelProxyExecutionManager(IRequestData requestData, Func actualProxyManagerCreator, IDataSerializer dataSerializer, int parallelLevel, Dictionary sourceToSourceDetailMap) - : base(actualProxyManagerCreator, parallelLevel) + internal ParallelProxyExecutionManager( + IRequestData requestData, + Func actualProxyManagerCreator, + IDataSerializer dataSerializer, + int parallelLevel, + List testHostProviders) { _requestData = requestData; _dataSerializer = dataSerializer; - _sourceToSourceDetailMap = sourceToSourceDetailMap; + _parallelOperationManager = new(actualProxyManagerCreator, parallelLevel); + _sourceToTestHostProviderMap = testHostProviders + .SelectMany(provider => provider.SourceDetails.Select(s => new KeyValuePair(s.Source, provider))) + .ToDictionary(pair => pair.Key, pair => pair.Value); } - #region IProxyExecutionManager - public void Initialize(bool skipDefaultAdapters) { _skipDefaultAdapters = skipDefaultAdapters; - // base class does not create any manager just save the info } public int StartTestRun(TestRunCriteria testRunCriteria, ITestRunEventsHandler eventHandler) { - _hasSpecificTestsRun = testRunCriteria.HasSpecificTests; - _actualTestRunCriteria = testRunCriteria; + var workloads = SplitToWorkloads(testRunCriteria, _sourceToTestHostProviderMap); + _availableWorkloads = workloads.Count; - if (_hasSpecificTestsRun) - { - var testCasesBySource = new Dictionary>(); - foreach (var test in testRunCriteria.Tests) - { - if (!testCasesBySource.ContainsKey(test.Source)) - { - testCasesBySource.Add(test.Source, new List()); - } + EqtTrace.Verbose("ParallelProxyExecutionManager: Start execution. Total sources: " + _availableWorkloads); - testCasesBySource[test.Source].Add(test); - } + // Reset the run complete data + _runCompletedClients = 0; - // Do not use "Dictionary.ValueCollection.Enumerator" - it becomes nondeterministic once we go out of scope of this method - // Use "ToArray" to copy ValueColleciton to a simple array and use it's enumerator - // Set the enumerator for parallel yielding of testCases - // Whenever a concurrent executor becomes free, it picks up the next set of testCases using this enumerator - var testCaseLists = testCasesBySource.Values.ToArray(); - _testCaseListEnumerator = testCaseLists.GetEnumerator(); - _availableTestSources = testCaseLists.Length; - } - else - { - // Set the enumerator for parallel yielding of sources - // Whenever a concurrent executor becomes free, it picks up the next source using this enumerator - _sourceEnumerator = testRunCriteria.Sources.GetEnumerator(); - _availableTestSources = testRunCriteria.Sources.Count(); - } + // One data aggregator per parallel run + _currentRunDataAggregator = new ParallelRunDataAggregator(testRunCriteria.TestRunSettings); - EqtTrace.Verbose("ParallelProxyExecutionManager: Start execution. Total sources: " + _availableTestSources); + _parallelOperationManager.StartWork(workloads, eventHandler, GetParallelEventHandler, StartTestRunOnConcurrentManager); - return StartTestRunPrivate(eventHandler); + // Why 1? Because this is supposed to be a processId, and that is just the default that was chosen by someone before me, + // and maybe is checked somewhere, but I don't see it checked in our codebase. + return 1; } public void Abort(ITestRunEventsHandler runEventsHandler) { // Test platform initiated abort. _abortRequested = true; - DoActionOnAllManagers((proxyManager) => proxyManager.Abort(runEventsHandler), doActionsInParallel: true); + _parallelOperationManager.DoActionOnAllManagers((proxyManager) => proxyManager.Abort(runEventsHandler), doActionsInParallel: true); } public void Cancel(ITestRunEventsHandler runEventsHandler) { - DoActionOnAllManagers((proxyManager) => proxyManager.Cancel(runEventsHandler), doActionsInParallel: true); + _parallelOperationManager.DoActionOnAllManagers((proxyManager) => proxyManager.Cancel(runEventsHandler), doActionsInParallel: true); } public void Close() { - DoActionOnAllManagers(proxyManager => proxyManager.Close(), doActionsInParallel: true); + _parallelOperationManager.DoActionOnAllManagers(proxyManager => proxyManager.Close(), doActionsInParallel: true); } - #endregion - - #region IParallelProxyExecutionManager methods - - /// - /// Handles Partial Run Complete event coming from a specific concurrent proxy execution manager - /// Each concurrent proxy execution manager will signal the parallel execution manager when its complete - /// - /// Concurrent Execution manager that completed the run - /// RunCompleteArgs for the concurrent run - /// LastChunk testresults for the concurrent run - /// RunAttachments for the concurrent run - /// ExecutorURIs of the adapters involved in executing the tests - /// True if parallel run is complete + /// public bool HandlePartialRunComplete( IProxyExecutionManager proxyExecutionManager, TestRunCompleteEventArgs testRunCompleteArgs, @@ -179,169 +144,127 @@ public bool HandlePartialRunComplete( allRunsCompleted = testRunCompleteArgs.IsCanceled || _abortRequested ? _runCompletedClients == _runStartedClients - : _runCompletedClients == _availableTestSources; + : _runCompletedClients == _availableWorkloads; EqtTrace.Verbose("ParallelProxyExecutionManager: HandlePartialRunComplete: Total completed clients = {0}, Run complete = {1}, Run canceled: {2}.", _runCompletedClients, allRunsCompleted, testRunCompleteArgs.IsCanceled); } - // verify that all executors are done with the execution and there are no more sources/testcases to execute if (allRunsCompleted) { - // Reset enumerators - _sourceEnumerator = null; - _testCaseListEnumerator = null; - - _currentRunDataAggregator = null; - _currentRunEventsHandler = null; - - // Dispose concurrent executors - // Do not do the cleanup task in the current thread as we will unnecessarily add to execution time - UpdateParallelLevel(0); - + _parallelOperationManager.StopAllManagers(); return true; } - var SharedHosts = false; - EqtTrace.Verbose("ParallelProxyExecutionManager: HandlePartialRunComplete: Replace execution manager. Shared: {0}, Aborted: {1}.", SharedHosts, testRunCompleteArgs.IsAborted); - // TODO: this should not always happen, this should happen only if we are going to the next non-shared host, but is has been like this forever, so be cautions when changing it. - // Maybe it is not as safe to re-use host for execution. - RemoveManager(proxyExecutionManager); - - TestRunCriteria testRunCriteria = null; - string source = null; - if (!_hasSpecificTestsRun) - { - if (TryFetchNextSource(_sourceEnumerator, out source)) - { - EqtTrace.Info("ProxyParallelExecutionManager: Triggering test run for next source: {0}", source); - testRunCriteria = new TestRunCriteria(new[] { source }, _actualTestRunCriteria); - } - } - else - { - if (TryFetchNextSource(_testCaseListEnumerator, out List nextSetOfTests)) - { - // TODO: How could this not have a value? None of the code I found should actually fail when this is null. - // We will just mix results from multiple sources. But I doubt we would get here if we had a testcase without path. - source = nextSetOfTests?.FirstOrDefault()?.Source; - EqtTrace.Info("ProxyParallelExecutionManager: Triggering test run for next source: {0}", source); - testRunCriteria = new TestRunCriteria(nextSetOfTests, _actualTestRunCriteria); - } - } - - if (testRunCriteria == null) - { - // We most likely have nothing to run, but let's still return false. This way we let the logic on the top - // of this method to decide if we are done with the run or not. - return false; - } - - // We have something more to run. - var sourceDetail = _sourceToSourceDetailMap[source]; - proxyExecutionManager = CreateNewConcurrentManager(sourceDetail); - var parallelEventsHandler = GetEventsHandler(proxyExecutionManager); - AddManager(proxyExecutionManager, parallelEventsHandler); - // If cancel is triggered for any one run or abort is requested by test platform, there is no reason to fetch next source - // and queue another test run + // and queue another test run. if (!testRunCompleteArgs.IsCanceled && !_abortRequested) { - StartTestRunOnConcurrentManager(proxyExecutionManager, testRunCriteria); + var hadMoreWork = _parallelOperationManager.RunNextWork(proxyExecutionManager); + if (!hadMoreWork) + { + return true; + } } return false; } - #endregion - - private int StartTestRunPrivate(ITestRunEventsHandler runEventsHandler) + /// + /// Split the incoming work into smaller workloads that we can run on different testhosts. + /// Each workload is associated with a type of provider that can run it. + /// + /// + /// + /// + private List> SplitToWorkloads(TestRunCriteria testRunCriteria, Dictionary sourceToTestHostProviderMap) { - _currentRunEventsHandler = runEventsHandler; - - // Reset the run complete data - _runCompletedClients = 0; - - // One data aggregator per parallel run - _currentRunDataAggregator = new ParallelRunDataAggregator(_actualTestRunCriteria.TestRunSettings); - - // Create as many handlers as we can, until we reach the max parallel level (this is number of logical processors, or the number of sources, or user - // preference. Originally this was done in the UpdateParallelLevel in base constructor, but we did not know which source will be the next there. So we could not - // start the manager specifically for the source with the correct tfm and platform. - var parallel = 0; - while (parallel < MaxParallelLevel) + // We split the work to workloads that will run on each testhost, we and add all of them + // to a bag of work that needs to be processed. (The workloads are just + // a single source, or all test cases for a given source.) + // + // For every workload we associated a given type of testhost that can run the work. + // This is important when we have shared testhosts. A shared testhost can re-use the same process + // to run more than one workload, as long as the provider is the same. + // + // We then start as many instances of testhost as we are allowed by parallel level, + // and we start sending them work. Once any testhost is done processing a given workload, + // we will get notified with the completed event for the work we are doing. For example for StartTestRun + // we will get TestExecutionCompleted, and we will call HandlePartialTestExecutionComplete. + // (The "partial" here refers to posibly having more work in the work bag. It does not mean that + // there was an error in the testhost and we only did part of execution.). + // + // At that point we know that at least one testhost is not busy doing work anymore. It either + // processed the workload and waits for another one, or it maybe crashed and we should move to + // another source. + // + // In the "partial" step we check if we have more workloads, and if the currently running testhost + // is shared we try to find a workload that is appropriate for it. If we don't find any work that the + // running testhost can do. Or if the testhost already exited (or maybe crashed), we start another one + // and give it the next workload. + List> workloads = new(); + if (testRunCriteria.HasSpecificTests) { - parallel++; - TestRunCriteria testRunCriteria = null; - string source = null; - if (!_hasSpecificTestsRun) + // We split test cases to their respective sources, and associate them with additional info about on + // which type of provider they can run so we can later select the correct workload for the provider + // if we already have a provider running, and it is shared. + var testCasesPerSource = testRunCriteria.Tests.GroupBy(t => t.Source); + foreach (var group in testCasesPerSource) { - if (TryFetchNextSource(_sourceEnumerator, out source)) - { - EqtTrace.Info("ProxyParallelExecutionManager: Triggering test run for next source: {0}", source); - var sourceDetail2 = _sourceToSourceDetailMap[source]; - var runSettingsXml = SourceDetailHelper.UpdateRunSettingsFromSourceDetail(_actualTestRunCriteria.TestRunSettings, sourceDetail2); - - // Copy the test run criteria, but use a single source, and updated runsettings. - testRunCriteria = new TestRunCriteria(new[] { source }, // <- - _actualTestRunCriteria.FrequencyOfRunStatsChangeEvent, - _actualTestRunCriteria.KeepAlive, - runSettingsXml, // <- - _actualTestRunCriteria.RunStatsChangeEventTimeout, - _actualTestRunCriteria.TestHostLauncher, - _actualTestRunCriteria.TestCaseFilter, - _actualTestRunCriteria.FilterOptions, - _actualTestRunCriteria.TestSessionInfo, - _actualTestRunCriteria.DebugEnabledForTestSession); - } - } - else - { - if (TryFetchNextSource(_testCaseListEnumerator, out List nextSetOfTests)) - { - source = nextSetOfTests?.FirstOrDefault()?.Source; - - var sourceDetail2 = _sourceToSourceDetailMap[source]; - var runSettingsXml = SourceDetailHelper.UpdateRunSettingsFromSourceDetail(_actualTestRunCriteria.TestRunSettings, sourceDetail2); - - EqtTrace.Info("ProxyParallelExecutionManager: Triggering test run for next source: {0}", source); - // Copy the test run criteria, but use the next set of tests, and updated runsettings. - testRunCriteria = new TestRunCriteria(nextSetOfTests, // <- - _actualTestRunCriteria.FrequencyOfRunStatsChangeEvent, - _actualTestRunCriteria.KeepAlive, - runSettingsXml, // <- - _actualTestRunCriteria.RunStatsChangeEventTimeout, - _actualTestRunCriteria.TestHostLauncher, - _actualTestRunCriteria.TestSessionInfo, - _actualTestRunCriteria.DebugEnabledForTestSession); - } + var testHostProviderInfo = sourceToTestHostProviderMap[group.Key]; + var runsettings = testHostProviderInfo.RunSettings; + var testCases = group.ToList(); + var updatedCriteria = CreateTestRunCriteriaFromTestCasesAndSettings(testCases, testRunCriteria, runsettings); + var workload = new ProviderSpecificWorkload(updatedCriteria, testHostProviderInfo); + workloads.Add(workload); } - if (source == null) + } + else + { + // We associate every source with additional info about on which type of provider it can run so we can later + // select the correct workload for the provider if we already have a provider running, and it is shared. + foreach (var source in testRunCriteria.Sources) { - // Why 1? Because this is supposed to be a processId, and that is just the default that was chosen, - // and maybe is checked somewhere, but I don't see it checked in our codebase. - return 1; + var testHostProviderInfo = sourceToTestHostProviderMap[source]; + var runsettings = testHostProviderInfo.RunSettings; + var updatedCriteria = CreateTestRunCriteriaFromSourceAndSettings(new[] { source }, testRunCriteria, runsettings); + var workload = new ProviderSpecificWorkload(updatedCriteria, testHostProviderInfo); + workloads.Add(workload); } + } - var sourceDetail = _sourceToSourceDetailMap[source]; - var concurrentManager = CreateNewConcurrentManager(sourceDetail); - - - var parallelEventsHandler = GetEventsHandler(concurrentManager); - UpdateHandlerForManager(concurrentManager, parallelEventsHandler); - if (!concurrentManager.IsInitialized) - { - concurrentManager.Initialize(_skipDefaultAdapters); - } - IsInitialized = true; + return workloads; - StartTestRunOnConcurrentManager(concurrentManager, testRunCriteria); + TestRunCriteria CreateTestRunCriteriaFromTestCasesAndSettings(IEnumerable testCases, TestRunCriteria criteria, string runsettingsXml) + { + return new TestRunCriteria( + testCases, + testRunCriteria.FrequencyOfRunStatsChangeEvent, + testRunCriteria.KeepAlive, + runsettingsXml, + testRunCriteria.RunStatsChangeEventTimeout, + testRunCriteria.TestHostLauncher, + testRunCriteria.TestSessionInfo, + testRunCriteria.DebugEnabledForTestSession); } - return 1; + TestRunCriteria CreateTestRunCriteriaFromSourceAndSettings(IEnumerable sources, TestRunCriteria criteria, string runsettingsXml) + { + return new TestRunCriteria( + sources, + testRunCriteria.FrequencyOfRunStatsChangeEvent, + testRunCriteria.KeepAlive, + runsettingsXml, + testRunCriteria.RunStatsChangeEventTimeout, + testRunCriteria.TestHostLauncher, + testRunCriteria.TestCaseFilter, + testRunCriteria.FilterOptions, + testRunCriteria.TestSessionInfo, + testRunCriteria.DebugEnabledForTestSession); + } } - private ParallelRunEventsHandler GetEventsHandler(IProxyExecutionManager concurrentManager) + private ParallelRunEventsHandler GetParallelEventHandler(ITestRunEventsHandler eventHandler, IProxyExecutionManager concurrentManager) { if (concurrentManager is ProxyExecutionManagerWithDataCollection) { @@ -351,7 +274,7 @@ private ParallelRunEventsHandler GetEventsHandler(IProxyExecutionManager concurr return new ParallelDataCollectionEventsHandler( _requestData, concurrentManagerWithDataCollection, - _currentRunEventsHandler, + eventHandler, this, _currentRunDataAggregator, attachmentsProcessingManager, @@ -361,7 +284,7 @@ private ParallelRunEventsHandler GetEventsHandler(IProxyExecutionManager concurr return new ParallelRunEventsHandler( _requestData, concurrentManager, - _currentRunEventsHandler, + eventHandler, this, _currentRunDataAggregator); } @@ -372,7 +295,7 @@ private ParallelRunEventsHandler GetEventsHandler(IProxyExecutionManager concurr /// /// Proxy execution manager instance. /// True, if execution triggered - private void StartTestRunOnConcurrentManager(IProxyExecutionManager proxyExecutionManager, TestRunCriteria testRunCriteria) + private void StartTestRunOnConcurrentManager(IProxyExecutionManager proxyExecutionManager, ITestRunEventsHandler eventHandler, TestRunCriteria testRunCriteria) { if (testRunCriteria != null) { @@ -386,7 +309,7 @@ private void StartTestRunOnConcurrentManager(IProxyExecutionManager proxyExecuti Interlocked.Increment(ref _runStartedClients); EqtTrace.Verbose("ParallelProxyExecutionManager: Execution started. Started clients: " + _runStartedClients); - proxyExecutionManager.StartTestRun(testRunCriteria, GetHandlerForGivenManager(proxyExecutionManager)); + proxyExecutionManager.StartTestRun(testRunCriteria, eventHandler); }) .ContinueWith(t => { @@ -395,7 +318,7 @@ private void StartTestRunOnConcurrentManager(IProxyExecutionManager proxyExecuti // execution will not terminate EqtTrace.Error("ParallelProxyExecutionManager: Failed to trigger execution. Exception: " + t.Exception); - var handler = GetHandlerForGivenManager(proxyExecutionManager); + var handler = eventHandler; var testMessagePayload = new TestMessagePayload { MessageLevel = TestMessageLevel.Error, Message = t.Exception.ToString() }; handler.HandleRawMessage(_dataSerializer.SerializePayload(MessageType.TestMessage, testMessagePayload)); handler.HandleLogMessage(TestMessageLevel.Error, t.Exception.ToString()); @@ -414,3 +337,28 @@ private void StartTestRunOnConcurrentManager(IProxyExecutionManager proxyExecuti EqtTrace.Verbose("ProxyParallelExecutionManager: No sources available for execution."); } } + +/// +/// A workload with a specification of a provider that can run that workload. The workload is a list of sources, +/// or a list of testcases. Provider is a given testhost manager, that is capable of running this workload, so +/// we end up running .NET sources on .NET testhost, and .NET Framework sources on .NET Framework provider. +/// +internal class ProviderSpecificWorkload : ProviderSpecificWorkload +{ + public T Work { get; } + + public ProviderSpecificWorkload(T work, TestRuntimeProviderInfo provider) + { + Provider = provider; + Work = work; + } +} + +/// +/// A "marker" for ProviderSpecificWorkload of T, so we can have a collection that either holds test cases, +/// or just sources and still know which provider they require. +/// +internal abstract class ProviderSpecificWorkload +{ + public TestRuntimeProviderInfo Provider { get; protected set; } +} diff --git a/src/Microsoft.TestPlatform.CrossPlatEngine/Client/TestRuntimeProviderInfo.cs b/src/Microsoft.TestPlatform.CrossPlatEngine/Client/TestRuntimeProviderInfo.cs new file mode 100644 index 0000000000..449b828beb --- /dev/null +++ b/src/Microsoft.TestPlatform.CrossPlatEngine/Client/TestRuntimeProviderInfo.cs @@ -0,0 +1,25 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +namespace Microsoft.VisualStudio.TestPlatform.CrossPlatEngine.Client; + +using System; +using System.Collections.Generic; + +using Microsoft.VisualStudio.TestPlatform.ObjectModel; + +public class TestRuntimeProviderInfo +{ + public Type Type { get; } + public bool Shared { get; } + public string RunSettings { get; } + public List SourceDetails { get; } + + public TestRuntimeProviderInfo(Type type, bool shared, string runSettings, List sourceDetails) + { + Type = type; + Shared = shared; + RunSettings = runSettings; + SourceDetails = sourceDetails; + } +} diff --git a/src/Microsoft.TestPlatform.CrossPlatEngine/TestEngine.cs b/src/Microsoft.TestPlatform.CrossPlatEngine/TestEngine.cs index af2d1960b4..acd6df5e73 100644 --- a/src/Microsoft.TestPlatform.CrossPlatEngine/TestEngine.cs +++ b/src/Microsoft.TestPlatform.CrossPlatEngine/TestEngine.cs @@ -28,6 +28,8 @@ namespace Microsoft.VisualStudio.TestPlatform.CrossPlatEngine; using PlatformAbstractions; using PlatformAbstractions.Interfaces; using Microsoft.VisualStudio.TestPlatform.Utilities; +using System.Text; +using Microsoft.VisualStudio.TestPlatform.Common.Interfaces; /// /// Cross platform test engine entry point for the client. @@ -79,41 +81,34 @@ public IProxyDiscoveryManager GetDiscoveryManager( requestData.MetricsCollection.Add(TelemetryDataConstants.ParallelEnabledDuringDiscovery, isParallelRun ? "True" : "False"); requestData.MetricsCollection.Add(TelemetryDataConstants.TestSessionId, discoveryCriteria.TestSessionInfo?.Id.ToString() ?? string.Empty); + // Get testhost managers by configuration, and either use it for in-process run. or for single source run. + List testHostManagers = GetTestRuntimeProvidersForUniqueConfigurations(discoveryCriteria.RunSettings, sourceToSourceDetailMap, out ITestRuntimeProvider testHostManager); + // This is a big if that figures out if we can run in process. In process run is very restricted, it is non-parallel run // that has the same target framework as the current process, and it also must not be running in DesignMode (server mode / under IDE) // and more conditions. In all other cases we run in a separate testhost process. - if (ShouldRunInProcess(discoveryCriteria.RunSettings, isParallelRun, isDataCollectorEnabled: false, sourceToSourceDetailMap)) + if (ShouldRunInProcess(discoveryCriteria.RunSettings, isParallelRun, isDataCollectorEnabled: false, testHostManagers)) { - // Get testhost manager by configuration, and either use it for in-process run. or for single source run. - // If there are more sources throw it away. We cannot avoid creating this manager because we don't know if it will be shared or not - // and we don't have any metadata for that info. - // We are running in process, so whatever the architecture and framework that was figured out is, it must be compatible. If we have more // changes that we want to to runsettings in the future, based on SourceDetail then it will depend on those details. But in general // we will have to check that all source details are the same. Otherwise we for sure cannot run in process. // E.g. if we get list of sources where one of them has different architecture we for sure cannot run in process, because the current // process can handle only single runsettings. - var runSettingsXml = discoveryCriteria.RunSettings; - ITestRuntimeProvider testHostManager = _testHostProviderManager.GetTestHostManagerByRunConfiguration(runSettingsXml); - ThrowExceptionIfTestHostManagerIsNull(testHostManager, runSettingsXml); - testHostManager.Initialize(TestSessionMessageLogger.Instance, runSettingsXml); + var testHostManagerInfo = testHostManagers.Single(); + testHostManager.Initialize(TestSessionMessageLogger.Instance, testHostManagerInfo.RunSettings); var isTelemetryOptedIn = requestData.IsTelemetryOptedIn; var newRequestData = GetRequestData(isTelemetryOptedIn); return new InProcessProxyDiscoveryManager(testHostManager, new TestHostManagerFactory(newRequestData)); } - Func proxyDiscoveryManagerCreator = sourceDetail => + Func proxyDiscoveryManagerCreator = runtimeProviderInfo => { - // Update runsettings to have the correct architecture and framework for the current source, - // we are (potentially) running with multiple target frameworks or multiple architectures. - // And this allows the ITestRuntimeProvider to have no idea about that, because it only needs to know about run settings for the source it is about to run. - // TODO: No idea why I did not actually patch the settings when I infer them, because then everyone along the path will know about the same runsettings. But at least here I know - // that I am making changes on the most up-to-date settings. - var runSettingsXml = SourceDetailHelper.UpdateRunSettingsFromSourceDetail(discoveryCriteria.RunSettings, sourceDetail); + var sources = runtimeProviderInfo.SourceDetails.Select(r => r.Source).ToList(); + var hostManager = _testHostProviderManager.GetTestHostManagerByRunConfiguration(runtimeProviderInfo.RunSettings, sources); + hostManager?.Initialize(TestSessionMessageLogger.Instance, runtimeProviderInfo.RunSettings); - var hostManager = _testHostProviderManager.GetTestHostManagerByRunConfiguration(runSettingsXml); - hostManager?.Initialize(TestSessionMessageLogger.Instance, runSettingsXml); + ThrowExceptionIfTestHostManagerIsNull(hostManager, runtimeProviderInfo.RunSettings); // This function is used to either take a pre-existing proxy operation manager from // the test pool or to create a new proxy operation manager on the spot. @@ -127,7 +122,7 @@ public IProxyDiscoveryManager GetDiscoveryManager( var proxyOperationManager = TestSessionPool.Instance.TryTakeProxy( discoveryCriteria.TestSessionInfo, source, - runSettingsXml); + runtimeProviderInfo.RunSettings); if (proxyOperationManager == null) { @@ -161,7 +156,7 @@ public IProxyDiscoveryManager GetDiscoveryManager( hostManager); }; - return new ParallelProxyDiscoveryManager(requestData, proxyDiscoveryManagerCreator, parallelLevel, sourceToSourceDetailMap); + return new ParallelProxyDiscoveryManager(requestData, proxyDiscoveryManagerCreator, parallelLevel, testHostManagers); } /// @@ -187,15 +182,18 @@ public IProxyExecutionManager GetExecutionManager( var isDataCollectorEnabled = XmlRunSettingsUtilities.IsDataCollectionEnabled(testRunCriteria.TestRunSettings); var isInProcDataCollectorEnabled = XmlRunSettingsUtilities.IsInProcDataCollectionEnabled(testRunCriteria.TestRunSettings); + var testHostProviders = GetTestRuntimeProvidersForUniqueConfigurations(testRunCriteria.TestRunSettings, sourceToSourceDetailMap, out ITestRuntimeProvider testHostManager); + if (ShouldRunInProcess( testRunCriteria.TestRunSettings, isParallelRun, isDataCollectorEnabled || isInProcDataCollectorEnabled, - sourceToSourceDetailMap)) + testHostProviders)) { // Not updating runsettings from source detail on purpose here. We are running in process, so whatever the settings we figured out at the start. They must be compatible // with the current process, otherwise we would not be able to run inside of the current process. - ITestRuntimeProvider testHostManager = _testHostProviderManager.GetTestHostManagerByRunConfiguration(testRunCriteria.TestRunSettings); + // + // We know that we only have a single testHostManager here, because we figure that out in ShouldRunInProcess. ThrowExceptionIfTestHostManagerIsNull(testHostManager, testRunCriteria.TestRunSettings); testHostManager.Initialize(TestSessionMessageLogger.Instance, testRunCriteria.TestRunSettings); @@ -215,13 +213,14 @@ public IProxyExecutionManager GetExecutionManager( // SetupChannel ProxyExecutionManager with data collection if data collectors are // specified in run settings. - Func proxyExecutionManagerCreator = sourceDetail => + Func proxyExecutionManagerCreator = runtimeProviderInfo => { // Create a new host manager, to be associated with individual // ProxyExecutionManager(&POM) - string runSettingsXml = SourceDetailHelper.UpdateRunSettingsFromSourceDetail(testRunCriteria.TestRunSettings, sourceDetail); - var hostManager = _testHostProviderManager.GetTestHostManagerByRunConfiguration(runSettingsXml); - hostManager?.Initialize(TestSessionMessageLogger.Instance, runSettingsXml); + var sources = runtimeProviderInfo.SourceDetails.Select(r => r.Source).ToList(); + var hostManager = _testHostProviderManager.GetTestHostManagerByRunConfiguration(runtimeProviderInfo.RunSettings, sources); + ThrowExceptionIfTestHostManagerIsNull(hostManager, runtimeProviderInfo.RunSettings); + hostManager.Initialize(TestSessionMessageLogger.Instance, runtimeProviderInfo.RunSettings); if (testRunCriteria.TestHostLauncher != null) { @@ -242,7 +241,7 @@ public IProxyExecutionManager GetExecutionManager( var proxyOperationManager = TestSessionPool.Instance.TryTakeProxy( testRunCriteria.TestSessionInfo, source, - runSettingsXml); + runtimeProviderInfo.RunSettings); if (proxyOperationManager == null) { @@ -281,7 +280,7 @@ public IProxyExecutionManager GetExecutionManager( hostManager, new ProxyDataCollectionManager( requestData, - runSettingsXml, + runtimeProviderInfo.RunSettings, GetSourcesFromTestRunCriteria(testRunCriteria))) : new ProxyExecutionManager( requestData, @@ -289,7 +288,7 @@ public IProxyExecutionManager GetExecutionManager( hostManager); }; - var executionManager = new ParallelProxyExecutionManager(requestData, proxyExecutionManagerCreator, parallelLevel, sourceToSourceDetailMap); + var executionManager = new ParallelProxyExecutionManager(requestData, proxyExecutionManagerCreator, parallelLevel, testHostProviders); EqtTrace.Verbose($"TestEngine.GetExecutionManager: Chosen execution manager '{executionManager.GetType().AssemblyQualifiedName}' ParallelLevel '{parallelLevel}'."); @@ -306,37 +305,40 @@ public IProxyTestSessionManager GetTestSessionManager( testSessionCriteria.Sources.Count, testSessionCriteria.RunSettings); + bool isParallelRun = parallelLevel > 1; requestData.MetricsCollection.Add( TelemetryDataConstants.ParallelEnabledDuringStartTestSession, - parallelLevel > 1 ? "True" : "False"); + isParallelRun ? "True" : "False"); var isDataCollectorEnabled = XmlRunSettingsUtilities.IsDataCollectionEnabled(testSessionCriteria.RunSettings); var isInProcDataCollectorEnabled = XmlRunSettingsUtilities.IsInProcDataCollectionEnabled(testSessionCriteria.RunSettings); + List testRuntimeProviders = GetTestRuntimeProvidersForUniqueConfigurations(testSessionCriteria.RunSettings, sourceToSourceDetailMap, out var _); + if (ShouldRunInProcess( testSessionCriteria.RunSettings, - parallelLevel > 1, + isParallelRun, isDataCollectorEnabled || isInProcDataCollectorEnabled, - sourceToSourceDetailMap)) + testRuntimeProviders)) { - // This condition is the equivalent of the in-process proxy execution manager case. - // In this case all tests will be run in the vstest.console process, so there's no - // test host to be started. As a consequence there'll be no session info. + // In this case all tests will be run in the current process (vstest.console), so there is no + // testhost to pre-start. No session will be created, and the session info will be null. return null; } - Func proxyCreator = () => + Func proxyCreator = testRuntimeProviderInfo => { - var hostManager = _testHostProviderManager.GetTestHostManagerByRunConfiguration(testSessionCriteria.RunSettings); - if (hostManager == null) + var sources = testRuntimeProviderInfo.SourceDetails.Select(x => x.Source).ToList(); + var hostManager = _testHostProviderManager.GetTestHostManagerByRunConfiguration(testRuntimeProviderInfo.RunSettings, sources); + + if (hostManager.GetType() != testRuntimeProviderInfo.Type) { - throw new TestPlatformException( - string.Format( - CultureInfo.CurrentCulture, - Resources.Resources.NoTestHostProviderFound)); + throw new Exception("hostmanager type is not the same, why?"); } - hostManager.Initialize(TestSessionMessageLogger.Instance, testSessionCriteria.RunSettings); + ThrowExceptionIfTestHostManagerIsNull(hostManager, testRuntimeProviderInfo.RunSettings); + + hostManager.Initialize(TestSessionMessageLogger.Instance, testRuntimeProviderInfo.RunSettings); if (testSessionCriteria.TestHostLauncher != null) { hostManager.SetCustomLauncher(testSessionCriteria.TestHostLauncher); @@ -365,7 +367,7 @@ public IProxyTestSessionManager GetTestSessionManager( // hostManager, // new ProxyDataCollectionManager( // requestData, - // testSessionCriteria.RunSettings, + // runsettingsXml, // testSessionCriteria.Sources)) // { // CloseRequestSenderChannelOnProxyClose = true @@ -376,13 +378,40 @@ public IProxyTestSessionManager GetTestSessionManager( hostManager); }; - var testhostManager = _testHostProviderManager.GetTestHostManagerByRunConfiguration(testSessionCriteria.RunSettings); - testhostManager.Initialize(TestSessionMessageLogger.Instance, testSessionCriteria.RunSettings); - var testhostCount = (parallelLevel > 1 || !testhostManager.Shared) - ? testSessionCriteria.Sources.Count - : 1; + // TODO: This condition should be returning the maxParallel level to avoid pre-starting way too many testhosts, because maxParallel level, + // can be smaller than the number of sources to run. + var maxTesthostCount = isParallelRun ? testSessionCriteria.Sources.Count : 1; + + return new ProxyTestSessionManager(testSessionCriteria, maxTesthostCount, proxyCreator, testRuntimeProviders); + } + + private List GetTestRuntimeProvidersForUniqueConfigurations( + string runSettings, + Dictionary sourceToSourceDetailMap, + out ITestRuntimeProvider mostRecentlyCreatedInstance) + { + // Group source details to get unique frameworks and architectures for which we will run, so we can figure + // out which runtime providers would run them, and if the runtime provider is shared or not. + mostRecentlyCreatedInstance = null; + var testRuntimeProviders = new List(); + var uniqueRunConfigurations = sourceToSourceDetailMap.Values.GroupBy(k => new { k.Framework, k.Architecture }); + foreach (var runConfiguration in uniqueRunConfigurations) + { + // It is okay to take the first (or any) source detail in the group. We are grouping to get the same source detail, so all architectures and frameworks are the same. + var sourceDetail = runConfiguration.First(); + var runsettingsXml = SourceDetailHelper.UpdateRunSettingsFromSourceDetail(runSettings, sourceDetail); + var sources = runConfiguration.Select(c => c.Source).ToList(); + // TODO: We could improve the implementation by adding an overload that won't create a new instance always, because we only need to know the Type. + var testRuntimeProvider = _testHostProviderManager.GetTestHostManagerByRunConfiguration(runsettingsXml, sources); + var testRuntimeProviderInfo = new TestRuntimeProviderInfo(testRuntimeProvider.GetType(), testRuntimeProvider.Shared, runsettingsXml, sourceDetails: runConfiguration.ToList()); + + // Outputting the instance, because the code for in-process run uses it, and we don't want to resolve a + mostRecentlyCreatedInstance = testRuntimeProvider; + testRuntimeProviders.Add(testRuntimeProviderInfo); + } - return new ProxyTestSessionManager(testSessionCriteria, testhostCount, proxyCreator); + ThrowExceptionIfAnyTestHostManagerIsNull(testRuntimeProviders); + return testRuntimeProviders; } /// @@ -471,15 +500,11 @@ private bool ShouldRunInProcess( string runsettings, bool isParallelEnabled, bool isDataCollectorEnabled, - Dictionary sourceToSourceDetailMap) + List testHostProviders) { - var allUseSameFramework = sourceToSourceDetailMap.Values.Select(detail => detail.Framework).Distinct().Count() == 1; - var allUseSameArchitecture = sourceToSourceDetailMap.Values.Select(detail => detail.Architecture).Distinct().Count() == 1; - var isMixedArchitectureOrFrameworkRun = !allUseSameFramework || !allUseSameArchitecture; - - if (isMixedArchitectureOrFrameworkRun) + if (testHostProviders.Count > 1) { - EqtTrace.Info("TestEngine.ShouldRunInNoIsolation: This is a run that mixes sources that have different architectures or frameworks, running in isolation (in a separate testhost proces)."); + EqtTrace.Info("TestEngine.ShouldRunInNoIsolation: This run has multiple different architectures or frameworks, running in isolation (in a separate testhost proces).."); return false; } @@ -557,14 +582,29 @@ private IEnumerable GetSourcesFromTestRunCriteria(TestRunCriteria testRu : testRunCriteria.Sources; } - private static void ThrowExceptionIfTestHostManagerIsNull( - ITestRuntimeProvider testHostManager, - string settingsXml) + private static void ThrowExceptionIfTestHostManagerIsNull(ITestRuntimeProvider testHostManager, string settingsXml) { if (testHostManager == null) { - EqtTrace.Error($"{nameof(TestPlatform)}.{nameof(ThrowExceptionIfTestHostManagerIsNull)}: No suitable testHostProvider found for runsettings: {settingsXml}"); + EqtTrace.Error($"{nameof(TestEngine)}.{nameof(ThrowExceptionIfTestHostManagerIsNull)}: No suitable testHostProvider found for runsettings: {settingsXml}"); throw new TestPlatformException(string.Format(CultureInfo.CurrentCulture, Resources.Resources.NoTestHostProviderFound)); } } + + private static void ThrowExceptionIfAnyTestHostManagerIsNull(List testRuntimeProviders) + { + var missingRuntimeProviders = testRuntimeProviders.Where(p => p.Type == null); + if (missingRuntimeProviders.Any()) + { + var stringBuilder = new StringBuilder(); + stringBuilder.AppendLine(string.Format(CultureInfo.CurrentCulture, Resources.Resources.NoTestHostProviderFound)); + foreach (var missingRuntimeProvider in missingRuntimeProviders) + { + EqtTrace.Error($"{nameof(TestEngine)}.{nameof(ThrowExceptionIfAnyTestHostManagerIsNull)}: No suitable testHostProvider found for sources {missingRuntimeProvider.SourceDetails.Select(s => s.Source)} and runsettings: {missingRuntimeProvider.RunSettings}"); + missingRuntimeProvider.SourceDetails.ForEach(detail => stringBuilder.AppendLine(detail.Source)); + } + + throw new TestPlatformException(stringBuilder.ToString()); + } + } } diff --git a/src/Microsoft.TestPlatform.CrossPlatEngine/TestSession/ProxyTestSessionManager.cs b/src/Microsoft.TestPlatform.CrossPlatEngine/TestSession/ProxyTestSessionManager.cs index aca9df677c..a81fecdeb8 100644 --- a/src/Microsoft.TestPlatform.CrossPlatEngine/TestSession/ProxyTestSessionManager.cs +++ b/src/Microsoft.TestPlatform.CrossPlatEngine/TestSession/ProxyTestSessionManager.cs @@ -18,7 +18,7 @@ namespace Microsoft.VisualStudio.TestPlatform.CrossPlatEngine; using ObjectModel.Engine; using CrossPlatResources = Resources.Resources; - +using System.Linq; /// /// Orchestrates test session operations for the engine communicating with the client. /// @@ -36,9 +36,10 @@ private enum TestSessionState private readonly object _proxyOperationLockObject = new(); private volatile bool _proxySetupFailed; private readonly StartTestSessionCriteria _testSessionCriteria; - private readonly int _testhostCount; + private readonly int _maxTesthostCount; private TestSessionInfo _testSessionInfo; - private readonly Func _proxyCreator; + private readonly Func _proxyCreator; + private readonly List _runtimeProviders; private readonly IList _proxyContainerList; private readonly IDictionary _proxyMap; private readonly Stopwatch _testSessionStopwatch; @@ -48,17 +49,18 @@ private enum TestSessionState /// /// /// The test session criteria. - /// The testhost count. + /// The testhost count. /// The proxy creator. public ProxyTestSessionManager( StartTestSessionCriteria criteria, - int testhostCount, - Func proxyCreator) + int maxTesthostCount, + Func proxyCreator, + List runtimeProviders) { _testSessionCriteria = criteria; - _testhostCount = testhostCount; + _maxTesthostCount = maxTesthostCount; _proxyCreator = proxyCreator; - + _runtimeProviders = runtimeProviders; _proxyContainerList = new List(); _proxyMap = new Dictionary(); _testSessionStopwatch = new Stopwatch(); @@ -80,33 +82,34 @@ public virtual bool StartSession(ITestSessionEventsHandler eventsHandler, IReque var stopwatch = new Stopwatch(); stopwatch.Start(); + // TODO: Right now we either pre-create 1 testhost if parallel is disabled, or we pre-create as many + // testhosts as we have sources. In the future we will have a maxParallelLevel set to the actual parallel level + // (which might be lower than the number of sources) and we should do some kind of thinking here to figure out how to split the sources. + // To follow the way parallel execution and discovery is (supposed to be) working, there should be as many testhosts + // as the maxParallel level pre-started,and marked with the Shared, and configuration that they can run. + + // Get dictionary from source -> runtimeProviderInfo, that has the type of runtime provider to create for this + // source, and updated runsettings. + var sourceToRuntimeProviderInfoMap = _runtimeProviders + .SelectMany(runtimeProviderInfo => runtimeProviderInfo.SourceDetails.Select(detail => new KeyValuePair(detail.Source, runtimeProviderInfo))) + .ToDictionary(pair => pair.Key, pair => pair.Value); + // Create all the proxies in parallel, one task per proxy. - var taskList = new Task[_testhostCount]; + var taskList = new Task[_maxTesthostCount]; for (int i = 0; i < taskList.Length; ++i) { - // The testhost count is equal to 1 because one of the following conditions - // holds true: - // 1. we're dealing with a shared testhost (e.g.: .NET Framework testhost) - // that must process multiple sources within the same testhost process; - // 2. we're dealing with a single testhost (shared or not, it doesn't matter) - // that must process a single source; - // Either way, no further processing of the original test source list is needed - // in either of those cases. - // - // Consequentely, if the testhost count is greater than one it means that the - // testhost is not shared (e.g.: .NET Core testhost), in which case each test - // source must be processed by a dedicated testhost, which is the reason we - // create a list with a single element, i.e. the current source to be processed. - var sources = (_testhostCount == 1) - ? _testSessionCriteria.Sources - : new List() { _testSessionCriteria.Sources[i] }; + // This is similar to what we do in ProxyExecutionManager, and ProxyDiscoveryManager, we split + // up the payload into multiple smaller pieces. Here it is one source per proxy. + var source = _testSessionCriteria.Sources[i]; + var sources = new List() { source }; + var runtimeProviderInfo = sourceToRuntimeProviderInfoMap[source]; taskList[i] = Task.Factory.StartNew(() => { - if (!SetupRawProxy( - sources, - _testSessionCriteria.RunSettings)) + var proxySetupSucceeded = SetupRawProxy(sources, runtimeProviderInfo); + if (!proxySetupSucceeded) { + // Set this only in the failed case, so we can check if any proxy failed to setup. _proxySetupFailed = true; } }); @@ -230,13 +233,16 @@ public virtual ProxyOperationManager DequeueProxy(string source, string runSetti // // TODO (copoiena): This run settings match is rudimentary. We should refine the // match criteria in the future. - if (!_testSessionCriteria.RunSettings.Equals(runSettings)) - { - throw new InvalidOperationException( - string.Format( - CultureInfo.CurrentUICulture, - CrossPlatResources.NoProxyMatchesDescription)); - } + // ERR: we actually have different run settings, because the framework and architecture migh be different, because we figure out the common framework and architecture, and we definitely want the architecture and framework to be the same. + // What should we do? Strip the info from runsettings when we infer it? Ideally we would have that info on the runsetting node. + //if (!_testSessionCriteria.RunSettings.Equals(runSettings)) + //{ + // EqtTrace.Verbose($"ProxyTestSessionManager.DequeueProxy: A proxy exists, but the runsettings do not match. Skipping it. Incoming settings: {runSettings}, Settings on proxy: {_testSessionCriteria.RunSettings}"); + // throw new InvalidOperationException( + // string.Format( + // CultureInfo.CurrentUICulture, + // CrossPlatResources.NoProxyMatchesDescription)); + //} // Get the actual proxy. proxyContainer = _proxyContainerList[_proxyMap[source]]; @@ -312,12 +318,12 @@ private int EnqueueNewProxy( private bool SetupRawProxy( IList sources, - string runSettings) + TestRuntimeProviderInfo runtimeProviderInfo) { try { // Create and cache the proxy. - var operationManagerProxy = _proxyCreator(); + var operationManagerProxy = _proxyCreator(runtimeProviderInfo); if (operationManagerProxy == null) { return false; @@ -327,7 +333,7 @@ private bool SetupRawProxy( operationManagerProxy.Initialize(skipDefaultAdapters: false); // Start the test host associated to the proxy. - if (!operationManagerProxy.SetupChannel(sources, runSettings)) + if (!operationManagerProxy.SetupChannel(sources, runtimeProviderInfo.RunSettings)) { return false; } diff --git a/src/Microsoft.TestPlatform.ObjectModel/Client/Interfaces/ITestPlatform.cs b/src/Microsoft.TestPlatform.ObjectModel/Client/Interfaces/ITestPlatform.cs index f263f14c54..7f52a2b75c 100644 --- a/src/Microsoft.TestPlatform.ObjectModel/Client/Interfaces/ITestPlatform.cs +++ b/src/Microsoft.TestPlatform.ObjectModel/Client/Interfaces/ITestPlatform.cs @@ -77,5 +77,6 @@ ITestRunRequest CreateTestRunRequest( bool StartTestSession( IRequestData requestData, StartTestSessionCriteria criteria, - ITestSessionEventsHandler eventsHandler); + ITestSessionEventsHandler eventsHandler, + Dictionary sourceToSourceDetailMap); } diff --git a/src/Microsoft.TestPlatform.ObjectModel/RunSettings/RunConfiguration.cs b/src/Microsoft.TestPlatform.ObjectModel/RunSettings/RunConfiguration.cs index 6be597bd72..b9baa31304 100644 --- a/src/Microsoft.TestPlatform.ObjectModel/RunSettings/RunConfiguration.cs +++ b/src/Microsoft.TestPlatform.ObjectModel/RunSettings/RunConfiguration.cs @@ -572,6 +572,7 @@ public static RunConfiguration FromXml(XmlReader reader) while (reader.NodeType == XmlNodeType.Element) { string elementName = reader.Name; + // TODO: make run settings nodes case insensitive? switch (elementName) { case "ResultsDirectory": diff --git a/src/vstest.console/TestPlatformHelpers/TestRequestManager.cs b/src/vstest.console/TestPlatformHelpers/TestRequestManager.cs index f5957a3147..17795a7b03 100644 --- a/src/vstest.console/TestPlatformHelpers/TestRequestManager.cs +++ b/src/vstest.console/TestPlatformHelpers/TestRequestManager.cs @@ -476,6 +476,13 @@ public void StartTestSession( payload.RunSettings = updatedRunsettings; } + var sourceToSourceDetailMap = payload.Sources.Select(source => new SourceDetail + { + Source = source, + Architecture = sourceToArchitectureMap[source], + Framework = sourceToFrameworkMap[source], + }).ToDictionary(k => k.Source); + if (InferRunSettingsHelper.AreRunSettingsCollectorsIncompatibleWithTestSettings(payload.RunSettings)) { throw new SettingsException( @@ -505,7 +512,8 @@ public void StartTestSession( TestHostLauncher = testHostLauncher }; - if (!_testPlatform.StartTestSession(requestData, criteria, eventsHandler)) + var testSessionStarted = _testPlatform.StartTestSession(requestData, criteria, eventsHandler, sourceToSourceDetailMap); + if (!testSessionStarted) { EqtTrace.Warning("TestRequestManager.StartTestSession: Unable to start test session."); } diff --git a/test/Microsoft.TestPlatform.CrossPlatEngine.UnitTests/Client/Parallel/ParallelOperationManagerTests.cs b/test/Microsoft.TestPlatform.CrossPlatEngine.UnitTests/Client/Parallel/ParallelOperationManagerTests.cs index 1cda1971da..fa5c9478d3 100644 --- a/test/Microsoft.TestPlatform.CrossPlatEngine.UnitTests/Client/Parallel/ParallelOperationManagerTests.cs +++ b/test/Microsoft.TestPlatform.CrossPlatEngine.UnitTests/Client/Parallel/ParallelOperationManagerTests.cs @@ -5,7 +5,6 @@ namespace TestPlatform.CrossPlatEngine.UnitTests.Client; -using Microsoft.VisualStudio.TestPlatform.CrossPlatEngine.Client; using Microsoft.VisualStudio.TestTools.UnitTesting; using System; diff --git a/test/vstest.ProgrammerTests/Fakes/ActionRecord.cs b/test/vstest.ProgrammerTests/Fakes/ActionRecord.cs new file mode 100644 index 0000000000..fca98e3d89 --- /dev/null +++ b/test/vstest.ProgrammerTests/Fakes/ActionRecord.cs @@ -0,0 +1,19 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +namespace vstest.ProgrammerTests.Fakes; + +/// +/// Contains information about something that happened, e.g. when a runtime provider resolve was called. +/// +/// +internal class ActionRecord +{ + public T Value { get; } + public string StackTrace { get; } + public ActionRecord (T value) + { + StackTrace = Environment.StackTrace; + Value = value; + } +} diff --git a/test/vstest.ProgrammerTests/Fakes/FakeCommunicationChannel.cs b/test/vstest.ProgrammerTests/Fakes/FakeCommunicationChannel.cs index 53bf0d681f..aa5d034ed3 100644 --- a/test/vstest.ProgrammerTests/Fakes/FakeCommunicationChannel.cs +++ b/test/vstest.ProgrammerTests/Fakes/FakeCommunicationChannel.cs @@ -206,5 +206,10 @@ private void ProcessIncomingMessages(TContext context) } } } + + public override string? ToString() + { + return NextResponses.Peek()?.ToString(); + } } diff --git a/test/vstest.ProgrammerTests/Fakes/FakeMessage.cs b/test/vstest.ProgrammerTests/Fakes/FakeMessage.cs index ffbeb8666d..079bc56572 100644 --- a/test/vstest.ProgrammerTests/Fakes/FakeMessage.cs +++ b/test/vstest.ProgrammerTests/Fakes/FakeMessage.cs @@ -50,4 +50,9 @@ public FakeMessage(string messageType, T payload, int version = 0) /// Version of the message to allow the internal serializer to choose the correct serialization strategy. /// public int Version { get; } + + public override string ToString() + { + return $"{MessageType} {{{ Payload }}}"; + } } diff --git a/test/vstest.ProgrammerTests/Fakes/FakeProcess.cs b/test/vstest.ProgrammerTests/Fakes/FakeProcess.cs index 3f1eb2ea96..6f4b68a6ad 100644 --- a/test/vstest.ProgrammerTests/Fakes/FakeProcess.cs +++ b/test/vstest.ProgrammerTests/Fakes/FakeProcess.cs @@ -86,4 +86,14 @@ internal void Exit() ExitCallback(this); } } + + public override string ToString() + { + var state = !Started + ? "not started" + : !Exited + ? "running" + : "exited"; + return $"{(Id != default ? Id : "")} {Name ?? ""} {state}"; + } } diff --git a/test/vstest.ProgrammerTests/Fakes/FakeTestRuntimeProvider.cs b/test/vstest.ProgrammerTests/Fakes/FakeTestRuntimeProvider.cs index c68d50fdab..2efd24f120 100644 --- a/test/vstest.ProgrammerTests/Fakes/FakeTestRuntimeProvider.cs +++ b/test/vstest.ProgrammerTests/Fakes/FakeTestRuntimeProvider.cs @@ -12,6 +12,7 @@ internal class FakeTestRuntimeProvider : ITestRuntimeProvider { public FakeProcessHelper FakeProcessHelper { get; } public FakeCommunicationEndpoint FakeCommunicationEndpoint { get; } + public FakeCommunicationChannel FakeCommunicationChannel { get; } public FakeErrorAggregator FakeErrorAggregator { get; } public FakeProcess TestHostProcess { get; private set; } public FakeFileHelper FileHelper { get; } @@ -30,6 +31,7 @@ public FakeTestRuntimeProvider(FakeProcessHelper fakeProcessHelper, FakeProcess FileHelper = fakeFileHelper; TestDlls = fakeTestDlls; FakeCommunicationEndpoint = fakeCommunicationEndpoint; + FakeCommunicationChannel = fakeCommunicationEndpoint.Channel; FakeErrorAggregator = fakeErrorAggregator; var architectures = fakeTestDlls.Select(dll => dll.Architecture).Distinct().ToList(); @@ -123,4 +125,9 @@ public void SetCustomLauncher(ITestHostLauncher customLauncher) { throw new NotImplementedException(); } + + public override string ToString() + { + return $"{nameof(FakeTestRuntimeProvider)} - ({TestHostProcess.ToString() ?? ""}) - {FakeCommunicationChannel}"; + } } diff --git a/test/vstest.ProgrammerTests/Fakes/FakeTestRuntimeProviderManager.cs b/test/vstest.ProgrammerTests/Fakes/FakeTestRuntimeProviderManager.cs index e665ea3a09..90f6bb22e5 100644 --- a/test/vstest.ProgrammerTests/Fakes/FakeTestRuntimeProviderManager.cs +++ b/test/vstest.ProgrammerTests/Fakes/FakeTestRuntimeProviderManager.cs @@ -3,8 +3,6 @@ namespace vstest.ProgrammerTests.Fakes; -using System.Collections.Concurrent; - using Microsoft.VisualStudio.TestPlatform.Common.Hosting; using Microsoft.VisualStudio.TestPlatform.ObjectModel.Host; @@ -15,30 +13,35 @@ public FakeTestRuntimeProviderManager(FakeErrorAggregator fakeErrorAggregator) FakeErrorAggregator = fakeErrorAggregator; } - public ConcurrentQueue TestRuntimeProviders { get; } = new(); - public List UsedTestRuntimeProviders { get; } = new(); + public List TestRuntimeProviders { get; } = new(); + public List> UsedTestRuntimeProviders { get; } = new(); public FakeErrorAggregator FakeErrorAggregator { get; } public void AddTestRuntimeProviders(params FakeTestRuntimeProvider[] runtimeProviders) { - foreach (var runtimeProvider in runtimeProviders) - { - TestRuntimeProviders.Enqueue(runtimeProvider); - } + TestRuntimeProviders.AddRange(runtimeProviders); } - public ITestRuntimeProvider GetTestHostManagerByRunConfiguration(string runConfiguration) + public ITestRuntimeProvider GetTestHostManagerByRunConfiguration(string _, List sources) { + var allMatchingProviders = TestRuntimeProviders + .Where(r => r.TestDlls.Select(dll => dll.Path) + .Any(path => sources.Contains(path))) + .ToList(); + if (allMatchingProviders.Count > 1) + { - if (!TestRuntimeProviders.TryDequeue(out var next)) + } + var match = allMatchingProviders.FirstOrDefault(); + if (match == null) { - throw new InvalidOperationException("There are no more TestRuntimeProviders to be provided"); + throw new InvalidOperationException("There is no FakeTestRuntimeProvider that would match the filter."); } - UsedTestRuntimeProviders.Add(next); - return next; + UsedTestRuntimeProviders.Add(new ActionRecord(match)); + return match; } public ITestRuntimeProvider GetTestHostManagerByUri(string hostUri) diff --git a/test/vstest.ProgrammerTests/Fakes/Fixture.cs b/test/vstest.ProgrammerTests/Fakes/Fixture.cs index 835a4a903a..1be140e409 100644 --- a/test/vstest.ProgrammerTests/Fakes/Fixture.cs +++ b/test/vstest.ProgrammerTests/Fakes/Fixture.cs @@ -42,7 +42,7 @@ internal class Fixture : IDisposable public List ExecutedTests => TestRunEventsRegistrar.RunChangedEvents.SelectMany(er => er.Data.NewTestResults).ToList(); public List DiscoveredTests => TestDiscoveryEventsRegistrar.DiscoveredTestsEvents.SelectMany(er => er.Data.DiscoveredTestCases).ToList(); - public ITestSessionEventsHandler TestSessionEventsHandler { get; } + public FakeTestSessionEventsHandler TestSessionEventsHandler { get; } public Fixture(FixtureOptions? fixtureOptions = null) { diff --git a/test/vstest.ProgrammerTests/Fakes/RequestResponsePair.cs b/test/vstest.ProgrammerTests/Fakes/RequestResponsePair.cs index 5384f9cace..94b5e95329 100644 --- a/test/vstest.ProgrammerTests/Fakes/RequestResponsePair.cs +++ b/test/vstest.ProgrammerTests/Fakes/RequestResponsePair.cs @@ -35,4 +35,9 @@ public RequestResponsePair(TRequest request, IEnumerable responses, A public Action? AfterAction { get; } public List Responses { get; } public bool Debug { get; } + + public override string ToString() + { + return $"\u8595;{Request} \u8593;{Responses.FirstOrDefault()}"; + } } diff --git a/test/vstest.ProgrammerTests/MultiTFMRunAndDiscovery.cs b/test/vstest.ProgrammerTests/MultiTFMRunAndDiscovery.cs index 8a5b817b47..ba599e3b21 100644 --- a/test/vstest.ProgrammerTests/MultiTFMRunAndDiscovery.cs +++ b/test/vstest.ProgrammerTests/MultiTFMRunAndDiscovery.cs @@ -209,7 +209,7 @@ public async Task B() var testDiscoveryPayload = new DiscoveryRequestPayload { Sources = new List { mstest1Dll.Path, mstest2Dll.Path }, - RunSettings = $"{KnownFrameworkStrings.Net7}" + RunSettings = $"{KnownFrameworkStrings.Net7}" }; await testRequestManager.ExecuteWithAbort(tm => tm.DiscoverTests(testDiscoveryPayload, fixture.TestDiscoveryEventsRegistrar, fixture.ProtocolConfig)); @@ -427,7 +427,7 @@ public async Task D() var testRunRequestPayload = new TestRunRequestPayload { Sources = new List { mstest1Dll.Path, mstest2Dll.Path }, - RunSettings = $"{KnownFrameworkStrings.Net7}" + RunSettings = $"{KnownFrameworkStrings.Net7}" }; await testRequestManager.ExecuteWithAbort(tm => tm.RunTests(testRunRequestPayload, testHostLauncher: null, fixture.TestRunEventsRegistrar, fixture.ProtocolConfig)); @@ -538,16 +538,22 @@ public async Task E() var startTestSessionPayload = new StartTestSessionPayload { - RunSettings = "", + // We need to have a parallel run, otherwise we will create just a single proxy, + // because 1 is the maximum number of proxies to start for non-parallel run. + RunSettings = "0", Sources = new[] { mstest1Dll.Path, mstest2Dll.Path } }; await testRequestManager.ExecuteWithAbort(tm => tm.StartTestSession(startTestSessionPayload, testHostLauncher: null, fixture.TestSessionEventsHandler, fixture.ProtocolConfig)); + // You need to pass this on, otherwise it will ignore the test session that you just started. This is a by product of being able to start multiple test sessions. + var testSessionInfo = fixture.TestSessionEventsHandler.StartTestSessionCompleteEvents.Single().TestSessionInfo; + var testRunRequestPayload = new TestRunRequestPayload { Sources = new List { mstest1Dll.Path, mstest2Dll.Path }, - RunSettings = $"{KnownFrameworkStrings.Net7}" + RunSettings = $"0", + TestSessionInfo = testSessionInfo, }; await testRequestManager.ExecuteWithAbort(tm => tm.RunTests(testRunRequestPayload, testHostLauncher: null, fixture.TestRunEventsRegistrar, fixture.ProtocolConfig)); @@ -563,13 +569,13 @@ public async Task E() var startWithSources1Text = startWithSources1.Request.Payload.Select(t => t.ToString()).JoinBySpace(); // We sent mstest1.dll. startWithSources1Text.Should().Contain("mstest1.dll"); - startWithSources1Text.Should().Contain(KnownFrameworkStrings.Net7); + startWithSources1Text.Should().Contain(mstest1Dll.FrameworkName.ToString()); var startWithSources2 = testhost2.FakeCommunicationChannel.ProcessedMessages.Single(m => m.Request.MessageType == MessageType.StartTestExecutionWithSources); var startWithSources2Text = startWithSources2.Request.Payload.Select(t => t.ToString()).JoinBySpace(); // We sent mstest2.dll. startWithSources2Text.Should().Contain("mstest2.dll"); - startWithSources2Text.Should().Contain(KnownFrameworkStrings.Net7); + startWithSources2Text.Should().Contain(mstest2Dll.FrameworkName.ToString()); fixture.ExecutedTests.Should().HaveCount(mstest1Dll.TestCount + mstest2Dll.TestCount); } From 3ca787de5bbf446cb781db4feffe0434656e687e Mon Sep 17 00:00:00 2001 From: nohwnd Date: Tue, 1 Mar 2022 12:51:26 +0100 Subject: [PATCH 010/112] Fix programmer tests --- .../Parallel/ParallelOperationManager.cs | 69 ++++++------------- .../Parallel/ParallelProxyDiscoveryManager.cs | 34 +++++---- .../TestEngine.cs | 2 +- .../BasicRunAndDiscovery.cs | 2 +- .../Fakes/FakeTestRuntimeProviderManager.cs | 35 ++++++++-- .../Fakes/RequestResponsePair.cs | 2 +- .../MultiTFMRunAndDiscovery.cs | 2 +- 7 files changed, 74 insertions(+), 72 deletions(-) diff --git a/src/Microsoft.TestPlatform.CrossPlatEngine/Client/Parallel/ParallelOperationManager.cs b/src/Microsoft.TestPlatform.CrossPlatEngine/Client/Parallel/ParallelOperationManager.cs index 294d5356a4..ca336a2fc2 100644 --- a/src/Microsoft.TestPlatform.CrossPlatEngine/Client/Parallel/ParallelOperationManager.cs +++ b/src/Microsoft.TestPlatform.CrossPlatEngine/Client/Parallel/ParallelOperationManager.cs @@ -4,7 +4,6 @@ namespace Microsoft.VisualStudio.TestPlatform.CrossPlatEngine.Client; using System; -using System.Collections.Concurrent; using System.Collections.Generic; using System.Linq; using System.Threading.Tasks; @@ -21,12 +20,6 @@ internal sealed class ParallelOperationManager _createNewManager; private readonly int _maxParallelLevel; - /// - /// Holds all active managers, so we can do actions on all of them, like initialize, run, cancel or close. - /// - // TODO: make this ConcurrentDictionary and use it's concurrent api, if we have the need. - private readonly IDictionary _managerToEventHandlerMap = new ConcurrentDictionary(); - /// /// Default number of Processes /// @@ -119,7 +112,9 @@ private bool RunWorkInParallel() } } - return false; + // Return true when we started more work. Or false, when there was nothing more to do. + // This will propagate to handling of partial discovery / run. + return workToRun.Count > 0; } public bool RunNextWork(TManager completedManager) @@ -139,54 +134,34 @@ public bool RunNextWork(TManager completedManager) } } - /// - /// Remove and dispose a manager from concurrent list of manager. - /// - /// Manager to remove - public void RemoveManager(TManager manager) - { - _managerToEventHandlerMap.Remove(manager); - } - - /// - /// Add a manager in concurrent list of manager. - /// - /// Manager to add - /// eventHandler of the manager - public void AddManager(TManager manager, TEventHandler handler) - { - _managerToEventHandlerMap.Add(manager, handler); - } - public void DoActionOnAllManagers(Action action, bool doActionsInParallel = false) { - lock (_lock) + // We don't need to lock here, we just grab the current list of + // managers and run action on each one of them. + var managers = _managerSlots.Select(slot => slot.Manager).ToList(); + int i = 0; + var actionTasks = new Task[managers.Count]; + foreach (var manager in managers) { - int i = 0; - var actionTasks = new Task[_managerToEventHandlerMap.Count]; - foreach (var manager in _managerSlots.Select(s => s.Manager)) - { - if (manager == null) - continue; - - // Read the array before firing the task - beware of closures - if (doActionsInParallel) - { - actionTasks[i] = Task.Run(() => action(manager)); - i++; - } - else - { - DoManagerAction(() => action(manager)); - } - } + if (manager == null) + continue; + // Read the array before firing the task - beware of closures if (doActionsInParallel) { - DoManagerAction(() => Task.WaitAll(actionTasks)); + actionTasks[i] = Task.Run(() => action(manager)); + i++; + } + else + { + DoManagerAction(() => action(manager)); } } + if (doActionsInParallel) + { + DoManagerAction(() => Task.WaitAll(actionTasks)); + } } private void DoManagerAction(Action action) diff --git a/src/Microsoft.TestPlatform.CrossPlatEngine/Client/Parallel/ParallelProxyDiscoveryManager.cs b/src/Microsoft.TestPlatform.CrossPlatEngine/Client/Parallel/ParallelProxyDiscoveryManager.cs index c15cbc6946..abf2a8891f 100644 --- a/src/Microsoft.TestPlatform.CrossPlatEngine/Client/Parallel/ParallelProxyDiscoveryManager.cs +++ b/src/Microsoft.TestPlatform.CrossPlatEngine/Client/Parallel/ParallelProxyDiscoveryManager.cs @@ -162,25 +162,31 @@ when testhost crashed by itself and when user requested it (f.e. through TW) private List> SplitToWorkloads(DiscoveryCriteria discoveryCriteria, Dictionary sourceToTestHostProviderMap) { - List> workloads = discoveryCriteria.Sources - .Select(source => new ProviderSpecificWorkload(NewDiscoveryCriteriaFromSource(source, discoveryCriteria), sourceToTestHostProviderMap[source])) - .ToList(); + List> workloads = new(); + foreach (var source in discoveryCriteria.Sources) + { + var testHostProviderInfo = sourceToTestHostProviderMap[source]; + var runsettingsXml = testHostProviderInfo.RunSettings; + var updatedDiscoveryCriteria = new ProviderSpecificWorkload(NewDiscoveryCriteriaFromSourceAndSettings(source, discoveryCriteria, runsettingsXml), testHostProviderInfo); + workloads.Add(updatedDiscoveryCriteria); + } return workloads; - } - private DiscoveryCriteria NewDiscoveryCriteriaFromSource(string source, DiscoveryCriteria discoveryCriteria) - { - var criteria = new DiscoveryCriteria( - new[] { source }, - discoveryCriteria.FrequencyOfDiscoveredTestsEvent, - discoveryCriteria.DiscoveredTestEventTimeout, - discoveryCriteria.RunSettings - ); + static DiscoveryCriteria NewDiscoveryCriteriaFromSourceAndSettings(string source, DiscoveryCriteria discoveryCriteria, string runsettingsXml) + { + var criteria = new DiscoveryCriteria( + new[] { source }, + discoveryCriteria.FrequencyOfDiscoveredTestsEvent, + discoveryCriteria.DiscoveredTestEventTimeout, + runsettingsXml, + discoveryCriteria.TestSessionInfo + ); - criteria.TestCaseFilter = discoveryCriteria.TestCaseFilter; + criteria.TestCaseFilter = discoveryCriteria.TestCaseFilter; - return criteria; + return criteria; + } } /// diff --git a/src/Microsoft.TestPlatform.CrossPlatEngine/TestEngine.cs b/src/Microsoft.TestPlatform.CrossPlatEngine/TestEngine.cs index acd6df5e73..ff6d92f535 100644 --- a/src/Microsoft.TestPlatform.CrossPlatEngine/TestEngine.cs +++ b/src/Microsoft.TestPlatform.CrossPlatEngine/TestEngine.cs @@ -394,7 +394,7 @@ private List GetTestRuntimeProvidersForUniqueConfigurat // out which runtime providers would run them, and if the runtime provider is shared or not. mostRecentlyCreatedInstance = null; var testRuntimeProviders = new List(); - var uniqueRunConfigurations = sourceToSourceDetailMap.Values.GroupBy(k => new { k.Framework, k.Architecture }); + var uniqueRunConfigurations = sourceToSourceDetailMap.Values.GroupBy(k => $"{k.Framework}|{k.Architecture}"); foreach (var runConfiguration in uniqueRunConfigurations) { // It is okay to take the first (or any) source detail in the group. We are grouping to get the same source detail, so all architectures and frameworks are the same. diff --git a/test/vstest.ProgrammerTests/BasicRunAndDiscovery.cs b/test/vstest.ProgrammerTests/BasicRunAndDiscovery.cs index c6ea10ecf8..01367a7dd0 100644 --- a/test/vstest.ProgrammerTests/BasicRunAndDiscovery.cs +++ b/test/vstest.ProgrammerTests/BasicRunAndDiscovery.cs @@ -75,7 +75,7 @@ Given multple test assemblies that use the same target framework. When we run tests. - Then all tests from all assemblies are run.. + Then all tests from all assemblies are run. ")] public async Task B() { diff --git a/test/vstest.ProgrammerTests/Fakes/FakeTestRuntimeProviderManager.cs b/test/vstest.ProgrammerTests/Fakes/FakeTestRuntimeProviderManager.cs index 90f6bb22e5..15c672190d 100644 --- a/test/vstest.ProgrammerTests/Fakes/FakeTestRuntimeProviderManager.cs +++ b/test/vstest.ProgrammerTests/Fakes/FakeTestRuntimeProviderManager.cs @@ -14,6 +14,7 @@ public FakeTestRuntimeProviderManager(FakeErrorAggregator fakeErrorAggregator) } public List TestRuntimeProviders { get; } = new(); + public Queue TestRuntimeProvidersByOrder { get; } = new(); public List> UsedTestRuntimeProviders { get; } = new(); public FakeErrorAggregator FakeErrorAggregator { get; } @@ -21,6 +22,17 @@ public FakeTestRuntimeProviderManager(FakeErrorAggregator fakeErrorAggregator) public void AddTestRuntimeProviders(params FakeTestRuntimeProvider[] runtimeProviders) { TestRuntimeProviders.AddRange(runtimeProviders); + + // In cases where we don't have multi tfm run, we will be asked for + // a provider with multiple sources. In that case we don't know exactly which one to provide + // so we need to go by order. We also do this resolve twice for each source in parallel run + // because we first need to know if the provider is shared. So we add to the queue twice. + // This is brittle, but there is no way around this :( + foreach(var provider in runtimeProviders) + { + TestRuntimeProvidersByOrder.Enqueue(provider); + TestRuntimeProvidersByOrder.Enqueue(provider); + } } public ITestRuntimeProvider GetTestHostManagerByRunConfiguration(string _, List sources) @@ -30,18 +42,27 @@ public ITestRuntimeProvider GetTestHostManagerByRunConfiguration(string _, List< .Any(path => sources.Contains(path))) .ToList(); - if (allMatchingProviders.Count > 1) + if (allMatchingProviders.Count == 0) { - + throw new InvalidOperationException($"There are no FakeTestRuntimeProviders associated with any of the incoming sources, make sure your testhost fixture has at least one dll: {sources.JoinByComma()}"); } - var match = allMatchingProviders.FirstOrDefault(); - if (match == null) + + if (allMatchingProviders.Count > 1) { - throw new InvalidOperationException("There is no FakeTestRuntimeProvider that would match the filter."); + // This is a single tfm run, or multiple dlls in the run have the same tfm. We need to provide + // providers by order. + if (!TestRuntimeProvidersByOrder.TryDequeue(out var provider)) + { + throw new InvalidOperationException("There are no more FakeTestRuntimeProviders to be provided."); + } + + UsedTestRuntimeProviders.Add(new ActionRecord(provider)); + return provider; } - UsedTestRuntimeProviders.Add(new ActionRecord(match)); - return match; + var single = allMatchingProviders.Single(); + UsedTestRuntimeProviders.Add(new ActionRecord(single)); + return single; } public ITestRuntimeProvider GetTestHostManagerByUri(string hostUri) diff --git a/test/vstest.ProgrammerTests/Fakes/RequestResponsePair.cs b/test/vstest.ProgrammerTests/Fakes/RequestResponsePair.cs index 94b5e95329..a62e514130 100644 --- a/test/vstest.ProgrammerTests/Fakes/RequestResponsePair.cs +++ b/test/vstest.ProgrammerTests/Fakes/RequestResponsePair.cs @@ -38,6 +38,6 @@ public RequestResponsePair(TRequest request, IEnumerable responses, A public override string ToString() { - return $"\u8595;{Request} \u8593;{Responses.FirstOrDefault()}"; + return $"\u2193{Request} \u2191{Responses.FirstOrDefault()}"; } } diff --git a/test/vstest.ProgrammerTests/MultiTFMRunAndDiscovery.cs b/test/vstest.ProgrammerTests/MultiTFMRunAndDiscovery.cs index ba599e3b21..5be274d7ca 100644 --- a/test/vstest.ProgrammerTests/MultiTFMRunAndDiscovery.cs +++ b/test/vstest.ProgrammerTests/MultiTFMRunAndDiscovery.cs @@ -466,7 +466,7 @@ When we execute tests and provide runsettings that define the desired target framework. Then two testhosts should be started that target the framework chosen by runsettings. - ")][Only] + ")] public async Task E() { // -- arrange From 14e66d8d5904568d67aa22a7292fde16f33a2b9f Mon Sep 17 00:00:00 2001 From: nohwnd Date: Tue, 1 Mar 2022 13:46:26 +0100 Subject: [PATCH 011/112] Skip source validation and enable checks --- test/Intent/Runner.cs | 1 + .../MultiTFMRunAndDiscovery.cs | 27 +++++++------------ 2 files changed, 10 insertions(+), 18 deletions(-) diff --git a/test/Intent/Runner.cs b/test/Intent/Runner.cs index e50bfcc846..26a0cb1955 100644 --- a/test/Intent/Runner.cs +++ b/test/Intent/Runner.cs @@ -1,6 +1,7 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT license. See LICENSE file in the project root for full license information. +using System.Diagnostics; using System.Reflection; namespace Intent; diff --git a/test/vstest.ProgrammerTests/MultiTFMRunAndDiscovery.cs b/test/vstest.ProgrammerTests/MultiTFMRunAndDiscovery.cs index 5be274d7ca..b7d71556f8 100644 --- a/test/vstest.ProgrammerTests/MultiTFMRunAndDiscovery.cs +++ b/test/vstest.ProgrammerTests/MultiTFMRunAndDiscovery.cs @@ -1,21 +1,17 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT license. See LICENSE file in the project root for full license information. -namespace vstest.ProgrammerTests; - +using Intent; using FluentAssertions; +using vstest.ProgrammerTests.Fakes; + using Microsoft.VisualStudio.TestPlatform.CommunicationUtilities.ObjectModel; using Microsoft.VisualStudio.TestPlatform.ObjectModel; using Microsoft.VisualStudio.TestPlatform.ObjectModel.Client; - -# if DEBUG -#endif - -using vstest.ProgrammerTests.Fakes; using Microsoft.VisualStudio.TestPlatform.Utilities; -using Intent; using Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.Payloads; +namespace vstest.ProgrammerTests; public class MultiTFM { public class MultiTFMDiscovery @@ -108,8 +104,7 @@ public async Task A() // -- assert fixture.AssertNoErrors(); // We figure out the framework for each assembly so there should be no incompatibility warnings - // REVIEW: This should be uncommented. Commenting it now, because it is helpful to see those warnings. - // fixture.TestRunEventsRegistrar.LoggedWarnings.Should().NotContainMatch("Test run detected DLL(s) which were built for different framework*"); + fixture.TestRunEventsRegistrar.LoggedWarnings.Should().NotContainMatch("Test run detected DLL(s) which were built for different framework*"); fixture.ProcessHelper.Processes.Where(p => p.Started).Should().HaveCount(2); var startWithSources1 = testhost1.FakeCommunicationChannel.ProcessedMessages.Single(m => m.Request.MessageType == MessageType.StartDiscovery); @@ -217,8 +212,7 @@ public async Task B() // -- assert fixture.AssertNoErrors(); // We figure out the framework for each assembly so there should be no incompatibility warnings - // REVIEW: This should be uncommented. Commenting it now, because it is helpful to see those warnings. - // fixture.TestRunEventsRegistrar.LoggedWarnings.Should().NotContainMatch("Test run detected DLL(s) which were built for different framework*"); + fixture.TestRunEventsRegistrar.LoggedWarnings.Should().NotContainMatch("Test run detected DLL(s) which were built for different framework*"); fixture.ProcessHelper.Processes.Where(p => p.Started).Should().HaveCount(2); var startWithSources1 = testhost1.FakeCommunicationChannel.ProcessedMessages.Single(m => m.Request.MessageType == MessageType.StartDiscovery); @@ -326,8 +320,7 @@ public async Task C() // -- assert fixture.AssertNoErrors(); // We figure out the framework for each assembly so there should be no incompatibility warnings - // REVIEW: This should be uncommented. Commenting it now, because it is helpful to see those warnings. - // fixture.TestRunEventsRegistrar.LoggedWarnings.Should().NotContainMatch("Test run detected DLL(s) which were built for different framework*"); + fixture.TestRunEventsRegistrar.LoggedWarnings.Should().NotContainMatch("Test run detected DLL(s) which were built for different framework*"); fixture.ProcessHelper.Processes.Where(p => p.Started).Should().HaveCount(2); var startWithSources1 = testhost1.FakeCommunicationChannel.ProcessedMessages.Single(m => m.Request.MessageType == MessageType.StartTestExecutionWithSources); @@ -435,8 +428,7 @@ public async Task D() // -- assert fixture.AssertNoErrors(); // We figure out the framework for each assembly so there should be no incompatibility warnings - // REVIEW: This should be uncommented. Commenting it now, because it is helpful to see those warnings. - // fixture.TestRunEventsRegistrar.LoggedWarnings.Should().NotContainMatch("Test run detected DLL(s) which were built for different framework*"); + fixture.TestRunEventsRegistrar.LoggedWarnings.Should().NotContainMatch("Test run detected DLL(s) which were built for different framework*"); fixture.ProcessHelper.Processes.Where(p => p.Started).Should().HaveCount(2); var startWithSources1 = testhost1.FakeCommunicationChannel.ProcessedMessages.Single(m => m.Request.MessageType == MessageType.StartTestExecutionWithSources); @@ -561,8 +553,7 @@ public async Task E() // -- assert fixture.AssertNoErrors(); // We figure out the framework for each assembly so there should be no incompatibility warnings - // REVIEW: This should be uncommented. Commenting it now, because it is helpful to see those warnings. - // fixture.TestRunEventsRegistrar.LoggedWarnings.Should().NotContainMatch("Test run detected DLL(s) which were built for different framework*"); + fixture.TestRunEventsRegistrar.LoggedWarnings.Should().NotContainMatch("Test run detected DLL(s) which were built for different framework*"); fixture.ProcessHelper.Processes.Where(p => p.Started).Should().HaveCount(2); var startWithSources1 = testhost1.FakeCommunicationChannel.ProcessedMessages.Single(m => m.Request.MessageType == MessageType.StartTestExecutionWithSources); From a8f5da85510cce80dc55798da17afa5ee8697941 Mon Sep 17 00:00:00 2001 From: nohwnd Date: Tue, 1 Mar 2022 14:42:35 +0100 Subject: [PATCH 012/112] Fix settings check in test sessions, and remove rest of review points --- .../Parallel/ParallelProxyDiscoveryManager.cs | 2 +- .../Parallel/ParallelProxyExecutionManager.cs | 2 +- .../TestEngine.cs | 10 ++---- .../TestSession/ProxyTestSessionManager.cs | 33 +++++++++---------- .../Hosting/DefaultTestHostManager.cs | 3 +- .../TestPlatformHelpers/TestRequestManager.cs | 3 -- 6 files changed, 21 insertions(+), 32 deletions(-) diff --git a/src/Microsoft.TestPlatform.CrossPlatEngine/Client/Parallel/ParallelProxyDiscoveryManager.cs b/src/Microsoft.TestPlatform.CrossPlatEngine/Client/Parallel/ParallelProxyDiscoveryManager.cs index 48a6c2244d..b8065ebaab 100644 --- a/src/Microsoft.TestPlatform.CrossPlatEngine/Client/Parallel/ParallelProxyDiscoveryManager.cs +++ b/src/Microsoft.TestPlatform.CrossPlatEngine/Client/Parallel/ParallelProxyDiscoveryManager.cs @@ -127,7 +127,7 @@ public void Close() public bool HandlePartialDiscoveryComplete(IProxyDiscoveryManager proxyDiscoveryManager, long totalTests, IEnumerable lastChunk, bool isAborted) { var allDiscoverersCompleted = false; - // REVIEW: Interlocked.Increment the count, and the condition below probably does not need to be in a lock? + // TODO: Interlocked.Increment the count, and the condition below probably does not need to be in a lock? lock (_discoveryStatusLockObject) { // Each concurrent Executor calls this method diff --git a/src/Microsoft.TestPlatform.CrossPlatEngine/Client/Parallel/ParallelProxyExecutionManager.cs b/src/Microsoft.TestPlatform.CrossPlatEngine/Client/Parallel/ParallelProxyExecutionManager.cs index dba610f17a..a06ea03839 100644 --- a/src/Microsoft.TestPlatform.CrossPlatEngine/Client/Parallel/ParallelProxyExecutionManager.cs +++ b/src/Microsoft.TestPlatform.CrossPlatEngine/Client/Parallel/ParallelProxyExecutionManager.cs @@ -135,7 +135,7 @@ public bool HandlePartialRunComplete( ICollection executorUris) { var allRunsCompleted = false; - // REVIEW: Interlocked.Increment _runCompletedClients, and the condition on the bottom probably does not need to be under lock? + // TODO: Interlocked.Increment _runCompletedClients, and the condition on the bottom probably does not need to be under lock?? lock (_executionStatusLockObject) { // Each concurrent Executor calls this method diff --git a/src/Microsoft.TestPlatform.CrossPlatEngine/TestEngine.cs b/src/Microsoft.TestPlatform.CrossPlatEngine/TestEngine.cs index d882556151..146628d8aa 100644 --- a/src/Microsoft.TestPlatform.CrossPlatEngine/TestEngine.cs +++ b/src/Microsoft.TestPlatform.CrossPlatEngine/TestEngine.cs @@ -330,12 +330,6 @@ public IProxyTestSessionManager GetTestSessionManager( { var sources = testRuntimeProviderInfo.SourceDetails.Select(x => x.Source).ToList(); var hostManager = _testHostProviderManager.GetTestHostManagerByRunConfiguration(testRuntimeProviderInfo.RunSettings, sources); - - if (hostManager.GetType() != testRuntimeProviderInfo.Type) - { - throw new Exception("hostmanager type is not the same, why?"); - } - ThrowExceptionIfTestHostManagerIsNull(hostManager, testRuntimeProviderInfo.RunSettings); hostManager.Initialize(TestSessionMessageLogger.Instance, testRuntimeProviderInfo.RunSettings); @@ -405,7 +399,7 @@ private List GetTestRuntimeProvidersForUniqueConfigurat var testRuntimeProvider = _testHostProviderManager.GetTestHostManagerByRunConfiguration(runsettingsXml, sources); var testRuntimeProviderInfo = new TestRuntimeProviderInfo(testRuntimeProvider.GetType(), testRuntimeProvider.Shared, runsettingsXml, sourceDetails: runConfiguration.ToList()); - // Outputting the instance, because the code for in-process run uses it, and we don't want to resolve a + // Outputting the instance, because the code for in-process run uses it, and we don't want to resolve it another time. mostRecentlyCreatedInstance = testRuntimeProvider; testRuntimeProviders.Add(testRuntimeProviderInfo); } @@ -504,7 +498,7 @@ private bool ShouldRunInProcess( { if (testHostProviders.Count > 1) { - EqtTrace.Info("TestEngine.ShouldRunInNoIsolation: This run has multiple different architectures or frameworks, running in isolation (in a separate testhost proces).."); + EqtTrace.Info("TestEngine.ShouldRunInNoIsolation: This run has multiple different architectures or frameworks, running in isolation (in a separate testhost proces)."); return false; } diff --git a/src/Microsoft.TestPlatform.CrossPlatEngine/TestSession/ProxyTestSessionManager.cs b/src/Microsoft.TestPlatform.CrossPlatEngine/TestSession/ProxyTestSessionManager.cs index 59299ff5a3..34fb9112f6 100644 --- a/src/Microsoft.TestPlatform.CrossPlatEngine/TestSession/ProxyTestSessionManager.cs +++ b/src/Microsoft.TestPlatform.CrossPlatEngine/TestSession/ProxyTestSessionManager.cs @@ -43,6 +43,7 @@ private enum TestSessionState private readonly IList _proxyContainerList; private readonly IDictionary _proxyMap; private readonly Stopwatch _testSessionStopwatch; + private readonly Dictionary _sourceToRuntimeProviderInfoMap; /// /// Initializes a new instance of the class. @@ -64,6 +65,12 @@ public ProxyTestSessionManager( _proxyContainerList = new List(); _proxyMap = new Dictionary(); _testSessionStopwatch = new Stopwatch(); + + // Get dictionary from source -> runtimeProviderInfo, that has the type of runtime provider to create for this + // source, and updated runsettings. + _sourceToRuntimeProviderInfoMap = _runtimeProviders + .SelectMany(runtimeProviderInfo => runtimeProviderInfo.SourceDetails.Select(detail => new KeyValuePair(detail.Source, runtimeProviderInfo))) + .ToDictionary(pair => pair.Key, pair => pair.Value); } // NOTE: The method is virtual for mocking purposes. @@ -88,12 +95,6 @@ public virtual bool StartSession(ITestSessionEventsHandler eventsHandler, IReque // To follow the way parallel execution and discovery is (supposed to be) working, there should be as many testhosts // as the maxParallel level pre-started,and marked with the Shared, and configuration that they can run. - // Get dictionary from source -> runtimeProviderInfo, that has the type of runtime provider to create for this - // source, and updated runsettings. - var sourceToRuntimeProviderInfoMap = _runtimeProviders - .SelectMany(runtimeProviderInfo => runtimeProviderInfo.SourceDetails.Select(detail => new KeyValuePair(detail.Source, runtimeProviderInfo))) - .ToDictionary(pair => pair.Key, pair => pair.Value); - // Create all the proxies in parallel, one task per proxy. var taskList = new Task[_maxTesthostCount]; for (int i = 0; i < taskList.Length; ++i) @@ -102,7 +103,7 @@ public virtual bool StartSession(ITestSessionEventsHandler eventsHandler, IReque // up the payload into multiple smaller pieces. Here it is one source per proxy. var source = _testSessionCriteria.Sources[i]; var sources = new List() { source }; - var runtimeProviderInfo = sourceToRuntimeProviderInfoMap[source]; + var runtimeProviderInfo = _sourceToRuntimeProviderInfoMap[source]; taskList[i] = Task.Factory.StartNew(() => { @@ -233,16 +234,14 @@ public virtual ProxyOperationManager DequeueProxy(string source, string runSetti // // TODO (copoiena): This run settings match is rudimentary. We should refine the // match criteria in the future. - // ERR: we actually have different run settings, because the framework and architecture migh be different, because we figure out the common framework and architecture, and we definitely want the architecture and framework to be the same. - // What should we do? Strip the info from runsettings when we infer it? Ideally we would have that info on the runsetting node. - //if (!_testSessionCriteria.RunSettings.Equals(runSettings)) - //{ - // EqtTrace.Verbose($"ProxyTestSessionManager.DequeueProxy: A proxy exists, but the runsettings do not match. Skipping it. Incoming settings: {runSettings}, Settings on proxy: {_testSessionCriteria.RunSettings}"); - // throw new InvalidOperationException( - // string.Format( - // CultureInfo.CurrentUICulture, - // CrossPlatResources.NoProxyMatchesDescription)); - //} + if (!_sourceToRuntimeProviderInfoMap[source].RunSettings.Equals(runSettings)) + { + EqtTrace.Verbose($"ProxyTestSessionManager.DequeueProxy: A proxy exists, but the runsettings do not match. Skipping it. Incoming settings: {runSettings}, Settings on proxy: {_testSessionCriteria.RunSettings}"); + throw new InvalidOperationException( + string.Format( + CultureInfo.CurrentUICulture, + CrossPlatResources.NoProxyMatchesDescription)); + } // Get the actual proxy. proxyContainer = _proxyContainerList[_proxyMap[source]]; diff --git a/src/Microsoft.TestPlatform.TestHostProvider/Hosting/DefaultTestHostManager.cs b/src/Microsoft.TestPlatform.TestHostProvider/Hosting/DefaultTestHostManager.cs index 913015ad2c..a522c7c150 100644 --- a/src/Microsoft.TestPlatform.TestHostProvider/Hosting/DefaultTestHostManager.cs +++ b/src/Microsoft.TestPlatform.TestHostProvider/Hosting/DefaultTestHostManager.cs @@ -305,8 +305,7 @@ public void Initialize(IMessageLogger logger, string runsettingsXml) _targetFramework = runConfiguration.TargetFramework; _testHostProcess = null; - // REVIEW: this is only here to force the use of the parallel proxy manager even when the default is netfmw and single source, so I don't have to fix all the logic right now - Shared = false; // !runConfiguration.DisableAppDomain; + Shared = !runConfiguration.DisableAppDomain; _hostExitedEventRaised = false; } diff --git a/src/vstest.console/TestPlatformHelpers/TestRequestManager.cs b/src/vstest.console/TestPlatformHelpers/TestRequestManager.cs index 2a1ca8980f..ea751aaca6 100644 --- a/src/vstest.console/TestPlatformHelpers/TestRequestManager.cs +++ b/src/vstest.console/TestPlatformHelpers/TestRequestManager.cs @@ -877,7 +877,6 @@ private bool UpdatePlatform( return false; } - // REVIEW: this should make the target platform empty, or "default" rather than the inferred platform, and we will then use the sources map to choose the correct testhost provider, and populate the runsettings there // Autodetect platform from sources, and return a single common platform. commonPlatform = _inferHelper.AutoDetectArchitecture(sources, defaultArchitecture, out sourceToPlatformMap); InferRunSettingsHelper.UpdateTargetPlatform(document, commonPlatform.ToString(), overwrite: true); @@ -910,8 +909,6 @@ private bool UpdateFrameworkInRunSettingsIfRequired( return false; } - - // REVIEW: this should make the target framework empty, or "default" rather than the inferred framework, and we will then use the sources map to choose the correct testhost provider, and populate the runsettings there // Autodetect framework from sources, and return a single common platform. commonFramework = _inferHelper.AutoDetectFramework(sources, out sourceToFrameworkMap); InferRunSettingsHelper.UpdateTargetFramework(document, commonFramework.ToString(), overwrite: true); From dbfdcd385e032ebcefdc01ca501a805f6edf209f Mon Sep 17 00:00:00 2001 From: nohwnd Date: Wed, 2 Mar 2022 10:11:37 +0100 Subject: [PATCH 013/112] Fixing tests --- playground/TestPlatform.Playground/Program.cs | 16 +- .../TestPlatform.cs | 19 +- .../Parallel/ParallelOperationManager.cs | 15 +- .../Parallel/ParallelProxyDiscoveryManager.cs | 1 + .../Client/TestRuntimeProviderInfo.cs | 3 +- .../PublicAPI/PublicAPI.Unshipped.txt | 12 +- .../TestEngine.cs | 10 +- .../TestSession/ProxyTestSessionManager.cs | 1 + .../Utilities/SourceDetailHelper.cs | 2 +- .../Friends.cs | 1 + .../PublicAPI/PublicAPI.Shipped.txt | 9 +- .../PublicAPI/PublicAPI.Unshipped.txt | 2 +- .../TestPlatformTests.cs | 165 +--- .../Hosting/TestHostProviderManagerTests.cs | 16 +- ...lectorAttachmentsProcessorsFactoryTests.cs | 2 +- .../Parallel/ParallelOperationManagerTests.cs | 366 ++++---- .../ParallelProxyDiscoveryManagerTests.cs | 22 +- .../ParallelProxyExecutionManagerTests.cs | 74 +- .../Client/ProxyTestSessionManagerTests.cs | 34 +- .../TestEngineTests.cs | 809 +++++++++++------- .../TestSession/TestSessionPoolTests.cs | 13 +- .../BasicRunAndDiscovery.cs | 4 +- .../Fakes/FakeDiscoveryEventsRegistrar.cs | 2 +- .../Fakes/FakeTestRuntimeProviderManager.cs | 2 - .../Fakes/FakeTestSessionEventsHandler.cs | 5 +- .../Fakes/TestMessage.cs | 5 +- .../CommandLine/InferHelperTests.cs | 57 +- ...llyQualifiedTestsArgumentProcessorTests.cs | 12 +- .../ListTestsArgumentProcessorTests.cs | 10 +- .../RunSpecificTestsArgumentProcessorTests.cs | 62 +- .../RunTestsArgumentProcessorTests.cs | 10 +- .../TestRequestManagerTests.cs | 107 +-- 32 files changed, 1034 insertions(+), 834 deletions(-) diff --git a/playground/TestPlatform.Playground/Program.cs b/playground/TestPlatform.Playground/Program.cs index c45644e110..8559ceaa90 100644 --- a/playground/TestPlatform.Playground/Program.cs +++ b/playground/TestPlatform.Playground/Program.cs @@ -39,13 +39,7 @@ static void Main(string[] args) var playground = Path.GetFullPath(Path.Combine(here, "..", "..", "..", "..")); var console = Path.Combine(here, "vstest.console", "vstest.console.exe"); - var consoleOptions = new ConsoleParameters - { - LogFilePath = Path.Combine(here, "logs", "log.txt"), - TraceLevel = TraceLevel.Verbose, - }; - var r = new VsTestConsoleWrapper(console, consoleOptions); var sourceSettings = @" @@ -77,8 +71,14 @@ static void Main(string[] args) } // design mode - // var options = new TestPlatformOptions(); - // r.RunTestsWithCustomTestHost(sources, sourceSettings, options, new TestRunHandler(), new DebuggerTestHostLauncher()); + var consoleOptions = new ConsoleParameters + { + LogFilePath = Path.Combine(here, "logs", "log.txt"), + TraceLevel = TraceLevel.Verbose, + }; + var options = new TestPlatformOptions(); + var r = new VsTestConsoleWrapper(console, consoleOptions); + r.RunTestsWithCustomTestHost(sources, sourceSettings, options, new TestRunHandler(), new DebuggerTestHostLauncher()); } public class TestRunHandler : ITestRunEventsHandler diff --git a/src/Microsoft.TestPlatform.Client/TestPlatform.cs b/src/Microsoft.TestPlatform.Client/TestPlatform.cs index c601be9aa5..dbe224a0d1 100644 --- a/src/Microsoft.TestPlatform.Client/TestPlatform.cs +++ b/src/Microsoft.TestPlatform.Client/TestPlatform.cs @@ -13,7 +13,6 @@ using Microsoft.VisualStudio.TestPlatform.Common; using Microsoft.VisualStudio.TestPlatform.Common.ExtensionFramework; using Microsoft.VisualStudio.TestPlatform.Common.Hosting; -using Microsoft.VisualStudio.TestPlatform.Common.Logging; using Microsoft.VisualStudio.TestPlatform.Common.Utilities; using Microsoft.VisualStudio.TestPlatform.CrossPlatEngine; using Microsoft.VisualStudio.TestPlatform.ObjectModel; @@ -71,7 +70,7 @@ protected internal TestPlatform( IFileHelper filehelper, ITestRuntimeProviderManager testHostProviderManager) { - TestEngine = testEngine; + _testEngine = testEngine; _fileHelper = filehelper; _testHostProviderManager = testHostProviderManager; } @@ -79,7 +78,7 @@ protected internal TestPlatform( /// /// Gets or sets the test engine instance. /// - private ITestEngine TestEngine { get; set; } + private readonly ITestEngine _testEngine; /// public IDiscoveryRequest CreateDiscoveryRequest( @@ -91,10 +90,10 @@ public IDiscoveryRequest CreateDiscoveryRequest( PopulateExtensions(discoveryCriteria.RunSettings, discoveryCriteria.Sources); // Initialize loggers. - ITestLoggerManager loggerManager = TestEngine.GetLoggerManager(requestData); + ITestLoggerManager loggerManager = _testEngine.GetLoggerManager(requestData); loggerManager.Initialize(discoveryCriteria.RunSettings); - IProxyDiscoveryManager discoveryManager = TestEngine.GetDiscoveryManager(requestData, discoveryCriteria, sourceToSourceDetailMap); + IProxyDiscoveryManager discoveryManager = _testEngine.GetDiscoveryManager(requestData, discoveryCriteria, sourceToSourceDetailMap); discoveryManager.Initialize(options?.SkipDefaultAdapters ?? false); return new DiscoveryRequest(requestData, discoveryCriteria, discoveryManager, loggerManager); @@ -111,10 +110,10 @@ public ITestRunRequest CreateTestRunRequest( PopulateExtensions(testRunCriteria.TestRunSettings, sources); // Initialize loggers. - ITestLoggerManager loggerManager = TestEngine.GetLoggerManager(requestData); + ITestLoggerManager loggerManager = _testEngine.GetLoggerManager(requestData); loggerManager.Initialize(testRunCriteria.TestRunSettings); - IProxyExecutionManager executionManager = TestEngine.GetExecutionManager(requestData, testRunCriteria, sourceToSourceDetailMap); + IProxyExecutionManager executionManager = _testEngine.GetExecutionManager(requestData, testRunCriteria, sourceToSourceDetailMap); executionManager.Initialize(options?.SkipDefaultAdapters ?? false); return new TestRunRequest(requestData, testRunCriteria, executionManager, loggerManager); @@ -137,7 +136,7 @@ public bool StartTestSession( return false; } - IProxyTestSessionManager testSessionManager = TestEngine.GetTestSessionManager(requestData, testSessionCriteria, sourceToSourceDetailMap); + IProxyTestSessionManager testSessionManager = _testEngine.GetTestSessionManager(requestData, testSessionCriteria, sourceToSourceDetailMap); if (testSessionManager == null) { // The test session manager is null because the combination of runsettings and @@ -179,13 +178,13 @@ public void UpdateExtensions( IEnumerable pathToAdditionalExtensions, bool skipExtensionFilters) { - TestEngine.GetExtensionManager().UseAdditionalExtensions(pathToAdditionalExtensions, skipExtensionFilters); + _testEngine.GetExtensionManager().UseAdditionalExtensions(pathToAdditionalExtensions, skipExtensionFilters); } /// public void ClearExtensions() { - TestEngine.GetExtensionManager().ClearExtensions(); + _testEngine.GetExtensionManager().ClearExtensions(); } private static void ThrowExceptionIfTestHostManagerIsNull( diff --git a/src/Microsoft.TestPlatform.CrossPlatEngine/Client/Parallel/ParallelOperationManager.cs b/src/Microsoft.TestPlatform.CrossPlatEngine/Client/Parallel/ParallelOperationManager.cs index 5e3079f8e6..1e00d5c2c1 100644 --- a/src/Microsoft.TestPlatform.CrossPlatEngine/Client/Parallel/ParallelOperationManager.cs +++ b/src/Microsoft.TestPlatform.CrossPlatEngine/Client/Parallel/ParallelOperationManager.cs @@ -22,9 +22,9 @@ internal sealed class ParallelOperationManager /// Default number of Processes /// - private TEventHandler _eventHandler; - private Func _getEventHandler; - private Action _runWorkload; + private TEventHandler? _eventHandler; + private Func? _getEventHandler; + private Action? _runWorkload; private readonly List> _workloads = new(); private readonly List _managerSlots = new(); @@ -71,6 +71,15 @@ private bool RunWorkInParallel() // where we should find a workload that fits the manager if any of them is shared. // Or tear it down, and start a new one. + if (_eventHandler == null) + throw new InvalidOperationException($"{nameof(_eventHandler)} was not provided."); + + if (_getEventHandler == null) + throw new InvalidOperationException($"{nameof(_eventHandler)} was not provided."); + + if (_runWorkload == null) + throw new InvalidOperationException($"{nameof(_runWorkload)} was not provided."); + List workToRun = new(); lock (_lock) { diff --git a/src/Microsoft.TestPlatform.CrossPlatEngine/Client/Parallel/ParallelProxyDiscoveryManager.cs b/src/Microsoft.TestPlatform.CrossPlatEngine/Client/Parallel/ParallelProxyDiscoveryManager.cs index b8065ebaab..8f0a19f37c 100644 --- a/src/Microsoft.TestPlatform.CrossPlatEngine/Client/Parallel/ParallelProxyDiscoveryManager.cs +++ b/src/Microsoft.TestPlatform.CrossPlatEngine/Client/Parallel/ParallelProxyDiscoveryManager.cs @@ -202,6 +202,7 @@ private void DiscoverTestsOnConcurrentManager(IProxyDiscoveryManager proxyDiscov { EqtTrace.Verbose("ParallelProxyDiscoveryManager: Discovery started."); + proxyDiscoveryManager.Initialize(_skipDefaultAdapters); proxyDiscoveryManager.DiscoverTests(discoveryCriteria, eventHandler); }) .ContinueWith(t => diff --git a/src/Microsoft.TestPlatform.CrossPlatEngine/Client/TestRuntimeProviderInfo.cs b/src/Microsoft.TestPlatform.CrossPlatEngine/Client/TestRuntimeProviderInfo.cs index 449b828beb..5a1a409a1f 100644 --- a/src/Microsoft.TestPlatform.CrossPlatEngine/Client/TestRuntimeProviderInfo.cs +++ b/src/Microsoft.TestPlatform.CrossPlatEngine/Client/TestRuntimeProviderInfo.cs @@ -1,13 +1,14 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT license. See LICENSE file in the project root for full license information. -namespace Microsoft.VisualStudio.TestPlatform.CrossPlatEngine.Client; using System; using System.Collections.Generic; using Microsoft.VisualStudio.TestPlatform.ObjectModel; +namespace Microsoft.VisualStudio.TestPlatform.CrossPlatEngine.Client; + public class TestRuntimeProviderInfo { public Type Type { get; } diff --git a/src/Microsoft.TestPlatform.CrossPlatEngine/PublicAPI/PublicAPI.Unshipped.txt b/src/Microsoft.TestPlatform.CrossPlatEngine/PublicAPI/PublicAPI.Unshipped.txt index 5fd225b90b..6500ab0e5f 100644 --- a/src/Microsoft.TestPlatform.CrossPlatEngine/PublicAPI/PublicAPI.Unshipped.txt +++ b/src/Microsoft.TestPlatform.CrossPlatEngine/PublicAPI/PublicAPI.Unshipped.txt @@ -2,4 +2,14 @@ virtual Microsoft.VisualStudio.TestPlatform.CrossPlatEngine.TestSessionPool.Kill virtual Microsoft.VisualStudio.TestPlatform.CrossPlatEngine.ProxyTestSessionManager.StartSession(Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.ITestSessionEventsHandler eventsHandler, Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.IRequestData requestData) -> bool virtual Microsoft.VisualStudio.TestPlatform.CrossPlatEngine.ProxyTestSessionManager.StopSession(Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.IRequestData requestData) -> bool Microsoft.VisualStudio.TestPlatform.CrossPlatEngine.Client.ProxyDiscoveryManager.Abort(Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.ITestDiscoveryEventsHandler2 eventHandler) -> void -Microsoft.VisualStudio.TestPlatform.CrossPlatEngine.Discovery.DiscoveryManager.Abort(Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.ITestDiscoveryEventsHandler2 eventHandler) -> void \ No newline at end of file +Microsoft.VisualStudio.TestPlatform.CrossPlatEngine.Discovery.DiscoveryManager.Abort(Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.ITestDiscoveryEventsHandler2 eventHandler) -> void +Microsoft.VisualStudio.TestPlatform.CrossPlatEngine.ProxyTestSessionManager.ProxyTestSessionManager(Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.StartTestSessionCriteria criteria, int maxTesthostCount, System.Func proxyCreator, System.Collections.Generic.List runtimeProviders) -> void +Microsoft.VisualStudio.TestPlatform.CrossPlatEngine.TestEngine.GetDiscoveryManager(Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.IRequestData requestData, Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.DiscoveryCriteria discoveryCriteria, System.Collections.Generic.Dictionary sourceToSourceDetailMap) -> Microsoft.VisualStudio.TestPlatform.ObjectModel.Engine.IProxyDiscoveryManager +Microsoft.VisualStudio.TestPlatform.CrossPlatEngine.TestEngine.GetExecutionManager(Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.IRequestData requestData, Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.TestRunCriteria testRunCriteria, System.Collections.Generic.Dictionary sourceToSourceDetailMap) -> Microsoft.VisualStudio.TestPlatform.ObjectModel.Engine.IProxyExecutionManager +Microsoft.VisualStudio.TestPlatform.CrossPlatEngine.TestEngine.GetTestSessionManager(Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.IRequestData requestData, Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.StartTestSessionCriteria testSessionCriteria, System.Collections.Generic.Dictionary sourceToSourceDetailMap) -> Microsoft.VisualStudio.TestPlatform.ObjectModel.Engine.IProxyTestSessionManager +Microsoft.VisualStudio.TestPlatform.CrossPlatEngine.Client.TestRuntimeProviderInfo +Microsoft.VisualStudio.TestPlatform.CrossPlatEngine.Client.TestRuntimeProviderInfo.RunSettings.get -> string +Microsoft.VisualStudio.TestPlatform.CrossPlatEngine.Client.TestRuntimeProviderInfo.Shared.get -> bool +Microsoft.VisualStudio.TestPlatform.CrossPlatEngine.Client.TestRuntimeProviderInfo.SourceDetails.get -> System.Collections.Generic.List +Microsoft.VisualStudio.TestPlatform.CrossPlatEngine.Client.TestRuntimeProviderInfo.TestRuntimeProviderInfo(System.Type type, bool shared, string runSettings, System.Collections.Generic.List sourceDetails) -> void +Microsoft.VisualStudio.TestPlatform.CrossPlatEngine.Client.TestRuntimeProviderInfo.Type.get -> System.Type diff --git a/src/Microsoft.TestPlatform.CrossPlatEngine/TestEngine.cs b/src/Microsoft.TestPlatform.CrossPlatEngine/TestEngine.cs index 146628d8aa..cf2adf39d1 100644 --- a/src/Microsoft.TestPlatform.CrossPlatEngine/TestEngine.cs +++ b/src/Microsoft.TestPlatform.CrossPlatEngine/TestEngine.cs @@ -25,7 +25,6 @@ using Microsoft.VisualStudio.TestPlatform.PlatformAbstractions.Interfaces; using Microsoft.VisualStudio.TestPlatform.Utilities; using System.Text; -using Microsoft.VisualStudio.TestPlatform.Common.Interfaces; #nullable disable @@ -404,7 +403,7 @@ private List GetTestRuntimeProvidersForUniqueConfigurat testRuntimeProviders.Add(testRuntimeProviderInfo); } - ThrowExceptionIfAnyTestHostManagerIsNull(testRuntimeProviders); + ThrowExceptionIfAnyTestHostManagerIsNullOrNoneAreFound(testRuntimeProviders); return testRuntimeProviders; } @@ -585,8 +584,11 @@ private static void ThrowExceptionIfTestHostManagerIsNull(ITestRuntimeProvider t } } - private static void ThrowExceptionIfAnyTestHostManagerIsNull(List testRuntimeProviders) + private static void ThrowExceptionIfAnyTestHostManagerIsNullOrNoneAreFound(List testRuntimeProviders) { + if (!testRuntimeProviders.Any()) + throw new ArgumentException(null, nameof(testRuntimeProviders)); + var missingRuntimeProviders = testRuntimeProviders.Where(p => p.Type == null); if (missingRuntimeProviders.Any()) { @@ -594,7 +596,7 @@ private static void ThrowExceptionIfAnyTestHostManagerIsNull(List s.Source)} and runsettings: {missingRuntimeProvider.RunSettings}"); + EqtTrace.Error($"{nameof(TestEngine)}.{nameof(ThrowExceptionIfAnyTestHostManagerIsNullOrNoneAreFound)}: No suitable testHostProvider found for sources {missingRuntimeProvider.SourceDetails.Select(s => s.Source)} and runsettings: {missingRuntimeProvider.RunSettings}"); missingRuntimeProvider.SourceDetails.ForEach(detail => stringBuilder.AppendLine(detail.Source)); } diff --git a/src/Microsoft.TestPlatform.CrossPlatEngine/TestSession/ProxyTestSessionManager.cs b/src/Microsoft.TestPlatform.CrossPlatEngine/TestSession/ProxyTestSessionManager.cs index 34fb9112f6..6a0a02d9cb 100644 --- a/src/Microsoft.TestPlatform.CrossPlatEngine/TestSession/ProxyTestSessionManager.cs +++ b/src/Microsoft.TestPlatform.CrossPlatEngine/TestSession/ProxyTestSessionManager.cs @@ -16,6 +16,7 @@ using CrossPlatResources = Microsoft.VisualStudio.TestPlatform.CrossPlatEngine.Resources.Resources; +#nullable disable namespace Microsoft.VisualStudio.TestPlatform.CrossPlatEngine; diff --git a/src/Microsoft.TestPlatform.CrossPlatEngine/Utilities/SourceDetailHelper.cs b/src/Microsoft.TestPlatform.CrossPlatEngine/Utilities/SourceDetailHelper.cs index 586a5aa34c..ea4813b6bd 100644 --- a/src/Microsoft.TestPlatform.CrossPlatEngine/Utilities/SourceDetailHelper.cs +++ b/src/Microsoft.TestPlatform.CrossPlatEngine/Utilities/SourceDetailHelper.cs @@ -1,7 +1,6 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT license. See LICENSE file in the project root for full license information. -namespace Microsoft.VisualStudio.TestPlatform.CrossPlatEngine.Utilities; using System.IO; using System.Xml; @@ -10,6 +9,7 @@ namespace Microsoft.VisualStudio.TestPlatform.CrossPlatEngine.Utilities; using Microsoft.VisualStudio.TestPlatform.ObjectModel.Utilities; using Microsoft.VisualStudio.TestPlatform.Utilities; +namespace Microsoft.VisualStudio.TestPlatform.CrossPlatEngine.Utilities; internal static class SourceDetailHelper { internal static string UpdateRunSettingsFromSourceDetail(string runSettings, SourceDetail sourceDetail) diff --git a/src/Microsoft.TestPlatform.ObjectModel/Friends.cs b/src/Microsoft.TestPlatform.ObjectModel/Friends.cs index 1eabaf534d..00df175bfe 100644 --- a/src/Microsoft.TestPlatform.ObjectModel/Friends.cs +++ b/src/Microsoft.TestPlatform.ObjectModel/Friends.cs @@ -12,6 +12,7 @@ [assembly: InternalsVisibleTo("Microsoft.VisualStudio.TestPlatform.Client, PublicKey=002400000480000094000000060200000024000052534131000400000100010007d1fa57c4aed9f0a32e84aa0faefd0de9e8fd6aec8f87fb03766c834c99921eb23be79ad9d5dcc1dd9ad236132102900b723cf980957fc4e177108fc607774f29e8320e92ea05ece4e821c0a5efe8f1645c4c0c93c1ab99285d622caa652c1dfad63d745d6f2de5f17e5eaf0fc4963d261c8a12436518206dc093344d5ad293")] [assembly: InternalsVisibleTo("Microsoft.TestPlatform.ObjectModel.UnitTests, PublicKey=002400000480000094000000060200000024000052534131000400000100010007d1fa57c4aed9f0a32e84aa0faefd0de9e8fd6aec8f87fb03766c834c99921eb23be79ad9d5dcc1dd9ad236132102900b723cf980957fc4e177108fc607774f29e8320e92ea05ece4e821c0a5efe8f1645c4c0c93c1ab99285d622caa652c1dfad63d745d6f2de5f17e5eaf0fc4963d261c8a12436518206dc093344d5ad293")] +[assembly: InternalsVisibleTo("Microsoft.TestPlatform.CrossPlatEngine.UnitTests, PublicKey=002400000480000094000000060200000024000052534131000400000100010007d1fa57c4aed9f0a32e84aa0faefd0de9e8fd6aec8f87fb03766c834c99921eb23be79ad9d5dcc1dd9ad236132102900b723cf980957fc4e177108fc607774f29e8320e92ea05ece4e821c0a5efe8f1645c4c0c93c1ab99285d622caa652c1dfad63d745d6f2de5f17e5eaf0fc4963d261c8a12436518206dc093344d5ad293")] [assembly: InternalsVisibleTo("datacollector.UnitTests, PublicKey=002400000480000094000000060200000024000052534131000400000100010007d1fa57c4aed9f0a32e84aa0faefd0de9e8fd6aec8f87fb03766c834c99921eb23be79ad9d5dcc1dd9ad236132102900b723cf980957fc4e177108fc607774f29e8320e92ea05ece4e821c0a5efe8f1645c4c0c93c1ab99285d622caa652c1dfad63d745d6f2de5f17e5eaf0fc4963d261c8a12436518206dc093344d5ad293")] [assembly: InternalsVisibleTo("Microsoft.TestPlatform.Extensions.EventLogCollector.UnitTests, PublicKey=002400000480000094000000060200000024000052534131000400000100010007d1fa57c4aed9f0a32e84aa0faefd0de9e8fd6aec8f87fb03766c834c99921eb23be79ad9d5dcc1dd9ad236132102900b723cf980957fc4e177108fc607774f29e8320e92ea05ece4e821c0a5efe8f1645c4c0c93c1ab99285d622caa652c1dfad63d745d6f2de5f17e5eaf0fc4963d261c8a12436518206dc093344d5ad293")] [assembly: InternalsVisibleTo("Microsoft.TestPlatform.ObjectModel.ManagedNameUtilities.UnitTests, PublicKey=002400000480000094000000060200000024000052534131000400000100010007d1fa57c4aed9f0a32e84aa0faefd0de9e8fd6aec8f87fb03766c834c99921eb23be79ad9d5dcc1dd9ad236132102900b723cf980957fc4e177108fc607774f29e8320e92ea05ece4e821c0a5efe8f1645c4c0c93c1ab99285d622caa652c1dfad63d745d6f2de5f17e5eaf0fc4963d261c8a12436518206dc093344d5ad293")] diff --git a/src/Microsoft.TestPlatform.ObjectModel/PublicAPI/PublicAPI.Shipped.txt b/src/Microsoft.TestPlatform.ObjectModel/PublicAPI/PublicAPI.Shipped.txt index 05faa716a2..dcd437d876 100644 --- a/src/Microsoft.TestPlatform.ObjectModel/PublicAPI/PublicAPI.Shipped.txt +++ b/src/Microsoft.TestPlatform.ObjectModel/PublicAPI/PublicAPI.Shipped.txt @@ -228,8 +228,8 @@ Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.ITestMessageEventHandler. Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.ITestMessageEventHandler.HandleRawMessage(string rawMessage) -> void Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.ITestPlatform Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.ITestPlatform.ClearExtensions() -> void -Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.ITestPlatform.CreateDiscoveryRequest(Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.IRequestData requestData, Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.DiscoveryCriteria discoveryCriteria, Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.TestPlatformOptions options) -> Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.IDiscoveryRequest -Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.ITestPlatform.CreateTestRunRequest(Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.IRequestData requestData, Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.TestRunCriteria testRunCriteria, Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.TestPlatformOptions options) -> Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.ITestRunRequest +Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.ITestPlatform.CreateDiscoveryRequest(Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.IRequestData requestData, Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.DiscoveryCriteria discoveryCriteria, Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.TestPlatformOptions options, System.Collections.Generic.Dictionary sourceToSourceDetailMap) -> Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.IDiscoveryRequest +Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.ITestPlatform.CreateTestRunRequest(Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.IRequestData requestData, Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.TestRunCriteria testRunCriteria, Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.TestPlatformOptions options, System.Collections.Generic.Dictionary sourceToSourceDetailMap) -> Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.ITestRunRequest Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.ITestPlatform.UpdateExtensions(System.Collections.Generic.IEnumerable pathToAdditionalExtensions, bool skipExtensionFilters) -> void Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.ITestPlatformCapabilities Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.ITestPlatformCapabilities.TestPlatformType.get -> Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.TestPlatformType @@ -881,3 +881,8 @@ Microsoft.VisualStudio.TestPlatform.ObjectModel.InvokedDataCollector.Uri.get -> override Microsoft.VisualStudio.TestPlatform.ObjectModel.InvokedDataCollector.Equals(object obj) -> bool override Microsoft.VisualStudio.TestPlatform.ObjectModel.InvokedDataCollector.GetHashCode() -> int override Microsoft.VisualStudio.TestPlatform.ObjectModel.InvokedDataCollector.ToString() -> string +Microsoft.VisualStudio.TestPlatform.ObjectModel.SourceDetail +Microsoft.VisualStudio.TestPlatform.ObjectModel.SourceDetail.Architecture.get -> Microsoft.VisualStudio.TestPlatform.ObjectModel.Architecture +Microsoft.VisualStudio.TestPlatform.ObjectModel.SourceDetail.Framework.get -> Microsoft.VisualStudio.TestPlatform.ObjectModel.Framework +Microsoft.VisualStudio.TestPlatform.ObjectModel.SourceDetail.Source.get -> string +Microsoft.VisualStudio.TestPlatform.ObjectModel.SourceDetail.SourceDetail() -> void diff --git a/src/Microsoft.TestPlatform.ObjectModel/PublicAPI/PublicAPI.Unshipped.txt b/src/Microsoft.TestPlatform.ObjectModel/PublicAPI/PublicAPI.Unshipped.txt index da26b150f1..4ea02d56ac 100644 --- a/src/Microsoft.TestPlatform.ObjectModel/PublicAPI/PublicAPI.Unshipped.txt +++ b/src/Microsoft.TestPlatform.ObjectModel/PublicAPI/PublicAPI.Unshipped.txt @@ -11,7 +11,7 @@ Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.DiscoveryCriteria.TestSes Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.DiscoveryCriteria.TestSessionInfo.set -> void Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.DiscoveryRequestPayload.TestSessionInfo.get -> Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.TestSessionInfo Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.DiscoveryRequestPayload.TestSessionInfo.set -> void -Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.ITestPlatform.StartTestSession(Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.IRequestData requestData, Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.StartTestSessionCriteria criteria, Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.ITestSessionEventsHandler eventsHandler) -> bool +Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.ITestPlatform.StartTestSession(Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.IRequestData requestData, Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.StartTestSessionCriteria criteria, Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.ITestSessionEventsHandler eventsHandler, System.Collections.Generic.Dictionary sourceToSourceDetailMap) -> bool Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.ITestSessionEventsHandler.HandleStartTestSessionComplete(Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.StartTestSessionCompleteEventArgs eventArgs) -> void Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.ITestSessionEventsHandler.HandleStopTestSessionComplete(Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.StopTestSessionCompleteEventArgs eventArgs) -> void Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.Payloads.StartTestSessionAckPayload.EventArgs.get -> Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.StartTestSessionCompleteEventArgs diff --git a/test/Microsoft.TestPlatform.Client.UnitTests/TestPlatformTests.cs b/test/Microsoft.TestPlatform.Client.UnitTests/TestPlatformTests.cs index 93205f1996..f450c50d4f 100644 --- a/test/Microsoft.TestPlatform.Client.UnitTests/TestPlatformTests.cs +++ b/test/Microsoft.TestPlatform.Client.UnitTests/TestPlatformTests.cs @@ -51,21 +51,20 @@ public TestPlatformTests() } [TestMethod] - public void CreateDiscoveryRequestShouldInitializeManagersAndCreateDiscoveryRequestWithGivenCriteriaAndReturnIt() + public void CreateDiscoveryRequestShouldInitializeDiscoveryManagerAndCreateDiscoveryRequestWithGivenCriteriaAndReturnIt() { _discoveryManager.Setup(dm => dm.Initialize(false)).Verifiable(); var discoveryCriteria = new DiscoveryCriteria(new List { "foo" }, 1, null); _hostManager.Setup(hm => hm.GetTestSources(discoveryCriteria.Sources)) .Returns(discoveryCriteria.Sources); - _testEngine.Setup(te => te.GetDiscoveryManager(_mockRequestData.Object, _hostManager.Object, It.IsAny())).Returns(_discoveryManager.Object); + _testEngine.Setup(te => te.GetDiscoveryManager(_mockRequestData.Object, It.IsAny(), It.IsAny>())).Returns(_discoveryManager.Object); _testEngine.Setup(te => te.GetExtensionManager()).Returns(_extensionManager.Object); _testEngine.Setup(te => te.GetLoggerManager(_mockRequestData.Object)).Returns(_loggerManager.Object); var tp = new TestableTestPlatform(_testEngine.Object, _hostManager.Object); - var discoveryRequest = tp.CreateDiscoveryRequest(_mockRequestData.Object, discoveryCriteria, new TestPlatformOptions()); + var discoveryRequest = tp.CreateDiscoveryRequest(_mockRequestData.Object, discoveryCriteria, new TestPlatformOptions(), It.IsAny>()); - _hostManager.Verify(hm => hm.Initialize(It.IsAny(), It.IsAny()), Times.Once); _discoveryManager.Verify(dm => dm.Initialize(false), Times.Once); Assert.AreEqual(discoveryCriteria, discoveryRequest.DiscoveryCriteria); } @@ -109,7 +108,7 @@ public void CreateDiscoveryRequestThrowsIfDiscoveryCriteriaIsNull() { TestPlatform tp = new(); - Assert.ThrowsException(() => tp.CreateDiscoveryRequest(_mockRequestData.Object, null, new TestPlatformOptions())); + Assert.ThrowsException(() => tp.CreateDiscoveryRequest(_mockRequestData.Object, null, new TestPlatformOptions(), It.IsAny>())); } [TestMethod] @@ -135,36 +134,6 @@ public void ClearExtensionsShouldClearTheExtensionsCachedInEngine() _extensionManager.Verify(em => em.ClearExtensions()); } - [TestMethod] - public void CreateTestRunRequestShouldThrowExceptionIfNoTestHostproviderFound() - { - _testEngine.Setup(te => te.GetLoggerManager(_mockRequestData.Object)).Returns(_loggerManager.Object); - - string settingsXml = - @" - - - .NETPortable,Version=v4.5 - - "; - - var testRunCriteria = new TestRunCriteria(new List { @"x:dummy\foo.dll" }, 10, false, settingsXml, TimeSpan.Zero); - var tp = new TestableTestPlatform(_testEngine.Object, _mockFileHelper.Object, null); - bool exceptionThrown = false; - - try - { - tp.CreateTestRunRequest(_mockRequestData.Object, testRunCriteria, new TestPlatformOptions()); - } - catch (TestPlatformException ex) - { - exceptionThrown = true; - Assert.AreEqual("No suitable test runtime provider found for this run.", ex.Message); - } - - Assert.IsTrue(exceptionThrown, "TestPlatformException should get thrown"); - } - [TestMethod] public void CreateTestRunRequestShouldUpdateLoggerExtensionWhenDesingModeIsFalseForRunAll() { @@ -186,13 +155,13 @@ public void CreateTestRunRequestShouldUpdateLoggerExtensionWhenDesingModeIsFalse _hostManager.Setup(hm => hm.GetTestSources(testRunCriteria.Sources)) .Returns(testRunCriteria.Sources); - _testEngine.Setup(te => te.GetExecutionManager(_mockRequestData.Object, _hostManager.Object, It.IsAny())).Returns(_executionManager.Object); + _testEngine.Setup(te => te.GetExecutionManager(_mockRequestData.Object, It.IsAny(), It.IsAny>())).Returns(_executionManager.Object); _testEngine.Setup(te => te.GetExtensionManager()).Returns(_extensionManager.Object); _testEngine.Setup(te => te.GetLoggerManager(_mockRequestData.Object)).Returns(_loggerManager.Object); var tp = new TestableTestPlatform(_testEngine.Object, _mockFileHelper.Object, _hostManager.Object); - var testRunRequest = tp.CreateTestRunRequest(_mockRequestData.Object, testRunCriteria, new TestPlatformOptions()); + var testRunRequest = tp.CreateTestRunRequest(_mockRequestData.Object, testRunCriteria, new TestPlatformOptions(), It.IsAny>()); _extensionManager.Verify(em => em.UseAdditionalExtensions(additionalExtensions, false)); } @@ -215,13 +184,13 @@ public void CreateTestRunRequestShouldUpdateLoggerExtensionWhenDesignModeIsFalse var testRunCriteria = new TestRunCriteria(new List { new TestCase("dll1.class1.test1", new Uri("hello://x/"), $"xyz{Path.DirectorySeparatorChar}1.dll") }, 10, false, settingsXml); - _testEngine.Setup(te => te.GetExecutionManager(_mockRequestData.Object, _hostManager.Object, It.IsAny())).Returns(_executionManager.Object); + _testEngine.Setup(te => te.GetExecutionManager(_mockRequestData.Object, It.IsAny(), It.IsAny>())).Returns(_executionManager.Object); _testEngine.Setup(te => te.GetExtensionManager()).Returns(_extensionManager.Object); _testEngine.Setup(te => te.GetLoggerManager(_mockRequestData.Object)).Returns(_loggerManager.Object); var tp = new TestableTestPlatform(_testEngine.Object, _mockFileHelper.Object, _hostManager.Object); - var testRunRequest = tp.CreateTestRunRequest(_mockRequestData.Object, testRunCriteria, new TestPlatformOptions()); + var testRunRequest = tp.CreateTestRunRequest(_mockRequestData.Object, testRunCriteria, new TestPlatformOptions(), It.IsAny>()); _extensionManager.Verify(em => em.UseAdditionalExtensions(additionalExtensions, false)); } @@ -246,13 +215,13 @@ public void CreateTestRunRequestShouldNotUpdateTestSourcesIfSelectedTestAreRun() _hostManager.Setup(hm => hm.GetTestSources(It.IsAny>())) .Returns(new List { $"xyz{Path.DirectorySeparatorChar}1.dll" }); - _testEngine.Setup(te => te.GetExecutionManager(_mockRequestData.Object, _hostManager.Object, It.IsAny())).Returns(_executionManager.Object); + _testEngine.Setup(te => te.GetExecutionManager(_mockRequestData.Object, It.IsAny(), It.IsAny>())).Returns(_executionManager.Object); _testEngine.Setup(te => te.GetExtensionManager()).Returns(_extensionManager.Object); _testEngine.Setup(te => te.GetLoggerManager(_mockRequestData.Object)).Returns(_loggerManager.Object); var tp = new TestableTestPlatform(_testEngine.Object, _mockFileHelper.Object, _hostManager.Object); - tp.CreateTestRunRequest(_mockRequestData.Object, testRunCriteria, new TestPlatformOptions()); + tp.CreateTestRunRequest(_mockRequestData.Object, testRunCriteria, new TestPlatformOptions(), It.IsAny>()); _extensionManager.Verify(em => em.UseAdditionalExtensions(additionalExtensions, false)); _hostManager.Verify(hm => hm.GetTestSources(It.IsAny>()), Times.Never); } @@ -261,7 +230,7 @@ public void CreateTestRunRequestShouldNotUpdateTestSourcesIfSelectedTestAreRun() public void CreateTestRunRequestShouldInitializeManagersAndCreateTestRunRequestWithSpecifiedCriteria() { _executionManager.Setup(dm => dm.Initialize(false)).Verifiable(); - _testEngine.Setup(te => te.GetExecutionManager(_mockRequestData.Object, _hostManager.Object, It.IsAny())).Returns(_executionManager.Object); + _testEngine.Setup(te => te.GetExecutionManager(_mockRequestData.Object, It.IsAny(), It.IsAny>())).Returns(_executionManager.Object); _testEngine.Setup(te => te.GetExtensionManager()).Returns(_extensionManager.Object); _testEngine.Setup(te => te.GetLoggerManager(_mockRequestData.Object)).Returns(_loggerManager.Object); @@ -270,11 +239,10 @@ public void CreateTestRunRequestShouldInitializeManagersAndCreateTestRunRequestW _hostManager.Setup(hm => hm.GetTestSources(testRunCriteria.Sources)) .Returns(testRunCriteria.Sources); - var testRunRequest = tp.CreateTestRunRequest(_mockRequestData.Object, testRunCriteria, new TestPlatformOptions()); + var testRunRequest = tp.CreateTestRunRequest(_mockRequestData.Object, testRunCriteria, new TestPlatformOptions(), It.IsAny>()); var actualTestRunRequest = testRunRequest as TestRunRequest; - _hostManager.Verify(hm => hm.Initialize(It.IsAny(), It.IsAny()), Times.Once); _executionManager.Verify(em => em.Initialize(false), Times.Once); Assert.AreEqual(testRunCriteria, actualTestRunRequest.TestRunCriteria); } @@ -313,65 +281,12 @@ public void CreateTestRunRequestShouldInitializeManagersWithFalseFlagWhenTestPla _executionManager.Verify(dm => dm.Initialize(false), Times.Once); } - [TestMethod] - public void CreateTestRunRequestShouldSetCustomHostLauncherOnEngineDefaultLauncherIfSpecified() - { - var mockCustomLauncher = new Mock(); - _executionManager.Setup(dm => dm.Initialize(false)).Verifiable(); - _testEngine.Setup(te => te.GetExecutionManager(_mockRequestData.Object, _hostManager.Object, It.IsAny())).Returns(_executionManager.Object); - _testEngine.Setup(te => te.GetExtensionManager()).Returns(_extensionManager.Object); - _testEngine.Setup(te => te.GetLoggerManager(_mockRequestData.Object)).Returns(_loggerManager.Object); - - var tp = new TestableTestPlatform(_testEngine.Object, _hostManager.Object); - var testRunCriteria = new TestRunCriteria(new List { "foo" }, 10, false, null, TimeSpan.Zero, mockCustomLauncher.Object); - _hostManager.Setup(hm => hm.GetTestSources(testRunCriteria.Sources)) - .Returns(testRunCriteria.Sources); - - var testRunRequest = tp.CreateTestRunRequest(_mockRequestData.Object, testRunCriteria, new TestPlatformOptions()); - - var actualTestRunRequest = testRunRequest as TestRunRequest; - Assert.AreEqual(testRunCriteria, actualTestRunRequest.TestRunCriteria); - _hostManager.Verify(hl => hl.SetCustomLauncher(mockCustomLauncher.Object), Times.Once); - } - [TestMethod] public void CreateTestRunRequestThrowsIfTestRunCriteriaIsNull() { var tp = new TestPlatform(); - Assert.ThrowsException(() => tp.CreateTestRunRequest(_mockRequestData.Object, null, new TestPlatformOptions())); - } - - - [TestMethod] - public void CreateDiscoveryRequestShouldThrowExceptionIfNoTestHostproviderFound() - { - _testEngine.Setup(te => te.GetLoggerManager(_mockRequestData.Object)) - .Returns(_loggerManager.Object); - - string settingsXml = - @" - - - .NETPortable,Version=v4.5 - - "; - - var discoveryCriteria = new DiscoveryCriteria(new List { @"x:dummy\foo.dll" }, 1, settingsXml); - var tp = new TestableTestPlatform(_testEngine.Object, _mockFileHelper.Object, null); - bool exceptionThrown = false; - - try - { - tp.CreateDiscoveryRequest(_mockRequestData.Object, discoveryCriteria, new TestPlatformOptions()); - } - catch (TestPlatformException ex) - { - exceptionThrown = true; - Assert.AreEqual("No suitable test runtime provider found for this run.", ex.Message); - } - - Assert.IsTrue(exceptionThrown, "TestPlatformException should get thrown"); + Assert.ThrowsException(() => tp.CreateTestRunRequest(_mockRequestData.Object, null, new TestPlatformOptions(), It.IsAny>())); } /// @@ -399,13 +314,13 @@ public void CreateDiscoveryRequestShouldUpdateLoggerExtensionWhenDesignModeIsFal _hostManager.Setup(hm => hm.GetTestSources(discoveryCriteria.Sources)) .Returns(discoveryCriteria.Sources); - _testEngine.Setup(te => te.GetDiscoveryManager(It.IsAny(), _hostManager.Object, It.IsAny())).Returns(_discoveryManager.Object); + _testEngine.Setup(te => te.GetDiscoveryManager(It.IsAny(), It.IsAny(), It.IsAny>())).Returns(_discoveryManager.Object); _testEngine.Setup(te => te.GetExtensionManager()).Returns(_extensionManager.Object); _testEngine.Setup(te => te.GetLoggerManager(_mockRequestData.Object)).Returns(_loggerManager.Object); var tp = new TestableTestPlatform(_testEngine.Object, _mockFileHelper.Object, _hostManager.Object); // Action - var discoveryRequest = tp.CreateDiscoveryRequest(_mockRequestData.Object, discoveryCriteria, new TestPlatformOptions()); + var discoveryRequest = tp.CreateDiscoveryRequest(_mockRequestData.Object, discoveryCriteria, new TestPlatformOptions(), It.IsAny>()); // Verify _extensionManager.Verify(em => em.UseAdditionalExtensions(additionalExtensions, false)); @@ -417,7 +332,7 @@ public void CreateDiscoveryRequestShouldUpdateLoggerExtensionWhenDesignModeIsFal [TestMethod] public void CreateTestRunRequestShouldInitializeLoggerManagerForDesignMode() { - _testEngine.Setup(te => te.GetExecutionManager(_mockRequestData.Object, _hostManager.Object, It.IsAny())).Returns(_executionManager.Object); + _testEngine.Setup(te => te.GetExecutionManager(_mockRequestData.Object, It.IsAny(), It.IsAny>())).Returns(_executionManager.Object); _testEngine.Setup(te => te.GetLoggerManager(_mockRequestData.Object)).Returns(_loggerManager.Object); string settingsXml = @@ -430,7 +345,7 @@ public void CreateTestRunRequestShouldInitializeLoggerManagerForDesignMode() var testRunCriteria = new TestRunCriteria(new List { @"x:dummy\foo.dll" }, 10, false, settingsXml); var tp = new TestableTestPlatform(_testEngine.Object, _hostManager.Object); - tp.CreateTestRunRequest(_mockRequestData.Object, testRunCriteria, new TestPlatformOptions()); + tp.CreateTestRunRequest(_mockRequestData.Object, testRunCriteria, new TestPlatformOptions(), It.IsAny>()); _loggerManager.Verify(lm => lm.Initialize(settingsXml)); } @@ -441,7 +356,7 @@ public void CreateTestRunRequestShouldInitializeLoggerManagerForDesignMode() [TestMethod] public void CreateDiscoveryRequestShouldInitializeLoggerManagerForDesignMode() { - _testEngine.Setup(te => te.GetDiscoveryManager(_mockRequestData.Object, _hostManager.Object, It.IsAny())).Returns(_discoveryManager.Object); + _testEngine.Setup(te => te.GetDiscoveryManager(_mockRequestData.Object, It.IsAny(), It.IsAny>())).Returns(_discoveryManager.Object); _testEngine.Setup(te => te.GetLoggerManager(_mockRequestData.Object)).Returns(_loggerManager.Object); string settingsXml = @@ -454,7 +369,7 @@ public void CreateDiscoveryRequestShouldInitializeLoggerManagerForDesignMode() var discoveryCriteria = new DiscoveryCriteria(new List { @"x:dummy\foo.dll" }, 10, settingsXml); var tp = new TestableTestPlatform(_testEngine.Object, _hostManager.Object); - tp.CreateDiscoveryRequest(_mockRequestData.Object, discoveryCriteria, new TestPlatformOptions()); + tp.CreateDiscoveryRequest(_mockRequestData.Object, discoveryCriteria, new TestPlatformOptions(), It.IsAny>()); _loggerManager.Verify(lm => lm.Initialize(settingsXml)); } @@ -465,7 +380,7 @@ public void CreateDiscoveryRequestShouldInitializeLoggerManagerForDesignMode() [TestMethod] public void CreateTestRunRequestShouldInitializeLoggerManagerForNonDesignMode() { - _testEngine.Setup(te => te.GetExecutionManager(_mockRequestData.Object, _hostManager.Object, It.IsAny())).Returns(_executionManager.Object); + _testEngine.Setup(te => te.GetExecutionManager(_mockRequestData.Object, It.IsAny(), It.IsAny>())).Returns(_executionManager.Object); _testEngine.Setup(te => te.GetLoggerManager(_mockRequestData.Object)).Returns(_loggerManager.Object); string settingsXml = @@ -478,7 +393,7 @@ public void CreateTestRunRequestShouldInitializeLoggerManagerForNonDesignMode() var testRunCriteria = new TestRunCriteria(new List { "foo" }, 10, false, settingsXml); var tp = new TestableTestPlatform(_testEngine.Object, _hostManager.Object); - tp.CreateTestRunRequest(_mockRequestData.Object, testRunCriteria, new TestPlatformOptions()); + tp.CreateTestRunRequest(_mockRequestData.Object, testRunCriteria, new TestPlatformOptions(), It.IsAny>()); _loggerManager.Verify(lm => lm.Initialize(settingsXml)); } @@ -489,7 +404,7 @@ public void CreateTestRunRequestShouldInitializeLoggerManagerForNonDesignMode() [TestMethod] public void CreateDiscoveryRequestShouldInitializeLoggerManagerForNonDesignMode() { - _testEngine.Setup(te => te.GetDiscoveryManager(_mockRequestData.Object, _hostManager.Object, It.IsAny())).Returns(_discoveryManager.Object); + _testEngine.Setup(te => te.GetDiscoveryManager(_mockRequestData.Object, It.IsAny(), It.IsAny>())).Returns(_discoveryManager.Object); _testEngine.Setup(te => te.GetLoggerManager(_mockRequestData.Object)).Returns(_loggerManager.Object); string settingsXml = @@ -502,7 +417,7 @@ public void CreateDiscoveryRequestShouldInitializeLoggerManagerForNonDesignMode( var discoveryCriteria = new DiscoveryCriteria(new List { "foo" }, 10, settingsXml); var tp = new TestableTestPlatform(_testEngine.Object, _hostManager.Object); - tp.CreateDiscoveryRequest(_mockRequestData.Object, discoveryCriteria, new TestPlatformOptions()); + tp.CreateDiscoveryRequest(_mockRequestData.Object, discoveryCriteria, new TestPlatformOptions(), It.IsAny>()); _loggerManager.Verify(lm => lm.Initialize(settingsXml)); } @@ -516,7 +431,8 @@ public void StartTestSessionShouldThrowExceptionIfTestSessionCriteriaIsNull() tp.StartTestSession( new Mock().Object, null, - new Mock().Object)); + new Mock().Object, + new Dictionary())); } [TestMethod] @@ -538,7 +454,8 @@ public void StartTestSessionShouldReturnFalseIfDesignModeIsDisabled() tp.StartTestSession( new Mock().Object, testSessionCriteria, - new Mock().Object)); + new Mock().Object, + It.IsAny>())); } [TestMethod] @@ -547,7 +464,8 @@ public void StartTestSessionShouldReturnFalseIfTestSessionManagerIsNull() _testEngine.Setup( te => te.GetTestSessionManager( It.IsAny(), - It.IsAny())) + It.IsAny(), + It.IsAny>())) .Returns((IProxyTestSessionManager)null); var tp = new TestableTestPlatform(_testEngine.Object, _hostManager.Object); @@ -576,7 +494,8 @@ public void StartTestSessionShouldReturnFalseIfTestSessionManagerIsNull() tp.StartTestSession( new Mock().Object, testSessionCriteria, - mockEventsHandler.Object)); + mockEventsHandler.Object, + It.IsAny>())); mockEventsHandler.Verify( eh => eh.HandleStartTestSessionComplete(It.IsAny()), @@ -609,14 +528,16 @@ public void StartTestSessionShouldReturnTrueIfTestSessionManagerStartSessionRetu _testEngine.Setup( te => te.GetTestSessionManager( It.IsAny(), - It.IsAny())) + It.IsAny(), + It.IsAny>())) .Returns(mockTestSessionManager.Object); Assert.IsTrue( tp.StartTestSession( new Mock().Object, testSessionCriteria, - mockEventsHandler.Object)); + mockEventsHandler.Object, + It.IsAny>())); mockTestSessionManager.Verify( tsm => tsm.StartSession(mockEventsHandler.Object, It.IsAny()), @@ -649,14 +570,16 @@ public void StartTestSessionShouldReturnFalseIfTestSessionManagerStartSessionRet _testEngine.Setup( te => te.GetTestSessionManager( It.IsAny(), - It.IsAny())) + It.IsAny(), + It.IsAny>())) .Returns(mockTestSessionManager.Object); Assert.IsFalse( tp.StartTestSession( mockRequestData.Object, testSessionCriteria, - mockEventsHandler.Object)); + mockEventsHandler.Object, + It.IsAny>())); mockTestSessionManager.Verify( tsm => tsm.StartSession(mockEventsHandler.Object, mockRequestData.Object), @@ -670,18 +593,18 @@ private void InvokeCreateDiscoveryRequest(TestPlatformOptions options = null) _hostManager.Setup(hm => hm.GetTestSources(discoveryCriteria.Sources)) .Returns(discoveryCriteria.Sources); - _testEngine.Setup(te => te.GetDiscoveryManager(_mockRequestData.Object, _hostManager.Object, It.IsAny())).Returns(_discoveryManager.Object); + _testEngine.Setup(te => te.GetDiscoveryManager(_mockRequestData.Object, It.IsAny(), It.IsAny>())).Returns(_discoveryManager.Object); _testEngine.Setup(te => te.GetExtensionManager()).Returns(_extensionManager.Object); _testEngine.Setup(te => te.GetLoggerManager(_mockRequestData.Object)).Returns(_loggerManager.Object); var tp = new TestableTestPlatform(_testEngine.Object, _hostManager.Object); - tp.CreateDiscoveryRequest(_mockRequestData.Object, discoveryCriteria, options); + tp.CreateDiscoveryRequest(_mockRequestData.Object, discoveryCriteria, options, new Dictionary()); } private void InvokeCreateTestRunRequest(TestPlatformOptions options = null) { _executionManager.Setup(dm => dm.Initialize(false)).Verifiable(); - _testEngine.Setup(te => te.GetExecutionManager(_mockRequestData.Object, _hostManager.Object, It.IsAny())).Returns(_executionManager.Object); + _testEngine.Setup(te => te.GetExecutionManager(_mockRequestData.Object, It.IsAny(), It.IsAny>())).Returns(_executionManager.Object); _testEngine.Setup(te => te.GetExtensionManager()).Returns(_extensionManager.Object); _testEngine.Setup(te => te.GetLoggerManager(_mockRequestData.Object)).Returns(_loggerManager.Object); @@ -690,7 +613,7 @@ private void InvokeCreateTestRunRequest(TestPlatformOptions options = null) _hostManager.Setup(hm => hm.GetTestSources(testRunCriteria.Sources)) .Returns(testRunCriteria.Sources); - tp.CreateTestRunRequest(_mockRequestData.Object, testRunCriteria, options); + tp.CreateTestRunRequest(_mockRequestData.Object, testRunCriteria, options, new Dictionary()); } private class TestableTestPlatform : TestPlatform @@ -714,7 +637,7 @@ public TestableTestRuntimeProviderManager(ITestRuntimeProvider hostProvider) _hostProvider = hostProvider; } - public override ITestRuntimeProvider GetTestHostManagerByRunConfiguration(string runConfiguration) + public override ITestRuntimeProvider GetTestHostManagerByRunConfiguration(string runConfiguration, List _) { return _hostProvider; } diff --git a/test/Microsoft.TestPlatform.Common.UnitTests/Hosting/TestHostProviderManagerTests.cs b/test/Microsoft.TestPlatform.Common.UnitTests/Hosting/TestHostProviderManagerTests.cs index b059b59803..6245c9c28b 100644 --- a/test/Microsoft.TestPlatform.Common.UnitTests/Hosting/TestHostProviderManagerTests.cs +++ b/test/Microsoft.TestPlatform.Common.UnitTests/Hosting/TestHostProviderManagerTests.cs @@ -57,7 +57,7 @@ public void TestHostProviderManagerShouldReturnTestHostBasedOnRunConfiguration() "; var manager = TestRuntimeProviderManager.Instance; - Assert.IsNotNull(manager.GetTestHostManagerByRunConfiguration(runSettingsXml)); + Assert.IsNotNull(manager.GetTestHostManagerByRunConfiguration(runSettingsXml, null)); } [TestMethod] @@ -69,7 +69,7 @@ public void GetDefaultTestHostManagerReturnsANonNullInstance() Framework.DefaultFramework.Name, " "); - Assert.IsNotNull(TestRuntimeProviderManager.Instance.GetTestHostManagerByRunConfiguration(runSettingsXml)); + Assert.IsNotNull(TestRuntimeProviderManager.Instance.GetTestHostManagerByRunConfiguration(runSettingsXml, null)); } [TestMethod] @@ -81,8 +81,8 @@ public void GetDefaultTestHostManagerReturnsANewInstanceEverytime() Framework.DefaultFramework.Name, " "); - var instance1 = TestRuntimeProviderManager.Instance.GetTestHostManagerByRunConfiguration(runSettingsXml); - var instance2 = TestRuntimeProviderManager.Instance.GetTestHostManagerByRunConfiguration(runSettingsXml); + var instance1 = TestRuntimeProviderManager.Instance.GetTestHostManagerByRunConfiguration(runSettingsXml, null); + var instance2 = TestRuntimeProviderManager.Instance.GetTestHostManagerByRunConfiguration(runSettingsXml, null); Assert.AreNotEqual(instance1, instance2); } @@ -96,7 +96,7 @@ public void GetDefaultTestHostManagerReturnsDotnetCoreHostManagerIfFrameworkIsNe ".NETCoreApp,Version=v1.0", " "); - var testHostManager = TestRuntimeProviderManager.Instance.GetTestHostManagerByRunConfiguration(runSettingsXml); + var testHostManager = TestRuntimeProviderManager.Instance.GetTestHostManagerByRunConfiguration(runSettingsXml, null); Assert.AreEqual(typeof(TestableTestHostManager), testHostManager.GetType()); } @@ -110,7 +110,7 @@ public void GetDefaultTestHostManagerReturnsASharedManagerIfDisableAppDomainIsFa ".NETFramework,Version=v4.5.1", " "); - var testHostManager = TestRuntimeProviderManager.Instance.GetTestHostManagerByRunConfiguration(runSettingsXml); + var testHostManager = TestRuntimeProviderManager.Instance.GetTestHostManagerByRunConfiguration(runSettingsXml, null); testHostManager.Initialize(null, runSettingsXml); Assert.IsNotNull(testHostManager); @@ -126,7 +126,7 @@ public void GetDefaultTestHostManagerReturnsANonSharedManagerIfDisableAppDomainI ".NETFramework,Version=v4.5.1", "true "); - var testHostManager = TestRuntimeProviderManager.Instance.GetTestHostManagerByRunConfiguration(runSettingsXml); + var testHostManager = TestRuntimeProviderManager.Instance.GetTestHostManagerByRunConfiguration(runSettingsXml, null); testHostManager.Initialize(null, runSettingsXml); Assert.IsNotNull(testHostManager); @@ -146,7 +146,7 @@ public void TestHostProviderManagerShouldReturnNullIfTargetFrameworkIsPortable() "; var manager = TestRuntimeProviderManager.Instance; - Assert.IsNull(manager.GetTestHostManagerByRunConfiguration(runSettingsXml)); + Assert.IsNull(manager.GetTestHostManagerByRunConfiguration(runSettingsXml, null)); } #region Implementations diff --git a/test/Microsoft.TestPlatform.CrossPlatEngine.UnitTests/AttachmentsProcessing/DataCollectorAttachmentsProcessorsFactoryTests.cs b/test/Microsoft.TestPlatform.CrossPlatEngine.UnitTests/AttachmentsProcessing/DataCollectorAttachmentsProcessorsFactoryTests.cs index 16543ad407..58965097b3 100644 --- a/test/Microsoft.TestPlatform.CrossPlatEngine.UnitTests/AttachmentsProcessing/DataCollectorAttachmentsProcessorsFactoryTests.cs +++ b/test/Microsoft.TestPlatform.CrossPlatEngine.UnitTests/AttachmentsProcessing/DataCollectorAttachmentsProcessorsFactoryTests.cs @@ -119,7 +119,7 @@ public void Create_ShouldLoadOrderingByFilePath() // arrange // We cannot cleanup at the end because assembly will be copied into tmp directory and loaded string testAssetsPath = GetTestAssetsFolder(); - var dataCollectorFilePath = Directory.GetFiles(testAssetsPath, "AttachmentProcessorDataCollector.dll", SearchOption.AllDirectories).Where(x => x.Contains("bin")).Single(); + var dataCollectorFilePath = Directory.GetFiles(testAssetsPath, "AttachmentProcessorDataCollector.dll", SearchOption.AllDirectories).Where(x => x.Contains("bin")).First(); string tmpDir = Path.Combine(Path.GetTempPath(), nameof(Create_ShouldLoadOrderingByFilePath)); Directory.CreateDirectory(tmpDir); string version1 = Path.Combine(tmpDir, "1.0.0"); diff --git a/test/Microsoft.TestPlatform.CrossPlatEngine.UnitTests/Client/Parallel/ParallelOperationManagerTests.cs b/test/Microsoft.TestPlatform.CrossPlatEngine.UnitTests/Client/Parallel/ParallelOperationManagerTests.cs index 2ea64aade8..d160f3b0af 100644 --- a/test/Microsoft.TestPlatform.CrossPlatEngine.UnitTests/Client/Parallel/ParallelOperationManagerTests.cs +++ b/test/Microsoft.TestPlatform.CrossPlatEngine.UnitTests/Client/Parallel/ParallelOperationManagerTests.cs @@ -1,183 +1,183 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT license. See LICENSE file in the project root for full license information. - -using System; -using System.Collections.Generic; -using System.Linq; - -using Microsoft.VisualStudio.TestTools.UnitTesting; - -#nullable disable - -namespace TestPlatform.CrossPlatEngine.UnitTests.Client; - -[TestClass] -public class ParallelOperationManagerTests -{ - private MockParallelOperationManager _proxyParallelManager; - - [TestInitialize] - public void InitializeTests() - { - Func sampleCreator = - () => new SampleConcurrentClass(); - - _proxyParallelManager = new MockParallelOperationManager(sampleCreator, 2, true); - } - - [TestMethod] - public void AbstractProxyParallelManagerShouldCreateCorrectNumberOfConcurrentObjects() - { - var createdSampleClasses = new List(); - Func sampleCreator = - () => - { - var sample = new SampleConcurrentClass(); - createdSampleClasses.Add(sample); - return sample; - }; - - _proxyParallelManager = new MockParallelOperationManager(sampleCreator, 3, true); - - Assert.AreEqual(3, createdSampleClasses.Count, "Number of Concurrent Objects created should be 3"); - } - - [TestMethod] - public void AbstractProxyParallelManagerShouldUpdateToCorrectNumberOfConcurrentObjects() - { - var createdSampleClasses = new List(); - Func sampleCreator = - () => - { - var sample = new SampleConcurrentClass(); - createdSampleClasses.Add(sample); - return sample; - }; - - _proxyParallelManager = new MockParallelOperationManager(sampleCreator, 1, true); - - Assert.AreEqual(1, createdSampleClasses.Count, "Number of Concurrent Objects created should be 1"); - - _proxyParallelManager.UpdateParallelLevel(4); - - Assert.AreEqual(4, createdSampleClasses.Count, "Number of Concurrent Objects created should be 4"); - } - - [TestMethod] - public void DoActionOnConcurrentObjectsShouldCallAllObjects() - { - var createdSampleClasses = new List(); - Func sampleCreator = - () => - { - var sample = new SampleConcurrentClass(); - createdSampleClasses.Add(sample); - return sample; - }; - - _proxyParallelManager = new MockParallelOperationManager(sampleCreator, 4, true); - - Assert.AreEqual(4, createdSampleClasses.Count, "Number of Concurrent Objects created should be 4"); - - int count = 0; - _proxyParallelManager.DoActionOnAllConcurrentObjects( - (sample) => - { - count++; - Assert.IsTrue(createdSampleClasses.Contains(sample), "Called object must be in the created list."); - // Make sure action is not called on same object multiple times - createdSampleClasses.Remove(sample); - }); - - Assert.AreEqual(4, count, "Number of Concurrent Objects called should be 4"); - - Assert.AreEqual(0, createdSampleClasses.Count, "All concurrent objects must be called."); - } - - [TestMethod] - public void AddManagerShouldAddAManagerWithHandlerInConcurrentManagerList() - { - // At the beginning it should be equal to parallel level - Assert.AreEqual(2, _proxyParallelManager.GetConcurrentManagersCount()); - - _proxyParallelManager.AddManager(new SampleConcurrentClass(true), new SampleHandlerClass()); - - Assert.AreEqual(3, _proxyParallelManager.GetConcurrentManagersCount()); - Assert.AreEqual(1, _proxyParallelManager.GetConcurrentManagerInstances().Count(m => m.CheckValue)); - } - - [TestMethod] - public void RemoveManagerShouldRemoveAManagerFromConcurrentManagerList() - { - var manager = new SampleConcurrentClass(true); - _proxyParallelManager.AddManager(manager, new SampleHandlerClass()); - - Assert.AreEqual(3, _proxyParallelManager.GetConcurrentManagersCount()); - - _proxyParallelManager.RemoveManager(manager); - - Assert.AreEqual(2, _proxyParallelManager.GetConcurrentManagersCount()); - Assert.AreEqual(0, _proxyParallelManager.GetConcurrentManagerInstances().Count(m => m.CheckValue)); - } - - [TestMethod] - public void UpdateHandlerForManagerShouldAddNewHandlerIfNotexist() - { - var manager = new SampleConcurrentClass(true); - _proxyParallelManager.UpdateHandlerForManager(manager, new SampleHandlerClass()); - - Assert.AreEqual(3, _proxyParallelManager.GetConcurrentManagersCount()); - Assert.AreEqual(1, _proxyParallelManager.GetConcurrentManagerInstances().Count(m => m.CheckValue)); - } - - [TestMethod] - public void UpdateHandlerForManagerShouldUpdateHandlerForGivenManager() - { - var manager = new SampleConcurrentClass(true); - _proxyParallelManager.AddManager(manager, new SampleHandlerClass()); - - // For current handler the value of variable CheckValue should be false; - Assert.IsFalse(_proxyParallelManager.GetHandlerForGivenManager(manager).CheckValue); - - var newHandler = new SampleHandlerClass(true); - - // Update manager with new handler - _proxyParallelManager.UpdateHandlerForManager(manager, newHandler); - - // It should not add new manager but update the current one - Assert.AreEqual(3, _proxyParallelManager.GetConcurrentManagersCount()); - Assert.IsTrue(_proxyParallelManager.GetHandlerForGivenManager(manager).CheckValue); - } - - private class MockParallelOperationManager : ParallelOperationManager - { - public MockParallelOperationManager(Func createNewClient, int parallelLevel, bool sharedHosts) : - base(createNewClient, parallelLevel, sharedHosts) - { - } - - public void DoActionOnAllConcurrentObjects(Action action) - { - DoActionOnAllManagers(action, false); - } - } - - private class SampleConcurrentClass - { - public readonly bool CheckValue; - public SampleConcurrentClass(bool value = false) - { - CheckValue = value; - } - } - - private class SampleHandlerClass - { - public readonly bool CheckValue; - public SampleHandlerClass(bool value = false) - { - CheckValue = value; - } - - } -} +//// Copyright (c) Microsoft Corporation. All rights reserved. +//// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +//using System; +//using System.Collections.Generic; +//using System.Linq; + +//using Microsoft.VisualStudio.TestTools.UnitTesting; + +//#nullable disable + +//namespace TestPlatform.CrossPlatEngine.UnitTests.Client; + +//[TestClass] +//public class ParallelOperationManagerTests +//{ +// private MockParallelOperationManager _proxyParallelManager; + +// [TestInitialize] +// public void InitializeTests() +// { +// Func sampleCreator = +// () => new SampleConcurrentClass(); + +// _proxyParallelManager = new MockParallelOperationManager(sampleCreator, 2, true); +// } + +// [TestMethod] +// public void AbstractProxyParallelManagerShouldCreateCorrectNumberOfConcurrentObjects() +// { +// var createdSampleClasses = new List(); +// Func sampleCreator = +// () => +// { +// var sample = new SampleConcurrentClass(); +// createdSampleClasses.Add(sample); +// return sample; +// }; + +// _proxyParallelManager = new MockParallelOperationManager(sampleCreator, 3, true); + +// Assert.AreEqual(3, createdSampleClasses.Count, "Number of Concurrent Objects created should be 3"); +// } + +// [TestMethod] +// public void AbstractProxyParallelManagerShouldUpdateToCorrectNumberOfConcurrentObjects() +// { +// var createdSampleClasses = new List(); +// Func sampleCreator = +// () => +// { +// var sample = new SampleConcurrentClass(); +// createdSampleClasses.Add(sample); +// return sample; +// }; + +// _proxyParallelManager = new MockParallelOperationManager(sampleCreator, 1, true); + +// Assert.AreEqual(1, createdSampleClasses.Count, "Number of Concurrent Objects created should be 1"); + +// _proxyParallelManager.UpdateParallelLevel(4); + +// Assert.AreEqual(4, createdSampleClasses.Count, "Number of Concurrent Objects created should be 4"); +// } + +// [TestMethod] +// public void DoActionOnConcurrentObjectsShouldCallAllObjects() +// { +// var createdSampleClasses = new List(); +// Func sampleCreator = +// () => +// { +// var sample = new SampleConcurrentClass(); +// createdSampleClasses.Add(sample); +// return sample; +// }; + +// _proxyParallelManager = new MockParallelOperationManager(sampleCreator, 4, true); + +// Assert.AreEqual(4, createdSampleClasses.Count, "Number of Concurrent Objects created should be 4"); + +// int count = 0; +// _proxyParallelManager.DoActionOnAllConcurrentObjects( +// (sample) => +// { +// count++; +// Assert.IsTrue(createdSampleClasses.Contains(sample), "Called object must be in the created list."); +// // Make sure action is not called on same object multiple times +// createdSampleClasses.Remove(sample); +// }); + +// Assert.AreEqual(4, count, "Number of Concurrent Objects called should be 4"); + +// Assert.AreEqual(0, createdSampleClasses.Count, "All concurrent objects must be called."); +// } + +// [TestMethod] +// public void AddManagerShouldAddAManagerWithHandlerInConcurrentManagerList() +// { +// // At the beginning it should be equal to parallel level +// Assert.AreEqual(2, _proxyParallelManager.GetConcurrentManagersCount()); + +// _proxyParallelManager.AddManager(new SampleConcurrentClass(true), new SampleHandlerClass()); + +// Assert.AreEqual(3, _proxyParallelManager.GetConcurrentManagersCount()); +// Assert.AreEqual(1, _proxyParallelManager.GetConcurrentManagerInstances().Count(m => m.CheckValue)); +// } + +// [TestMethod] +// public void RemoveManagerShouldRemoveAManagerFromConcurrentManagerList() +// { +// var manager = new SampleConcurrentClass(true); +// _proxyParallelManager.AddManager(manager, new SampleHandlerClass()); + +// Assert.AreEqual(3, _proxyParallelManager.GetConcurrentManagersCount()); + +// _proxyParallelManager.RemoveManager(manager); + +// Assert.AreEqual(2, _proxyParallelManager.GetConcurrentManagersCount()); +// Assert.AreEqual(0, _proxyParallelManager.GetConcurrentManagerInstances().Count(m => m.CheckValue)); +// } + +// [TestMethod] +// public void UpdateHandlerForManagerShouldAddNewHandlerIfNotexist() +// { +// var manager = new SampleConcurrentClass(true); +// _proxyParallelManager.UpdateHandlerForManager(manager, new SampleHandlerClass()); + +// Assert.AreEqual(3, _proxyParallelManager.GetConcurrentManagersCount()); +// Assert.AreEqual(1, _proxyParallelManager.GetConcurrentManagerInstances().Count(m => m.CheckValue)); +// } + +// [TestMethod] +// public void UpdateHandlerForManagerShouldUpdateHandlerForGivenManager() +// { +// var manager = new SampleConcurrentClass(true); +// //_proxyParallelManager.AddManager(manager, new SampleHandlerClass()); + +// // For current handler the value of variable CheckValue should be false; +// Assert.IsFalse(_proxyParallelManager.GetHandlerForGivenManager(manager).CheckValue); + +// var newHandler = new SampleHandlerClass(true); + +// // Update manager with new handler +// _proxyParallelManager.UpdateHandlerForManager(manager, newHandler); + +// // It should not add new manager but update the current one +// Assert.AreEqual(3, _proxyParallelManager.GetConcurrentManagersCount()); +// Assert.IsTrue(_proxyParallelManager.GetHandlerForGivenManager(manager).CheckValue); +// } + +// private class MockParallelOperationManager : ParallelOperationManager +// { +// public MockParallelOperationManager(Func createNewClient, int parallelLevel, bool sharedHosts) : +// base(createNewClient, parallelLevel, sharedHosts) +// { +// } + +// public void DoActionOnAllConcurrentObjects(Action action) +// { +// DoActionOnAllManagers(action, false); +// } +// } + +// private class SampleConcurrentClass +// { +// public readonly bool CheckValue; +// public SampleConcurrentClass(bool value = false) +// { +// CheckValue = value; +// } +// } + +// private class SampleHandlerClass +// { +// public readonly bool CheckValue; +// public SampleHandlerClass(bool value = false) +// { +// CheckValue = value; +// } + +// } +//} diff --git a/test/Microsoft.TestPlatform.CrossPlatEngine.UnitTests/Client/Parallel/ParallelProxyDiscoveryManagerTests.cs b/test/Microsoft.TestPlatform.CrossPlatEngine.UnitTests/Client/Parallel/ParallelProxyDiscoveryManagerTests.cs index e2759ff9ff..e9e714d4b3 100644 --- a/test/Microsoft.TestPlatform.CrossPlatEngine.UnitTests/Client/Parallel/ParallelProxyDiscoveryManagerTests.cs +++ b/test/Microsoft.TestPlatform.CrossPlatEngine.UnitTests/Client/Parallel/ParallelProxyDiscoveryManagerTests.cs @@ -29,7 +29,7 @@ public class ParallelProxyDiscoveryManagerTests { private const int TaskTimeout = 15 * 1000; // In milliseconds. private readonly List> _createdMockManagers; - private readonly Func _proxyManagerFunc; + private readonly Func _proxyManagerFunc; private readonly Mock _mockHandler; private readonly List _sources = new() { "1.dll", "2.dll" }; private readonly DiscoveryCriteria _testDiscoveryCriteria; @@ -42,7 +42,7 @@ public ParallelProxyDiscoveryManagerTests() { _processedSources = new List(); _createdMockManagers = new List>(); - _proxyManagerFunc = () => + _proxyManagerFunc = _ => { _proxyManagerFuncCalled = true; var manager = new Mock(); @@ -78,7 +78,7 @@ public void InitializeShouldCallAllConcurrentManagersWithTrueFlagIfSkipDefaultAd [TestMethod] public void AbortShouldCallAllConcurrentManagersOnce() { - var parallelDiscoveryManager = new ParallelProxyDiscoveryManager(_mockRequestData.Object, _proxyManagerFunc, 4, false); + var parallelDiscoveryManager = new ParallelProxyDiscoveryManager(_mockRequestData.Object, _proxyManagerFunc, 4, new List()); parallelDiscoveryManager.Abort(); @@ -103,7 +103,7 @@ public void DiscoverTestsShouldProcessAllSources() [TestMethod] public void HandlePartialDiscoveryCompleteShouldReturnTrueIfDiscoveryWasAbortedWithEventHandler() { - var parallelDiscoveryManager = new ParallelProxyDiscoveryManager(_mockRequestData.Object, _proxyManagerFunc, 1, false); + var parallelDiscoveryManager = new ParallelProxyDiscoveryManager(_mockRequestData.Object, _proxyManagerFunc, 1, new List()); var proxyDiscovermanager = new ProxyDiscoveryManager(_mockRequestData.Object, new Mock().Object, new Mock().Object); parallelDiscoveryManager.Abort(_mockHandler.Object); @@ -115,7 +115,7 @@ public void HandlePartialDiscoveryCompleteShouldReturnTrueIfDiscoveryWasAbortedW [TestMethod] public void HandlePartialDiscoveryCompleteShouldReturnTrueIfDiscoveryWasAborted() { - var parallelDiscoveryManager = new ParallelProxyDiscoveryManager(_mockRequestData.Object, _proxyManagerFunc, 1, false); + var parallelDiscoveryManager = new ParallelProxyDiscoveryManager(_mockRequestData.Object, _proxyManagerFunc, 1, new List()); var proxyDiscovermanager = new ProxyDiscoveryManager(_mockRequestData.Object, new Mock().Object, new Mock().Object); parallelDiscoveryManager.Abort(); @@ -130,7 +130,7 @@ public void DiscoveryTestsShouldStopDiscoveryIfAbortionWasRequested() // Since the hosts are aborted, total aggregated tests sent across will be -1 var discoveryManagerMock = new Mock(); _createdMockManagers.Add(discoveryManagerMock); - var parallelDiscoveryManager = SetupDiscoveryManager(() => discoveryManagerMock.Object, 1, true, totalTests: -1); + var parallelDiscoveryManager = SetupDiscoveryManager(_ => discoveryManagerMock.Object, 1, true, totalTests: -1); Task.Run(() => { @@ -148,7 +148,7 @@ public void DiscoveryTestsShouldStopDiscoveryIfAbortionWithEventHandlerWasReques // Since the hosts are aborted, total aggregated tests sent across will be -1 var discoveryManagerMock = new Mock(); _createdMockManagers.Add(discoveryManagerMock); - var parallelDiscoveryManager = SetupDiscoveryManager(() => discoveryManagerMock.Object, 1, true, totalTests: -1); + var parallelDiscoveryManager = SetupDiscoveryManager(_ => discoveryManagerMock.Object, 1, true, totalTests: -1); Task.Run(() => { @@ -221,7 +221,7 @@ public void DiscoveryTestsShouldCatchExceptionAndHandleRawMessageOfTestMessage() public void HandlePartialDiscoveryCompleteShouldCreateANewProxyDiscoveryManagerIfIsAbortedIsTrue() { _proxyManagerFuncCalled = false; - var parallelDiscoveryManager = new ParallelProxyDiscoveryManager(_mockRequestData.Object, _proxyManagerFunc, 1, false); + var parallelDiscoveryManager = new ParallelProxyDiscoveryManager(_mockRequestData.Object, _proxyManagerFunc, 1, new List()); var proxyDiscovermanager = new ProxyDiscoveryManager(_mockRequestData.Object, new Mock().Object, new Mock().Object); parallelDiscoveryManager.HandlePartialDiscoveryComplete(proxyDiscovermanager, 20, new List(), isAborted: true); @@ -229,9 +229,9 @@ public void HandlePartialDiscoveryCompleteShouldCreateANewProxyDiscoveryManagerI Assert.IsTrue(_proxyManagerFuncCalled); } - private IParallelProxyDiscoveryManager SetupDiscoveryManager(Func getProxyManager, int parallelLevel, bool abortDiscovery, int totalTests = 20) + private IParallelProxyDiscoveryManager SetupDiscoveryManager(Func getProxyManager, int parallelLevel, bool abortDiscovery, int totalTests = 20) { - var parallelDiscoveryManager = new ParallelProxyDiscoveryManager(_mockRequestData.Object, getProxyManager, parallelLevel, false); + var parallelDiscoveryManager = new ParallelProxyDiscoveryManager(_mockRequestData.Object, getProxyManager, parallelLevel, new List()); SetupDiscoveryTests(_processedSources, abortDiscovery); // Setup a complete handler for parallel discovery manager @@ -289,7 +289,7 @@ private void AssertMissingAndDuplicateSources(List processedSources) private void InvokeAndVerifyInitialize(int concurrentManagersCount, bool skipDefaultAdapters = false) { - var parallelDiscoveryManager = new ParallelProxyDiscoveryManager(_mockRequestData.Object, _proxyManagerFunc, concurrentManagersCount, false); + var parallelDiscoveryManager = new ParallelProxyDiscoveryManager(_mockRequestData.Object, _proxyManagerFunc, concurrentManagersCount, new List()); // Action parallelDiscoveryManager.Initialize(skipDefaultAdapters); diff --git a/test/Microsoft.TestPlatform.CrossPlatEngine.UnitTests/Client/Parallel/ParallelProxyExecutionManagerTests.cs b/test/Microsoft.TestPlatform.CrossPlatEngine.UnitTests/Client/Parallel/ParallelProxyExecutionManagerTests.cs index ea83b52559..791091dec6 100644 --- a/test/Microsoft.TestPlatform.CrossPlatEngine.UnitTests/Client/Parallel/ParallelProxyExecutionManagerTests.cs +++ b/test/Microsoft.TestPlatform.CrossPlatEngine.UnitTests/Client/Parallel/ParallelProxyExecutionManagerTests.cs @@ -34,7 +34,7 @@ public class ParallelProxyExecutionManagerTests private static readonly int TaskTimeout = 15 * 1000; // In milliseconds private readonly List> _createdMockManagers; - private Func _proxyManagerFunc; + private Func _proxyManagerFunc; private readonly Mock _mockHandler; private Mock _mockTestHostManager; @@ -56,7 +56,7 @@ public ParallelProxyExecutionManagerTests() { _executionCompleted = new ManualResetEventSlim(false); _createdMockManagers = new List>(); - _proxyManagerFunc = () => + _proxyManagerFunc = _ => { _proxyManagerFuncCalled = true; var manager = new Mock(); @@ -99,7 +99,7 @@ public void InitializeShouldCallAllConcurrentManagersWithTrueFlagIfSkipDefaultAd [TestMethod] public void AbortShouldCallAllConcurrentManagersOnce() { - var parallelExecutionManager = new ParallelProxyExecutionManager(_mockRequestData.Object, _proxyManagerFunc, 4); + var parallelExecutionManager = new ParallelProxyExecutionManager(_mockRequestData.Object, _proxyManagerFunc, 4, new List()); parallelExecutionManager.Abort(It.IsAny()); @@ -110,7 +110,7 @@ public void AbortShouldCallAllConcurrentManagersOnce() [TestMethod] public void CancelShouldCallAllConcurrentManagersOnce() { - var parallelExecutionManager = new ParallelProxyExecutionManager(_mockRequestData.Object, _proxyManagerFunc, 4); + var parallelExecutionManager = new ParallelProxyExecutionManager(_mockRequestData.Object, _proxyManagerFunc, 4, new List()); parallelExecutionManager.Cancel(It.IsAny()); @@ -171,31 +171,31 @@ public void HandlePartialRunCompleteShouldCreateNewProxyExecutionManagerIfDataCo Assert.IsTrue(_proxyManagerFuncCalled); } - [TestMethod] - public void HandlePartialRunCompleteShouldCreateNewProxyExecutionManagerIfDataCollectionEnabledAndCreatorWithDataCollection() - { - var completeArgs = new TestRunCompleteEventArgs(null, true, true, null, null, null, TimeSpan.Zero); - _mockTestHostManager = new Mock(); - _mockRequestSender = new Mock(); - _mockDataCollectionManager = new Mock(); - var proxyDataCollectionManager = new ProxyExecutionManagerWithDataCollection(_mockRequestData.Object, _mockRequestSender.Object, _mockTestHostManager.Object, _mockDataCollectionManager.Object); - var managers = new List>(); - _proxyManagerFunc = () => - { - _proxyManagerFuncCalled = true; - var manager = new Mock(_mockRequestData.Object, _mockRequestSender.Object, _mockTestHostManager.Object, _mockDataCollectionManager.Object); - managers.Add(manager); - return manager.Object; - }; - var parallelExecutionManager = SetupExecutionManager(_proxyManagerFunc, 2, setupTestCases: true); - - _proxyManagerFuncCalled = false; - parallelExecutionManager.HandlePartialRunComplete(proxyDataCollectionManager, completeArgs, null, null, null); - Assert.IsTrue(_proxyManagerFuncCalled); - - var handler = parallelExecutionManager.GetHandlerForGivenManager(managers.Last().Object); - Assert.IsTrue(handler is ParallelDataCollectionEventsHandler); - } + //[TestMethod] + //public void HandlePartialRunCompleteShouldCreateNewProxyExecutionManagerIfDataCollectionEnabledAndCreatorWithDataCollection() + //{ + // var completeArgs = new TestRunCompleteEventArgs(null, true, true, null, null, null, TimeSpan.Zero); + // _mockTestHostManager = new Mock(); + // _mockRequestSender = new Mock(); + // _mockDataCollectionManager = new Mock(); + // var proxyDataCollectionManager = new ProxyExecutionManagerWithDataCollection(_mockRequestData.Object, _mockRequestSender.Object, _mockTestHostManager.Object, _mockDataCollectionManager.Object); + // var managers = new List>(); + // _proxyManagerFunc = _ => + // { + // _proxyManagerFuncCalled = true; + // var manager = new Mock(_mockRequestData.Object, _mockRequestSender.Object, _mockTestHostManager.Object, _mockDataCollectionManager.Object); + // managers.Add(manager); + // return manager.Object; + // }; + // var parallelExecutionManager = SetupExecutionManager(_proxyManagerFunc, 2, setupTestCases: true); + + // _proxyManagerFuncCalled = false; + // parallelExecutionManager.HandlePartialRunComplete(proxyDataCollectionManager, completeArgs, null, null, null); + // Assert.IsTrue(_proxyManagerFuncCalled); + + // var handler = parallelExecutionManager.GetHandlerForGivenManager(managers.Last().Object); + // Assert.IsTrue(handler is ParallelDataCollectionEventsHandler); + //} [TestMethod] public void HandlePartialRunCompleteShouldCreateNewProxyExecutionManagerIfIsAbortedIsTrue() @@ -227,7 +227,7 @@ public void StartTestRunWithTestsShouldNotSendCompleteUntilAllTestsAreProcessed( public void StartTestRunShouldNotProcessAllSourcesOnExecutionCancelsForAnySource() { var executionManagerMock = new Mock(); - var parallelExecutionManager = new ParallelProxyExecutionManager(_mockRequestData.Object, () => executionManagerMock.Object, 1); + var parallelExecutionManager = new ParallelProxyExecutionManager(_mockRequestData.Object, _ => executionManagerMock.Object, 1, new List()); _createdMockManagers.Add(executionManagerMock); SetupMockManagers(_processedSources, isCanceled: true, isAborted: false); SetupHandleTestRunComplete(_executionCompleted); @@ -242,7 +242,7 @@ public void StartTestRunShouldNotProcessAllSourcesOnExecutionCancelsForAnySource public void StartTestRunShouldNotProcessAllSourcesOnExecutionAborted() { var executionManagerMock = new Mock(); - var parallelExecutionManager = new ParallelProxyExecutionManager(_mockRequestData.Object, () => executionManagerMock.Object, 1); + var parallelExecutionManager = new ParallelProxyExecutionManager(_mockRequestData.Object, _ => executionManagerMock.Object, 1, new List()); _createdMockManagers.Add(executionManagerMock); SetupMockManagers(_processedSources, isCanceled: false, isAborted: false); SetupHandleTestRunComplete(_executionCompleted); @@ -258,7 +258,7 @@ public void StartTestRunShouldNotProcessAllSourcesOnExecutionAborted() public void StartTestRunShouldProcessAllSourcesOnExecutionAbortsForAnySource() { var executionManagerMock = new Mock(); - var parallelExecutionManager = new ParallelProxyExecutionManager(_mockRequestData.Object, () => executionManagerMock.Object, 1); + var parallelExecutionManager = new ParallelProxyExecutionManager(_mockRequestData.Object, _ => executionManagerMock.Object, 1, new List()); _createdMockManagers.Add(executionManagerMock); SetupMockManagers(_processedSources, isCanceled: false, isAborted: true); SetupHandleTestRunComplete(_executionCompleted); @@ -316,7 +316,7 @@ public void StartTestRunShouldCatchExceptionAndHandleRawMessageOfTestMessage() [TestMethod] public void StartTestRunShouldAggregateRunData() { - var parallelExecutionManager = new ParallelProxyExecutionManager(_mockRequestData.Object, _proxyManagerFunc, 2); + var parallelExecutionManager = new ParallelProxyExecutionManager(_mockRequestData.Object, _proxyManagerFunc, 2, new List()); var syncObject = new object(); foreach (var manager in _createdMockManagers) @@ -405,14 +405,14 @@ public void StartTestRunShouldAggregateRunData() AssertMissingAndDuplicateSources(_processedSources); } - private ParallelProxyExecutionManager SetupExecutionManager(Func proxyManagerFunc, int parallelLevel) + private ParallelProxyExecutionManager SetupExecutionManager(Func proxyManagerFunc, int parallelLevel) { return SetupExecutionManager(proxyManagerFunc, parallelLevel, false); } - private ParallelProxyExecutionManager SetupExecutionManager(Func proxyManagerFunc, int parallelLevel, bool setupTestCases) + private ParallelProxyExecutionManager SetupExecutionManager(Func proxyManagerFunc, int parallelLevel, bool setupTestCases) { - var parallelExecutionManager = new ParallelProxyExecutionManager(_mockRequestData.Object, proxyManagerFunc, parallelLevel); + var parallelExecutionManager = new ParallelProxyExecutionManager(_mockRequestData.Object, proxyManagerFunc, parallelLevel, new List()); if (setupTestCases) { @@ -552,7 +552,7 @@ private void SetupMockManagers(List processedSources, bool isCanceled = private void InvokeAndVerifyInitialize(int concurrentManagersCount, bool skipDefaultAdapters = false) { - var parallelExecutionManager = new ParallelProxyExecutionManager(_mockRequestData.Object, _proxyManagerFunc, concurrentManagersCount); + var parallelExecutionManager = new ParallelProxyExecutionManager(_mockRequestData.Object, _proxyManagerFunc, concurrentManagersCount, new List()); parallelExecutionManager.Initialize(skipDefaultAdapters); diff --git a/test/Microsoft.TestPlatform.CrossPlatEngine.UnitTests/Client/ProxyTestSessionManagerTests.cs b/test/Microsoft.TestPlatform.CrossPlatEngine.UnitTests/Client/ProxyTestSessionManagerTests.cs index fc49143ab4..57132fc3c6 100644 --- a/test/Microsoft.TestPlatform.CrossPlatEngine.UnitTests/Client/ProxyTestSessionManagerTests.cs +++ b/test/Microsoft.TestPlatform.CrossPlatEngine.UnitTests/Client/ProxyTestSessionManagerTests.cs @@ -3,12 +3,14 @@ using System; using System.Collections.Generic; +using System.Linq; using Microsoft.VisualStudio.TestPlatform.Common.Telemetry; using Microsoft.VisualStudio.TestPlatform.CrossPlatEngine; using Microsoft.VisualStudio.TestPlatform.CrossPlatEngine.Client; using Microsoft.VisualStudio.TestPlatform.ObjectModel; using Microsoft.VisualStudio.TestPlatform.ObjectModel.Client; +using Microsoft.VisualStudio.TestPlatform.ObjectModel.Host; using Microsoft.VisualStudio.TestTools.UnitTesting; using Moq; @@ -21,6 +23,7 @@ namespace Microsoft.TestPlatform.CrossPlatEngine.UnitTests.Client; public class ProxyTestSessionManagerTests { private readonly IList _fakeTestSources = new List() { @"C:\temp\FakeTestAsset.dll" }; + private Dictionary _fakeTestSourcesToRuntimeProviderMap; private readonly IList _fakeTestMultipleSources = new List() { @"C:\temp\FakeTestAsset1.dll", @"C:\temp\FakeTestAsset2.dll", @@ -31,6 +34,7 @@ public class ProxyTestSessionManagerTests @"C:\temp\FakeTestAsset7.dll", @"C:\temp\FakeTestAsset8.dll", }; + private readonly string _fakeRunSettings = "FakeRunSettings"; private readonly ProtocolConfig _protocolConfig = new() { Version = 1 }; private Mock _mockEventsHandler; @@ -69,6 +73,18 @@ public void TestInitialize() _mockMetricsCollection.Setup(mc => mc.Metrics).Returns(metrics); _mockMetricsCollection.Setup(mc => mc.Add(It.IsAny(), It.IsAny())) .Callback((string metric, object value) => metrics.Add(metric, value)); + + _fakeTestSourcesToRuntimeProviderMap = new Dictionary + { + [_fakeTestSources[0]] = new TestRuntimeProviderInfo(typeof(ITestRuntimeProvider), false, _fakeRunSettings, new List + { + new SourceDetail { + Source = _fakeTestSources[0], + Architecture = Architecture.X86, + Framework = Framework.DefaultFramework + } + }) + }; } [TestMethod] @@ -364,10 +380,26 @@ private ProxyTestSessionManager CreateProxy( StartTestSessionCriteria testSessionCriteria, ProxyOperationManager proxyOperationManager) { + var runtimeProviderInfo = new TestRuntimeProviderInfo + ( + typeof(ITestRuntimeProvider), + shared: false, + _fakeRunSettings, + testSessionCriteria.Sources.Select(s => new SourceDetail + { + Source = s, + Architecture = Architecture.X86, + Framework = Framework.DefaultFramework + }).ToList() + ); + + var runtimeProviders = new List { runtimeProviderInfo }; return new ProxyTestSessionManager( testSessionCriteria, testSessionCriteria.Sources.Count, - () => proxyOperationManager); + _ => proxyOperationManager, + runtimeProviders + ); } private void CheckStopSessionTelemetry(bool exists) diff --git a/test/Microsoft.TestPlatform.CrossPlatEngine.UnitTests/TestEngineTests.cs b/test/Microsoft.TestPlatform.CrossPlatEngine.UnitTests/TestEngineTests.cs index a47b7e9ca3..b34d05791a 100644 --- a/test/Microsoft.TestPlatform.CrossPlatEngine.UnitTests/TestEngineTests.cs +++ b/test/Microsoft.TestPlatform.CrossPlatEngine.UnitTests/TestEngineTests.cs @@ -31,7 +31,6 @@ public class TestEngineTests private ITestEngine _testEngine; private readonly Mock _mockProcessHelper; private readonly ProtocolConfig _protocolConfig = new() { Version = 1 }; - private ITestRuntimeProvider _testableTestRuntimeProvider; private readonly Mock _mockRequestData; private readonly Mock _mockMetricsCollection; @@ -39,7 +38,6 @@ public TestEngineTests() { TestPluginCacheHelper.SetupMockExtensions(new[] { typeof(TestEngineTests).GetTypeInfo().Assembly.Location }, () => { }); _mockProcessHelper = new Mock(); - _testableTestRuntimeProvider = new TestableRuntimeProvider(true); _mockRequestData = new Mock(); _mockMetricsCollection = new Mock(); _mockRequestData.Setup(rd => rd.MetricsCollection).Returns(_mockMetricsCollection.Object); @@ -55,173 +53,208 @@ public void Init() [TestMethod] public void GetDiscoveryManagerShouldReturnANonNullInstance() - { - var discoveryCriteria = new DiscoveryCriteria(new List { "1.dll" }, 100, null); - Assert.IsNotNull(_testEngine.GetDiscoveryManager(_mockRequestData.Object, _testableTestRuntimeProvider, discoveryCriteria)); - } - - [TestMethod] - public void GetDiscoveryManagerShouldReturnsNewInstanceOfProxyDiscoveryManagerIfTestHostIsShared() { string settingXml = @" - - true - - "; + + true + + "; var discoveryCriteria = new DiscoveryCriteria(new List { "1.dll" }, 100, settingXml); - var discoveryManager = _testEngine.GetDiscoveryManager(_mockRequestData.Object, _testableTestRuntimeProvider, discoveryCriteria); + var sourceToSourceDetailMap = new Dictionary + { + ["1.dll"] = new SourceDetail { Source = "1.dll", Architecture = Architecture.X86, Framework = Framework.DefaultFramework }, + }; - Assert.AreNotSame(discoveryManager, _testEngine.GetDiscoveryManager(_mockRequestData.Object, _testableTestRuntimeProvider, discoveryCriteria)); - Assert.IsInstanceOfType(_testEngine.GetDiscoveryManager(_mockRequestData.Object, _testableTestRuntimeProvider, discoveryCriteria), typeof(ProxyDiscoveryManager)); + Assert.IsNotNull(_testEngine.GetDiscoveryManager(_mockRequestData.Object, discoveryCriteria, sourceToSourceDetailMap)); } + [TestMethod] - public void GetDiscoveryManagerShouldReturnsParallelDiscoveryManagerIfTestHostIsNotShared() + public void GetDiscoveryManagerShouldReturnParallelProxyDiscoveryManagerIfNotRunningInProcess() { string settingXml = @" - - true - - "; + + true + + "; var discoveryCriteria = new DiscoveryCriteria(new List { "1.dll" }, 100, settingXml); - _testableTestRuntimeProvider = new TestableRuntimeProvider(false); + var sourceToSourceDetailMap = new Dictionary + { + ["1.dll"] = new SourceDetail { Source = "1.dll", Architecture = Architecture.X86, Framework = Framework.DefaultFramework }, + }; - Assert.IsNotNull(_testEngine.GetDiscoveryManager(_mockRequestData.Object, _testableTestRuntimeProvider, discoveryCriteria)); - Assert.IsInstanceOfType(_testEngine.GetDiscoveryManager(_mockRequestData.Object, _testableTestRuntimeProvider, discoveryCriteria), typeof(ParallelProxyDiscoveryManager)); + var discoveryManager = _testEngine.GetDiscoveryManager(_mockRequestData.Object, discoveryCriteria, sourceToSourceDetailMap); + Assert.IsNotNull(discoveryManager); + Assert.IsInstanceOfType(discoveryManager, typeof(ParallelProxyDiscoveryManager)); } [TestMethod] - public void GetDiscoveryManagerShouldNotReturnsInProcessProxyDiscoveryManagerIfCurrentProcessIsDotnet() + public void GetDiscoveryManagerShouldNotReturnInProcessProxyDiscoveryManagerIfCurrentProcessIsDotnet() { - var discoveryCriteria = new DiscoveryCriteria(new List { "1.dll" }, 100, null); + string settingXml = + @" + + true + + "; + var discoveryCriteria = new DiscoveryCriteria(new List { "1.dll" }, 100, settingXml); + var sourceToSourceDetailMap = new Dictionary + { + ["1.dll"] = new SourceDetail { Source = "1.dll", Architecture = Architecture.X86, Framework = Framework.DefaultFramework }, + }; + _mockProcessHelper.Setup(o => o.GetCurrentProcessFileName()).Returns("dotnet.exe"); - var discoveryManager = _testEngine.GetDiscoveryManager(_mockRequestData.Object, _testableTestRuntimeProvider, discoveryCriteria); + var discoveryManager = _testEngine.GetDiscoveryManager(_mockRequestData.Object, discoveryCriteria, sourceToSourceDetailMap); Assert.IsNotNull(discoveryManager); Assert.IsNotInstanceOfType(discoveryManager, typeof(InProcessProxyDiscoveryManager)); } [TestMethod] - public void GetDiscoveryManagerShouldNotReturnsInProcessProxyDiscoveryManagerIfDisableAppDomainIsSet() + public void GetDiscoveryManagerShouldNotReturnInProcessProxyDiscoveryManagerIfDisableAppDomainIsSet() { string settingXml = @" - - x86 - true - false - .NETFramework, Version=v4.5 - - "; + + x86 + true + false + .NETFramework, Version=v4.5 + + "; var discoveryCriteria = new DiscoveryCriteria(new List { "1.dll" }, 100, settingXml); + var sourceToSourceDetailMap = new Dictionary + { + ["1.dll"] = new SourceDetail { Source = "1.dll", Architecture = Architecture.X86, Framework = Framework.DefaultFramework }, + }; - var discoveryManager = _testEngine.GetDiscoveryManager(_mockRequestData.Object, _testableTestRuntimeProvider, discoveryCriteria); + var discoveryManager = _testEngine.GetDiscoveryManager(_mockRequestData.Object, discoveryCriteria, sourceToSourceDetailMap); Assert.IsNotNull(discoveryManager); Assert.IsNotInstanceOfType(discoveryManager, typeof(InProcessProxyDiscoveryManager)); } [TestMethod] - public void GetDiscoveryManagerShouldNotReturnsInProcessProxyDiscoveryManagerIfDesignModeIsTrue() + public void GetDiscoveryManagerShouldNotReturnInProcessProxyDiscoveryManagerIfDesignModeIsTrue() { string settingXml = @" - - x86 - false - true - .NETFramework, Version=v4.5 - - "; + + x86 + false + true + .NETFramework, Version=v4.5 + + "; var discoveryCriteria = new DiscoveryCriteria(new List { "1.dll" }, 100, settingXml); + var sourceToSourceDetailMap = new Dictionary + { + ["1.dll"] = new SourceDetail { Source = "1.dll", Architecture = Architecture.X86, Framework = Framework.DefaultFramework }, + }; - var discoveryManager = _testEngine.GetDiscoveryManager(_mockRequestData.Object, _testableTestRuntimeProvider, discoveryCriteria); + var discoveryManager = _testEngine.GetDiscoveryManager(_mockRequestData.Object, discoveryCriteria, sourceToSourceDetailMap); Assert.IsNotNull(discoveryManager); Assert.IsNotInstanceOfType(discoveryManager, typeof(InProcessProxyDiscoveryManager)); } [TestMethod] - public void GetDiscoveryManagerShouldNotReturnsInProcessProxyDiscoveryManagereIfTargetFrameworkIsNetcoreApp() + public void GetDiscoveryManagerShouldNotReturnInProcessProxyDiscoveryManagereIfTargetFrameworkIsNetcoreApp() { string settingXml = @" - - x86 - false - false - .NETCoreApp, Version=v1.1 - - "; + + x86 + false + false + .NETCoreApp, Version=v1.1 + + "; var discoveryCriteria = new DiscoveryCriteria(new List { "1.dll" }, 100, settingXml); + var sourceToSourceDetailMap = new Dictionary + { + ["1.dll"] = new SourceDetail { Source = "1.dll", Architecture = Architecture.X86, Framework = Framework.DefaultFramework }, + }; - var discoveryManager = _testEngine.GetDiscoveryManager(_mockRequestData.Object, _testableTestRuntimeProvider, discoveryCriteria); + var discoveryManager = _testEngine.GetDiscoveryManager(_mockRequestData.Object, discoveryCriteria, sourceToSourceDetailMap); Assert.IsNotNull(discoveryManager); Assert.IsNotInstanceOfType(discoveryManager, typeof(InProcessProxyDiscoveryManager)); } [TestMethod] - public void GetDiscoveryManagerShouldNotReturnsInProcessProxyDiscoveryManagereIfTargetFrameworkIsNetStandard() + public void GetDiscoveryManagerShouldNotReturnInProcessProxyDiscoveryManagereIfTargetFrameworkIsNetStandard() { string settingXml = @" - - x86 - false - false - .NETStandard, Version=v1.4 - - "; + + x86 + false + false + .NETStandard, Version=v1.4 + + "; var discoveryCriteria = new DiscoveryCriteria(new List { "1.dll" }, 100, settingXml); + var sourceToSourceDetailMap = new Dictionary + { + ["1.dll"] = new SourceDetail { Source = "1.dll", Architecture = Architecture.X86, Framework = Framework.DefaultFramework }, + }; - var discoveryManager = _testEngine.GetDiscoveryManager(_mockRequestData.Object, _testableTestRuntimeProvider, discoveryCriteria); + var discoveryManager = _testEngine.GetDiscoveryManager(_mockRequestData.Object, discoveryCriteria, sourceToSourceDetailMap); Assert.IsNotNull(discoveryManager); Assert.IsNotInstanceOfType(discoveryManager, typeof(InProcessProxyDiscoveryManager)); } [TestMethod] - public void GetDiscoveryManagerShouldNotReturnsInProcessProxyDiscoveryManagereIfTargetPlatformIsX64() + public void GetDiscoveryManagerShouldNotReturnInProcessProxyDiscoveryManagereIfTargetPlatformIsX64() { string settingXml = @" - - x64 - false - false - .NETStandard, Version=v1.4 - - "; + + x64 + false + false + .NETStandard, Version=v1.4 + + "; var discoveryCriteria = new DiscoveryCriteria(new List { "1.dll" }, 100, settingXml); + var sourceToSourceDetailMap = new Dictionary + { + ["1.dll"] = new SourceDetail { Source = "1.dll", Architecture = Architecture.X86, Framework = Framework.DefaultFramework }, + }; - var discoveryManager = _testEngine.GetDiscoveryManager(_mockRequestData.Object, _testableTestRuntimeProvider, discoveryCriteria); + var discoveryManager = _testEngine.GetDiscoveryManager(_mockRequestData.Object, discoveryCriteria, sourceToSourceDetailMap); Assert.IsNotNull(discoveryManager); Assert.IsNotInstanceOfType(discoveryManager, typeof(InProcessProxyDiscoveryManager)); } [TestMethod] - public void GetDiscoveryManagerShouldNotReturnsInProcessProxyDiscoveryManagereIfrunsettingsHasTestSettingsInIt() + public void GetDiscoveryManagerShouldNotReturnInProcessProxyDiscoveryManagereIfrunsettingsHasTestSettingsInIt() { string settingXml = @" - - x86 - false - false - .NETFramework, Version=v4.5 - - - C:\temp.testsettings - - "; + + x86 + false + false + .NETFramework, Version=v4.5 + + + C:\temp.testsettings + + "; var discoveryCriteria = new DiscoveryCriteria(new List { "1.dll" }, 100, settingXml); + var sourceToSourceDetailMap = new Dictionary + { + ["1.dll"] = new SourceDetail { Source = "1.dll", Architecture = Architecture.X86, Framework = Framework.DefaultFramework }, + }; - var discoveryManager = _testEngine.GetDiscoveryManager(_mockRequestData.Object, _testableTestRuntimeProvider, discoveryCriteria); + var discoveryManager = _testEngine.GetDiscoveryManager(_mockRequestData.Object, discoveryCriteria, sourceToSourceDetailMap); Assert.IsNotNull(discoveryManager); Assert.IsNotInstanceOfType(discoveryManager, typeof(InProcessProxyDiscoveryManager)); } @@ -231,17 +264,21 @@ public void GetDiscoveryManagerShouldReturnsInProcessProxyDiscoveryManager() { string settingXml = @" - - x64 - false - false - .NETFramework, Version=v4.5 - - "; + + x64 + false + false + .NETFramework, Version=v4.5 + + "; var discoveryCriteria = new DiscoveryCriteria(new List { "1.dll" }, 100, settingXml); + var sourceToSourceDetailMap = new Dictionary + { + ["1.dll"] = new SourceDetail { Source = "1.dll", Architecture = Architecture.X86, Framework = Framework.DefaultFramework }, + }; - var discoveryManager = _testEngine.GetDiscoveryManager(_mockRequestData.Object, _testableTestRuntimeProvider, discoveryCriteria); + var discoveryManager = _testEngine.GetDiscoveryManager(_mockRequestData.Object, discoveryCriteria, sourceToSourceDetailMap); Assert.IsNotNull(discoveryManager); Assert.IsInstanceOfType(discoveryManager, typeof(InProcessProxyDiscoveryManager)); } @@ -249,54 +286,49 @@ public void GetDiscoveryManagerShouldReturnsInProcessProxyDiscoveryManager() [TestMethod] public void GetExecutionManagerShouldReturnANonNullInstance() { - var testRunCriteria = new TestRunCriteria(new List { "1.dll" }, 100); + string settingsXml = @""; + var testRunCriteria = new TestRunCriteria(new List { "1.dll" }, 100, false, settingsXml); + var sourceToSourceDetailMap = new Dictionary + { + ["1.dll"] = new SourceDetail { Source = "1.dll", Architecture = Architecture.X86, Framework = Framework.DefaultFramework }, + }; - Assert.IsNotNull(_testEngine.GetExecutionManager(_mockRequestData.Object, _testableTestRuntimeProvider, testRunCriteria)); + Assert.IsNotNull(_testEngine.GetExecutionManager(_mockRequestData.Object, testRunCriteria, sourceToSourceDetailMap)); } [TestMethod] public void GetExecutionManagerShouldReturnNewInstance() { - var testRunCriteria = new TestRunCriteria(new List { "1.dll" }, 100); - var executionManager = _testEngine.GetExecutionManager(_mockRequestData.Object, _testableTestRuntimeProvider, testRunCriteria); - - Assert.AreNotSame(executionManager, _testEngine.GetExecutionManager(_mockRequestData.Object, _testableTestRuntimeProvider, testRunCriteria)); - } - - [TestMethod] - public void GetExecutionManagerShouldReturnDefaultExecutionManagerIfParallelDisabled() - { - string settingXml = @"true"; - var testRunCriteria = new TestRunCriteria(new List { "1.dll" }, 100, false, settingXml); + string settingsXml = @""; + var testRunCriteria = new TestRunCriteria(new List { "1.dll" }, 100, false, settingsXml); + var sourceToSourceDetailMap = new Dictionary + { + ["1.dll"] = new SourceDetail { Source = "1.dll", Architecture = Architecture.X86, Framework = Framework.DefaultFramework }, + }; + var executionManager = _testEngine.GetExecutionManager(_mockRequestData.Object, testRunCriteria, sourceToSourceDetailMap); - Assert.IsNotNull(_testEngine.GetExecutionManager(_mockRequestData.Object, _testableTestRuntimeProvider, testRunCriteria)); - Assert.IsInstanceOfType(_testEngine.GetExecutionManager(_mockRequestData.Object, _testableTestRuntimeProvider, testRunCriteria), typeof(ProxyExecutionManager)); + Assert.AreNotSame(executionManager, _testEngine.GetExecutionManager(_mockRequestData.Object, testRunCriteria, sourceToSourceDetailMap)); } [TestMethod] - public void GetExecutionManagerWithSingleSourceShouldReturnDefaultExecutionManagerEvenIfParallelEnabled() + public void GetExecutionManagerShouldReturnParallelExecutionManagerIfParallelEnabled() { string settingXml = @" - - 2 - true - - "; - var testRunCriteria = new TestRunCriteria(new List { "1.dll" }, 100, false, settingXml); + + 2 + + "; - Assert.IsNotNull(_testEngine.GetExecutionManager(_mockRequestData.Object, _testableTestRuntimeProvider, testRunCriteria)); - Assert.IsInstanceOfType(_testEngine.GetExecutionManager(_mockRequestData.Object, _testableTestRuntimeProvider, testRunCriteria), typeof(ProxyExecutionManager)); - } - - [TestMethod] - public void GetExecutionManagerShouldReturnParallelExecutionManagerIfParallelEnabled() - { - string settingXml = @"2"; var testRunCriteria = new TestRunCriteria(new List { "1.dll", "2.dll" }, 100, false, settingXml); + var sourceToSourceDetailMap = new Dictionary + { + ["1.dll"] = new SourceDetail { Source = "1.dll", Architecture = Architecture.X86, Framework = Framework.DefaultFramework }, + ["2.dll"] = new SourceDetail { Source = "2.dll", Architecture = Architecture.X86, Framework = Framework.DefaultFramework }, + }; - Assert.IsNotNull(_testEngine.GetExecutionManager(_mockRequestData.Object, _testableTestRuntimeProvider, testRunCriteria)); - Assert.IsInstanceOfType(_testEngine.GetExecutionManager(_mockRequestData.Object, _testableTestRuntimeProvider, testRunCriteria), typeof(ParallelProxyExecutionManager)); + Assert.IsNotNull(_testEngine.GetExecutionManager(_mockRequestData.Object, testRunCriteria, sourceToSourceDetailMap)); + Assert.IsInstanceOfType(_testEngine.GetExecutionManager(_mockRequestData.Object, testRunCriteria, sourceToSourceDetailMap), typeof(ParallelProxyExecutionManager)); } [TestMethod] @@ -304,23 +336,39 @@ public void GetExecutionManagerShouldReturnParallelExecutionManagerIfHostIsNotSh { string settingXml = @" - - true - - "; - _testableTestRuntimeProvider = new TestableRuntimeProvider(false); + + true + + "; + var testRunCriteria = new TestRunCriteria(new List { "1.dll", "2.dll" }, 100, false, settingXml); + var sourceToSourceDetailMap = new Dictionary + { + ["1.dll"] = new SourceDetail { Source = "1.dll", Architecture = Architecture.X86, Framework = Framework.DefaultFramework }, + ["2.dll"] = new SourceDetail { Source = "2.dll", Architecture = Architecture.X86, Framework = Framework.DefaultFramework }, + }; - Assert.IsNotNull(_testEngine.GetExecutionManager(_mockRequestData.Object, _testableTestRuntimeProvider, testRunCriteria)); - Assert.IsInstanceOfType(_testEngine.GetExecutionManager(_mockRequestData.Object, _testableTestRuntimeProvider, testRunCriteria), typeof(ParallelProxyExecutionManager)); + Assert.IsNotNull(_testEngine.GetExecutionManager(_mockRequestData.Object, testRunCriteria, sourceToSourceDetailMap)); + Assert.IsInstanceOfType(_testEngine.GetExecutionManager(_mockRequestData.Object, testRunCriteria, sourceToSourceDetailMap), typeof(ParallelProxyExecutionManager)); } [TestMethod] - public void GetExcecutionManagerShouldReturnExectuionManagerWithDataCollectionIfDataCollectionIsEnabled() + public void GetExecutionManagerShouldReturnExecutionManagerWithDataCollectionIfDataCollectionIsEnabled() { - var settingXml = @""; + var settingXml = + @" + + + + + + "; var testRunCriteria = new TestRunCriteria(new List { "1.dll" }, 100, false, settingXml); - var result = _testEngine.GetExecutionManager(_mockRequestData.Object, _testableTestRuntimeProvider, testRunCriteria); + var sourceToSourceDetailMap = new Dictionary + { + ["1.dll"] = new SourceDetail { Source = "1.dll", Architecture = Architecture.X86, Framework = Framework.DefaultFramework }, + }; + var result = _testEngine.GetExecutionManager(_mockRequestData.Object, testRunCriteria, sourceToSourceDetailMap); Assert.IsNotNull(result); Assert.IsInstanceOfType(result, typeof(ProxyExecutionManagerWithDataCollection)); @@ -331,17 +379,22 @@ public void GetExecutionManagerShouldNotReturnInProcessProxyexecutionManagerIfIn { string settingXml = @" - - true - false - false - .NETFramework, Version=v4.5 - - "; + + true + false + false + .NETFramework, Version=v4.5 + + "; var testRunCriteria = new TestRunCriteria(new List { "1.dll", "2.dll" }, 100, false, settingXml); + var sourceToSourceDetailMap = new Dictionary + { + ["1.dll"] = new SourceDetail { Source = "1.dll", Architecture = Architecture.X86, Framework = Framework.DefaultFramework }, + ["2.dll"] = new SourceDetail { Source = "2.dll", Architecture = Architecture.X86, Framework = Framework.DefaultFramework }, + }; - var executionManager = _testEngine.GetExecutionManager(_mockRequestData.Object, _testableTestRuntimeProvider, testRunCriteria); + var executionManager = _testEngine.GetExecutionManager(_mockRequestData.Object, testRunCriteria, sourceToSourceDetailMap); Assert.IsNotNull(executionManager); Assert.IsNotInstanceOfType(executionManager, typeof(InProcessProxyExecutionManager)); @@ -352,17 +405,22 @@ public void GetExecutionManagerShouldNotReturnInProcessProxyexecutionManagerIfPa { string settingXml = @" - - false - false - .NETFramework, Version=v4.5 - 2 - - "; + + false + false + .NETFramework, Version=v4.5 + 2 + + "; var testRunCriteria = new TestRunCriteria(new List { "1.dll", "2.dll" }, 100, false, settingXml); + var sourceToSourceDetailMap = new Dictionary + { + ["1.dll"] = new SourceDetail { Source = "1.dll", Architecture = Architecture.X86, Framework = Framework.DefaultFramework }, + ["2.dll"] = new SourceDetail { Source = "2.dll", Architecture = Architecture.X86, Framework = Framework.DefaultFramework }, + }; - var executionManager = _testEngine.GetExecutionManager(_mockRequestData.Object, _testableTestRuntimeProvider, testRunCriteria); + var executionManager = _testEngine.GetExecutionManager(_mockRequestData.Object, testRunCriteria, sourceToSourceDetailMap); Assert.IsNotNull(executionManager); Assert.IsNotInstanceOfType(executionManager, typeof(InProcessProxyExecutionManager)); @@ -373,23 +431,28 @@ public void GetExecutionManagerShouldNotReturnInProcessProxyexecutionManagerIfDa { string settingXml = @" - - false - false - .NETFramework, Version=v4.5 - 1 - - - - - - - - "; + + false + false + .NETFramework, Version=v4.5 + 1 + + + + + + + + "; var testRunCriteria = new TestRunCriteria(new List { "1.dll", "2.dll" }, 100, false, settingXml); + var sourceToSourceDetailMap = new Dictionary + { + ["1.dll"] = new SourceDetail { Source = "1.dll", Architecture = Architecture.X86, Framework = Framework.DefaultFramework }, + ["2.dll"] = new SourceDetail { Source = "2.dll", Architecture = Architecture.X86, Framework = Framework.DefaultFramework }, + }; - var executionManager = _testEngine.GetExecutionManager(_mockRequestData.Object, _testableTestRuntimeProvider, testRunCriteria); + var executionManager = _testEngine.GetExecutionManager(_mockRequestData.Object, testRunCriteria, sourceToSourceDetailMap); Assert.IsNotNull(executionManager); Assert.IsNotInstanceOfType(executionManager, typeof(InProcessProxyExecutionManager)); @@ -400,26 +463,31 @@ public void GetExecutionManagerShouldNotReturnInProcessProxyexecutionManagerIfIn { string settingXml = @" - - false - false - .NETFramework, Version=v4.5 - 1 - - - - - - 4312 - - - - - "; + + false + false + .NETFramework, Version=v4.5 + 1 + + + + + + 4312 + + + + + "; var testRunCriteria = new TestRunCriteria(new List { "1.dll", "2.dll" }, 100, false, settingXml); + var sourceToSourceDetailMap = new Dictionary + { + ["1.dll"] = new SourceDetail { Source = "1.dll", Architecture = Architecture.X86, Framework = Framework.DefaultFramework }, + ["2.dll"] = new SourceDetail { Source = "2.dll", Architecture = Architecture.X86, Framework = Framework.DefaultFramework }, + }; - var executionManager = _testEngine.GetExecutionManager(_mockRequestData.Object, _testableTestRuntimeProvider, testRunCriteria); + var executionManager = _testEngine.GetExecutionManager(_mockRequestData.Object, testRunCriteria, sourceToSourceDetailMap); Assert.IsNotNull(executionManager); Assert.IsNotInstanceOfType(executionManager, typeof(InProcessProxyExecutionManager)); @@ -430,20 +498,25 @@ public void GetExecutionManagerShouldNotReturnInProcessProxyexecutionManagerIfru { string settingXml = @" - - false - false - .NETFramework, Version=v4.5 - 1 - - - C:\temp.testsettings - - "; + + false + false + .NETFramework, Version=v4.5 + 1 + + + C:\temp.testsettings + + "; var testRunCriteria = new TestRunCriteria(new List { "1.dll", "2.dll" }, 100, false, settingXml); + var sourceToSourceDetailMap = new Dictionary + { + ["1.dll"] = new SourceDetail { Source = "1.dll", Architecture = Architecture.X86, Framework = Framework.DefaultFramework }, + ["2.dll"] = new SourceDetail { Source = "2.dll", Architecture = Architecture.X86, Framework = Framework.DefaultFramework }, + }; - var executionManager = _testEngine.GetExecutionManager(_mockRequestData.Object, _testableTestRuntimeProvider, testRunCriteria); + var executionManager = _testEngine.GetExecutionManager(_mockRequestData.Object, testRunCriteria, sourceToSourceDetailMap); Assert.IsNotNull(executionManager); Assert.IsNotInstanceOfType(executionManager, typeof(InProcessProxyExecutionManager)); @@ -455,17 +528,22 @@ public void GetExecutionManagerShouldReturnInProcessProxyexecutionManager() { string settingXml = @" - - false - false - .NETFramework, Version=v4.5 - 1 - - "; + + false + false + .NETFramework, Version=v4.5 + 1 + + "; var testRunCriteria = new TestRunCriteria(new List { "1.dll", "2.dll" }, 100, false, settingXml); + var sourceToSourceDetailMap = new Dictionary + { + ["1.dll"] = new SourceDetail { Source = "1.dll", Architecture = Architecture.X86, Framework = Framework.DefaultFramework }, + ["2.dll"] = new SourceDetail { Source = "2.dll", Architecture = Architecture.X86, Framework = Framework.DefaultFramework }, + }; - var executionManager = _testEngine.GetExecutionManager(_mockRequestData.Object, _testableTestRuntimeProvider, testRunCriteria); + var executionManager = _testEngine.GetExecutionManager(_mockRequestData.Object, testRunCriteria, sourceToSourceDetailMap); Assert.IsNotNull(executionManager); Assert.IsInstanceOfType(executionManager, typeof(InProcessProxyExecutionManager)); @@ -482,17 +560,22 @@ public void GetExtensionManagerShouldCollectMetrics() { string settingXml = @" - - false - false - .NETFramework, Version=v4.5 - 1 - - "; + + false + false + .NETFramework, Version=v4.5 + 1 + + "; var testRunCriteria = new TestRunCriteria(new List { "1.dll", "2.dll" }, 100, false, settingXml); + var sourceToSourceDetailMap = new Dictionary + { + ["1.dll"] = new SourceDetail { Source = "1.dll", Architecture = Architecture.X86, Framework = Framework.DefaultFramework }, + ["2.dll"] = new SourceDetail { Source = "2.dll", Architecture = Architecture.X86, Framework = Framework.DefaultFramework }, + }; - var executionManager = _testEngine.GetExecutionManager(_mockRequestData.Object, _testableTestRuntimeProvider, testRunCriteria); + var executionManager = _testEngine.GetExecutionManager(_mockRequestData.Object, testRunCriteria, sourceToSourceDetailMap); _mockMetricsCollection.Verify(mc => mc.Add(TelemetryDataConstants.ParallelEnabledDuringExecution, It.IsAny()), Times.Once); } @@ -500,26 +583,51 @@ public void GetExtensionManagerShouldCollectMetrics() [TestMethod] public void ProxyDataCollectionManagerShouldBeInitialzedWithCorrectTestSourcesWhenTestRunCriteriaContainsSourceList() { - var settingXml = @""; + var settingXml = + @" + + + + + + "; var testRunCriteria = new TestRunCriteria(new List { "1.dll", "2.dll" }, 100, false, settingXml); + var sourceToSourceDetailMap = new Dictionary + { + ["1.dll"] = new SourceDetail { Source = "1.dll", Architecture = Architecture.X86, Framework = Framework.DefaultFramework }, + ["2.dll"] = new SourceDetail { Source = "2.dll", Architecture = Architecture.X86, Framework = Framework.DefaultFramework }, + }; - var executionManager = _testEngine.GetExecutionManager(_mockRequestData.Object, _testableTestRuntimeProvider, testRunCriteria); + var executionManager = _testEngine.GetExecutionManager(_mockRequestData.Object, testRunCriteria, sourceToSourceDetailMap); - Assert.IsTrue((executionManager as ProxyExecutionManagerWithDataCollection).ProxyDataCollectionManager.Sources.Contains("1.dll")); + Assert.IsInstanceOfType(executionManager, typeof(ProxyExecutionManagerWithDataCollection)); + Assert.IsTrue(((ProxyExecutionManagerWithDataCollection)executionManager).ProxyDataCollectionManager.Sources.Contains("1.dll")); } [TestMethod] public void ProxyDataCollectionManagerShouldBeInitialzedWithCorrectTestSourcesWhenTestRunCriteriaContainsTestCaseList() { - var settingXml = @""; + var settingXml = + @" + + + + + + "; var testCaseList = new List { new TestCase("x.y.z", new Uri("uri://dummy"), "x.dll") }; var testRunCriteria = new TestRunCriteria(testCaseList, 100, false, settingXml); + var sourceToSourceDetailMap = new Dictionary + { + ["x.dll"] = new SourceDetail { Source = "x.dll", Architecture = Architecture.X86, Framework = Framework.DefaultFramework }, + }; - var executionManager = _testEngine.GetExecutionManager(_mockRequestData.Object, _testableTestRuntimeProvider, testRunCriteria); + var executionManager = _testEngine.GetExecutionManager(_mockRequestData.Object, testRunCriteria, sourceToSourceDetailMap); - Assert.IsTrue((executionManager as ProxyExecutionManagerWithDataCollection).ProxyDataCollectionManager.Sources.Contains("x.dll")); + Assert.IsInstanceOfType(executionManager, typeof(ProxyExecutionManagerWithDataCollection)); + Assert.IsTrue(((ProxyExecutionManagerWithDataCollection)executionManager).ProxyDataCollectionManager.Sources.Contains("x.dll")); } /// @@ -550,9 +658,15 @@ public void GetTestSessionManagerShouldReturnAValidInstance() RunSettings = settingXml }; + var sourceToSourceDetailMap = new Dictionary + { + ["1.dll"] = new SourceDetail { Source = "1.dll", Architecture = Architecture.X86, Framework = Framework.DefaultFramework }, + }; + Assert.IsNotNull(_testEngine.GetTestSessionManager( _mockRequestData.Object, - testSessionCriteria)); + testSessionCriteria, + sourceToSourceDetailMap)); } [TestMethod] @@ -565,30 +679,49 @@ public void GetTestSessionManagerShouldReturnNewInstance() RunSettings = settingXml }; + var sourceToSourceDetailMap = new Dictionary + { + ["1.dll"] = new SourceDetail { Source = "1.dll", Architecture = Architecture.X86, Framework = Framework.DefaultFramework }, + }; + var testSessionManager1 = _testEngine.GetTestSessionManager( _mockRequestData.Object, - testSessionCriteria); + testSessionCriteria, + sourceToSourceDetailMap); Assert.AreNotSame( _testEngine.GetTestSessionManager( _mockRequestData.Object, - testSessionCriteria), + testSessionCriteria, + sourceToSourceDetailMap), testSessionManager1); } [TestMethod] public void GetTestSessionManagerShouldReturnDefaultTestSessionManagerIfParallelDisabled() { - var settingXml = @"true"; + var settingXml = + @" + + true + + "; + var testSessionCriteria = new StartTestSessionCriteria() { Sources = new List { "1.dll" }, RunSettings = settingXml }; + var sourceToSourceDetailMap = new Dictionary + { + ["1.dll"] = new SourceDetail { Source = "1.dll", Architecture = Architecture.X86, Framework = Framework.DefaultFramework }, + }; + var testSessionManager = _testEngine.GetTestSessionManager( _mockRequestData.Object, - testSessionCriteria); + testSessionCriteria, + sourceToSourceDetailMap); Assert.IsNotNull(testSessionManager); Assert.IsInstanceOfType(testSessionManager, typeof(ProxyTestSessionManager)); @@ -599,20 +732,27 @@ public void GetTestSessionManagerShouldReturnDefaultTestSessionManagerEvenIfPara { string settingXml = @" - - 2 - true - - "; + + 2 + true + + "; + var testSessionCriteria = new StartTestSessionCriteria() { Sources = new List { "1.dll" }, RunSettings = settingXml }; + var sourceToSourceDetailMap = new Dictionary + { + ["1.dll"] = new SourceDetail { Source = "1.dll", Architecture = Architecture.X86, Framework = Framework.DefaultFramework }, + }; + var testSessionManager = _testEngine.GetTestSessionManager( _mockRequestData.Object, - testSessionCriteria); + testSessionCriteria, + sourceToSourceDetailMap); Assert.IsNotNull(testSessionManager); Assert.IsInstanceOfType(testSessionManager, typeof(ProxyTestSessionManager)); @@ -621,16 +761,28 @@ public void GetTestSessionManagerShouldReturnDefaultTestSessionManagerEvenIfPara [TestMethod] public void GetTestSessionManagerShouldReturnDefaultTestSessionManagerIfParallelEnabled() { - string settingXml = @"2"; + string settingXml = + @" + + 2 + + "; var testSessionCriteria = new StartTestSessionCriteria() { Sources = new List { "1.dll", "2.dll" }, RunSettings = settingXml }; + var sourceToSourceDetailMap = new Dictionary + { + ["1.dll"] = new SourceDetail { Source = "1.dll", Architecture = Architecture.X86, Framework = Framework.DefaultFramework }, + ["2.dll"] = new SourceDetail { Source = "2.dll", Architecture = Architecture.X86, Framework = Framework.DefaultFramework }, + }; + var testSessionManager = _testEngine.GetTestSessionManager( _mockRequestData.Object, - testSessionCriteria); + testSessionCriteria, + sourceToSourceDetailMap); Assert.IsNotNull(testSessionManager); Assert.IsInstanceOfType(testSessionManager, typeof(ProxyTestSessionManager)); @@ -641,19 +793,26 @@ public void GetTestSessionManagerShouldReturnDefaultTestSessionManagerIfHostIsNo { string settingXml = @" - - true - - "; + + true + + "; var testSessionCriteria = new StartTestSessionCriteria() { Sources = new List { "1.dll", "2.dll" }, RunSettings = settingXml }; + var sourceToSourceDetailMap = new Dictionary + { + ["1.dll"] = new SourceDetail { Source = "1.dll", Architecture = Architecture.X86, Framework = Framework.DefaultFramework }, + ["2.dll"] = new SourceDetail { Source = "2.dll", Architecture = Architecture.X86, Framework = Framework.DefaultFramework }, + }; + var testSessionManager = _testEngine.GetTestSessionManager( _mockRequestData.Object, - testSessionCriteria); + testSessionCriteria, + sourceToSourceDetailMap); Assert.IsNotNull(testSessionManager); Assert.IsInstanceOfType(testSessionManager, typeof(ProxyTestSessionManager)); @@ -664,22 +823,27 @@ public void GetTestSessionManagerShouldReturnDefaultTestSessionManagerIfDataColl { var settingXml = @" - - - - - - - "; + + + + + + "; var testSessionCriteria = new StartTestSessionCriteria() { Sources = new List { "1.dll" }, RunSettings = settingXml }; + var sourceToSourceDetailMap = new Dictionary + { + ["1.dll"] = new SourceDetail { Source = "1.dll", Architecture = Architecture.X86, Framework = Framework.DefaultFramework }, + }; + var testSessionManager = _testEngine.GetTestSessionManager( _mockRequestData.Object, - testSessionCriteria); + testSessionCriteria, + sourceToSourceDetailMap); Assert.IsNotNull(testSessionManager); Assert.IsInstanceOfType(testSessionManager, typeof(ProxyTestSessionManager)); @@ -690,19 +854,26 @@ public void GetTestSessionManagerShouldReturnNullWhenTargetFrameworkIsNetFramewo { var settingXml = @" - - .NETFramework, Version=v4.5 - - "; + + .NETFramework, Version=v4.5 + + "; + var testSessionCriteria = new StartTestSessionCriteria() { Sources = new List { "1.dll" }, RunSettings = settingXml }; + var sourceToSourceDetailMap = new Dictionary + { + ["1.dll"] = new SourceDetail { Source = "1.dll", Architecture = Architecture.X86, Framework = Framework.DefaultFramework }, + }; + Assert.IsNull(_testEngine.GetTestSessionManager( _mockRequestData.Object, - testSessionCriteria)); + testSessionCriteria, + sourceToSourceDetailMap)); } [TestMethod] @@ -711,13 +882,20 @@ public void GetTestSessionManagerShouldReturnNotNullIfCurrentProcessIsDotnet() var testSessionCriteria = new StartTestSessionCriteria() { Sources = new List { "1.dll" }, - RunSettings = null + RunSettings = @"" + }; + + var sourceToSourceDetailMap = new Dictionary + { + ["1.dll"] = new SourceDetail { Source = "1.dll", Architecture = Architecture.X86, Framework = Framework.DefaultFramework }, }; + _mockProcessHelper.Setup(ph => ph.GetCurrentProcessFileName()).Returns("dotnet.exe"); var testSessionManager = _testEngine.GetTestSessionManager( _mockRequestData.Object, - testSessionCriteria); + testSessionCriteria, + sourceToSourceDetailMap); Assert.IsNotNull(testSessionManager); } @@ -727,23 +905,28 @@ public void GetTestSessionManagerShouldReturnNotNullIfDisableAppDomainIsSet() { string settingXml = @" - - x86 - true - false - .NETFramework, Version=v4.5 - - "; + + x86 + true + false + .NETFramework, Version=v4.5 + + "; var testSessionCriteria = new StartTestSessionCriteria() { Sources = new List { "1.dll" }, RunSettings = settingXml }; + var sourceToSourceDetailMap = new Dictionary + { + ["1.dll"] = new SourceDetail { Source = "1.dll", Architecture = Architecture.X86, Framework = Framework.DefaultFramework }, + }; var testSessionManager = _testEngine.GetTestSessionManager( _mockRequestData.Object, - testSessionCriteria); + testSessionCriteria, + sourceToSourceDetailMap); Assert.IsNotNull(testSessionManager); } @@ -753,13 +936,13 @@ public void GetTestSessionManagerShouldReturnNotNullIfDesignModeIsTrue() { string settingXml = @" - - x86 - false - true - .NETFramework, Version=v4.5 - - "; + + x86 + false + true + .NETFramework, Version=v4.5 + + "; var testSessionCriteria = new StartTestSessionCriteria() { @@ -767,9 +950,15 @@ public void GetTestSessionManagerShouldReturnNotNullIfDesignModeIsTrue() RunSettings = settingXml }; + var sourceToSourceDetailMap = new Dictionary + { + ["1.dll"] = new SourceDetail { Source = "1.dll", Architecture = Architecture.X86, Framework = Framework.DefaultFramework }, + }; + var testSessionManager = _testEngine.GetTestSessionManager( _mockRequestData.Object, - testSessionCriteria); + testSessionCriteria, + sourceToSourceDetailMap); Assert.IsNotNull(testSessionManager); } @@ -779,13 +968,13 @@ public void GetTestSessionManagerShouldReturnNotNullIfTargetFrameworkIsNetcoreAp { string settingXml = @" - - x86 - false - false - .NETCoreApp, Version=v1.1 - - "; + + x86 + false + false + .NETCoreApp, Version=v1.1 + + "; var testSessionCriteria = new StartTestSessionCriteria() { @@ -793,9 +982,15 @@ public void GetTestSessionManagerShouldReturnNotNullIfTargetFrameworkIsNetcoreAp RunSettings = settingXml }; + var sourceToSourceDetailMap = new Dictionary + { + ["1.dll"] = new SourceDetail { Source = "1.dll", Architecture = Architecture.X86, Framework = Framework.DefaultFramework }, + }; + var testSessionManager = _testEngine.GetTestSessionManager( _mockRequestData.Object, - testSessionCriteria); + testSessionCriteria, + sourceToSourceDetailMap); Assert.IsNotNull(testSessionManager); } @@ -805,13 +1000,13 @@ public void GetTestSessionManagerShouldReturnNotNullIfTargetFrameworkIsNetStanda { string settingXml = @" - - x86 - false - false - .NETStandard, Version=v1.4 - - "; + + x86 + false + false + .NETStandard, Version=v1.4 + + "; var testSessionCriteria = new StartTestSessionCriteria() { @@ -819,9 +1014,15 @@ public void GetTestSessionManagerShouldReturnNotNullIfTargetFrameworkIsNetStanda RunSettings = settingXml }; + var sourceToSourceDetailMap = new Dictionary + { + ["1.dll"] = new SourceDetail { Source = "1.dll", Architecture = Architecture.X86, Framework = Framework.DefaultFramework }, + }; + var testSessionManager = _testEngine.GetTestSessionManager( _mockRequestData.Object, - testSessionCriteria); + testSessionCriteria, + sourceToSourceDetailMap); Assert.IsNotNull(testSessionManager); } @@ -831,13 +1032,13 @@ public void GetTestSessionManagerShouldReturnNotNullIfTargetPlatformIsX64() { string settingXml = @" - - x64 - false - false - .NETStandard, Version=v1.4 - - "; + + x64 + false + false + .NETStandard, Version=v1.4 + + "; var testSessionCriteria = new StartTestSessionCriteria() { @@ -845,9 +1046,15 @@ public void GetTestSessionManagerShouldReturnNotNullIfTargetPlatformIsX64() RunSettings = settingXml }; + var sourceToSourceDetailMap = new Dictionary + { + ["1.dll"] = new SourceDetail { Source = "1.dll", Architecture = Architecture.X86, Framework = Framework.DefaultFramework }, + }; + var testSessionManager = _testEngine.GetTestSessionManager( _mockRequestData.Object, - testSessionCriteria); + testSessionCriteria, + sourceToSourceDetailMap); Assert.IsNotNull(testSessionManager); } @@ -857,16 +1064,16 @@ public void GetTestSessionManagerShouldReturnNotNullIfRunSettingsHasTestSettings { string settingXml = @" - - x86 - false - false - .NETFramework, Version=v4.5 - - - C:\temp.testsettings - - "; + + x86 + false + false + .NETFramework, Version=v4.5 + + + C:\temp.testsettings + + "; var testSessionCriteria = new StartTestSessionCriteria() { @@ -874,9 +1081,15 @@ public void GetTestSessionManagerShouldReturnNotNullIfRunSettingsHasTestSettings RunSettings = settingXml }; + var sourceToSourceDetailMap = new Dictionary + { + ["1.dll"] = new SourceDetail { Source = "1.dll", Architecture = Architecture.X86, Framework = Framework.DefaultFramework }, + }; + var testSessionManager = _testEngine.GetTestSessionManager( _mockRequestData.Object, - testSessionCriteria); + testSessionCriteria, + sourceToSourceDetailMap); Assert.IsNotNull(testSessionManager); } diff --git a/test/Microsoft.TestPlatform.CrossPlatEngine.UnitTests/TestSession/TestSessionPoolTests.cs b/test/Microsoft.TestPlatform.CrossPlatEngine.UnitTests/TestSession/TestSessionPoolTests.cs index 8cc88cf62f..8d947f6f54 100644 --- a/test/Microsoft.TestPlatform.CrossPlatEngine.UnitTests/TestSession/TestSessionPoolTests.cs +++ b/test/Microsoft.TestPlatform.CrossPlatEngine.UnitTests/TestSession/TestSessionPoolTests.cs @@ -2,6 +2,7 @@ // Licensed under the MIT license. See LICENSE file in the project root for full license information. using System; +using System.Collections.Generic; using Microsoft.VisualStudio.TestPlatform.CrossPlatEngine; using Microsoft.VisualStudio.TestPlatform.CrossPlatEngine.Client; @@ -26,7 +27,8 @@ public void AddSessionShouldSucceedIfTestSessionInfoIsUnique() var proxyTestSessionManager = new ProxyTestSessionManager( new StartTestSessionCriteria(), 1, - () => null); + _ => null, + new List()); Assert.IsTrue(TestSessionPool.Instance.AddSession(testSessionInfo, proxyTestSessionManager)); Assert.IsFalse(TestSessionPool.Instance.AddSession(testSessionInfo, proxyTestSessionManager)); @@ -41,7 +43,8 @@ public void KillSessionShouldSucceedIfTestSessionExists() var mockProxyTestSessionManager = new Mock( new StartTestSessionCriteria(), 1, - (Func)(() => null)); + (Func)(_ => null), + new List()); var mockRequestData = new Mock(); mockProxyTestSessionManager.SetupSequence(tsm => tsm.StopSession(It.IsAny())) @@ -69,7 +72,8 @@ public void TakeProxyShouldSucceedIfMatchingCriteriaAreCorrect() var mockProxyTestSessionManager = new Mock( new StartTestSessionCriteria(), 1, - (Func)(() => null)); + (Func)(_ => null), + new List()); mockProxyTestSessionManager.SetupSequence(tsm => tsm.DequeueProxy(It.IsAny(), It.IsAny())) .Throws(new InvalidOperationException("Test Exception")) @@ -99,7 +103,8 @@ public void ReturnProxyShouldSucceedIfProxyIdIsValid() var mockProxyTestSessionManager = new Mock( new StartTestSessionCriteria(), 1, - (Func)(() => null)); + (Func)(_ => null), + new List()); mockProxyTestSessionManager.SetupSequence(tsm => tsm.EnqueueProxy(It.IsAny())) .Throws(new ArgumentException("Test Exception")) diff --git a/test/vstest.ProgrammerTests/BasicRunAndDiscovery.cs b/test/vstest.ProgrammerTests/BasicRunAndDiscovery.cs index 01367a7dd0..0a75a47e48 100644 --- a/test/vstest.ProgrammerTests/BasicRunAndDiscovery.cs +++ b/test/vstest.ProgrammerTests/BasicRunAndDiscovery.cs @@ -1,7 +1,6 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT license. See LICENSE file in the project root for full license information. -namespace vstest.ProgrammerTests; using FluentAssertions; using Microsoft.VisualStudio.TestPlatform.ObjectModel; @@ -10,10 +9,11 @@ namespace vstest.ProgrammerTests; using vstest.ProgrammerTests.Fakes; using Intent; +namespace vstest.ProgrammerTests; + // Tests are run by Intent library that is executed from our Program.Main. To debug press F5 in VS, and maybe mark just a single test with [Only]. // To just run, press Ctrl+F5 to run without debugging. It will use short timeout for abort in case something is wrong with your test. - public class BasicRunAndDiscovery { [Test(@" diff --git a/test/vstest.ProgrammerTests/Fakes/FakeDiscoveryEventsRegistrar.cs b/test/vstest.ProgrammerTests/Fakes/FakeDiscoveryEventsRegistrar.cs index 63a8f82354..fdd1cb0ac2 100644 --- a/test/vstest.ProgrammerTests/Fakes/FakeDiscoveryEventsRegistrar.cs +++ b/test/vstest.ProgrammerTests/Fakes/FakeDiscoveryEventsRegistrar.cs @@ -1,12 +1,12 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT license. See LICENSE file in the project root for full license information. -namespace vstest.ProgrammerTests.Fakes; using Microsoft.VisualStudio.TestPlatform.Common.Interfaces; using Microsoft.VisualStudio.TestPlatform.ObjectModel.Client; using Microsoft.VisualStudio.TestPlatform.ObjectModel.Logging; +namespace vstest.ProgrammerTests.Fakes; internal class FakeTestDiscoveryEventsRegistrar : ITestDiscoveryEventsRegistrar { private readonly FakeErrorAggregator _fakeErrorAggregator; diff --git a/test/vstest.ProgrammerTests/Fakes/FakeTestRuntimeProviderManager.cs b/test/vstest.ProgrammerTests/Fakes/FakeTestRuntimeProviderManager.cs index 7f2ec9c4ef..5a8267a0b4 100644 --- a/test/vstest.ProgrammerTests/Fakes/FakeTestRuntimeProviderManager.cs +++ b/test/vstest.ProgrammerTests/Fakes/FakeTestRuntimeProviderManager.cs @@ -1,8 +1,6 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT license. See LICENSE file in the project root for full license information. -using System.Collections.Concurrent; - using Microsoft.VisualStudio.TestPlatform.Common.Hosting; using Microsoft.VisualStudio.TestPlatform.ObjectModel.Host; diff --git a/test/vstest.ProgrammerTests/Fakes/FakeTestSessionEventsHandler.cs b/test/vstest.ProgrammerTests/Fakes/FakeTestSessionEventsHandler.cs index 142e057ebb..514734e59c 100644 --- a/test/vstest.ProgrammerTests/Fakes/FakeTestSessionEventsHandler.cs +++ b/test/vstest.ProgrammerTests/Fakes/FakeTestSessionEventsHandler.cs @@ -1,13 +1,12 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT license. See LICENSE file in the project root for full license information. -namespace vstest.ProgrammerTests.Fakes; - -using System.Diagnostics.Contracts; using Microsoft.VisualStudio.TestPlatform.ObjectModel.Client; using Microsoft.VisualStudio.TestPlatform.ObjectModel.Logging; +namespace vstest.ProgrammerTests.Fakes; + internal class FakeTestSessionEventsHandler : ITestSessionEventsHandler { private readonly FakeErrorAggregator _fakeErrorAggregator; diff --git a/test/vstest.ProgrammerTests/Fakes/TestMessage.cs b/test/vstest.ProgrammerTests/Fakes/TestMessage.cs index 7cf02e33ac..45308b34cc 100644 --- a/test/vstest.ProgrammerTests/Fakes/TestMessage.cs +++ b/test/vstest.ProgrammerTests/Fakes/TestMessage.cs @@ -1,10 +1,11 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT license. See LICENSE file in the project root for full license information. -namespace vstest.ProgrammerTests.Fakes; using Microsoft.VisualStudio.TestPlatform.ObjectModel.Logging; +namespace vstest.ProgrammerTests.Fakes; + internal class TestMessage { public TestMessage(TestMessageLevel level, string message) @@ -15,4 +16,4 @@ public TestMessage(TestMessageLevel level, string message) public TestMessageLevel Level { get; } public string Message { get; } -} \ No newline at end of file +} diff --git a/test/vstest.console.UnitTests/CommandLine/InferHelperTests.cs b/test/vstest.console.UnitTests/CommandLine/InferHelperTests.cs index 1e280c4e8a..b69f909f09 100644 --- a/test/vstest.console.UnitTests/CommandLine/InferHelperTests.cs +++ b/test/vstest.console.UnitTests/CommandLine/InferHelperTests.cs @@ -28,60 +28,57 @@ public class InferHelperTests private readonly Framework _frameworkCore10 = Framework.FromString(".NETCoreApp,Version=1.0"); private readonly Framework _frameworkCore11 = Framework.FromString(".NETCoreApp,Version=1.1"); private readonly IDictionary _sourceFrameworks; - private readonly IDictionary _sourceArchitectures; public InferHelperTests() { _mockAssemblyHelper = new Mock(); _inferHelper = new InferHelper(_mockAssemblyHelper.Object); - _sourceFrameworks = new Dictionary(); - _sourceArchitectures = new Dictionary(); } [TestMethod] public void AutoDetectArchitectureShouldReturnDefaultArchitectureOnNullSources() { - Assert.AreEqual(_defaultArchitecture, _inferHelper.AutoDetectArchitecture(null, _defaultArchitecture, _sourceArchitectures)); + Assert.AreEqual(_defaultArchitecture, _inferHelper.AutoDetectArchitecture(null, _defaultArchitecture, out _)); } [TestMethod] public void AutoDetectArchitectureShouldReturnDefaultArchitectureOnEmptySources() { - Assert.AreEqual(_defaultArchitecture, _inferHelper.AutoDetectArchitecture(new List(0), _defaultArchitecture, _sourceArchitectures)); + Assert.AreEqual(_defaultArchitecture, _inferHelper.AutoDetectArchitecture(new List(0), _defaultArchitecture, out _)); } [TestMethod] public void AutoDetectArchitectureShouldReturnDefaultArchitectureOnNullItemInSources() { - Assert.AreEqual(_defaultArchitecture, _inferHelper.AutoDetectArchitecture(new List() { null }, _defaultArchitecture, _sourceArchitectures)); + Assert.AreEqual(_defaultArchitecture, _inferHelper.AutoDetectArchitecture(new List() { null }, _defaultArchitecture, out _)); } [TestMethod] public void AutoDetectArchitectureShouldReturnDefaultArchitectureOnWhiteSpaceItemInSources() { - Assert.AreEqual(_defaultArchitecture, _inferHelper.AutoDetectArchitecture(new List() { " " }, _defaultArchitecture, _sourceArchitectures)); + Assert.AreEqual(_defaultArchitecture, _inferHelper.AutoDetectArchitecture(new List() { " " }, _defaultArchitecture, out _)); } [TestMethod] public void AutoDetectArchitectureShouldReturnCorrectArchForOneSource() { _mockAssemblyHelper.Setup(ah => ah.GetArchitecture(It.IsAny())).Returns(Architecture.X86); - Assert.AreEqual(Architecture.X86, _inferHelper.AutoDetectArchitecture(new List() { "1.dll" }, _defaultArchitecture, _sourceArchitectures)); + Assert.AreEqual(Architecture.X86, _inferHelper.AutoDetectArchitecture(new List() { "1.dll" }, _defaultArchitecture, out _)); _mockAssemblyHelper.Verify(ah => ah.GetArchitecture(It.IsAny())); } [TestMethod] public void AutoDetectArchitectureShouldReturnCorrectDefaultArchForNotDotNetAssembly() { - Assert.AreEqual(_defaultArchitecture, _inferHelper.AutoDetectArchitecture(new List() { "NotDotNetAssebly.appx" }, _defaultArchitecture, _sourceArchitectures)); + Assert.AreEqual(_defaultArchitecture, _inferHelper.AutoDetectArchitecture(new List() { "NotDotNetAssebly.appx" }, _defaultArchitecture, out _)); _mockAssemblyHelper.Verify(ah => ah.GetArchitecture(It.IsAny()), Times.Never); } [TestMethod] public void AutoDetectArchitectureShouldSetAnyCpuArchForNotDotNetAssembly() { - _inferHelper.AutoDetectArchitecture(new List() { "NotDotNetAssebly.appx" }, _defaultArchitecture, _sourceArchitectures); - Assert.AreEqual(Architecture.AnyCPU, _sourceArchitectures["NotDotNetAssebly.appx"]); + _inferHelper.AutoDetectArchitecture(new List() { "NotDotNetAssebly.appx" }, _defaultArchitecture, out var sourceArchitectures); + Assert.AreEqual(Architecture.AnyCPU, sourceArchitectures["NotDotNetAssebly.appx"]); } [TestMethod] @@ -89,7 +86,7 @@ public void AutoDetectArchitectureShouldReturnDefaultArchForAllAnyCpuAssemblies( { _mockAssemblyHelper.SetupSequence(ah => ah.GetArchitecture(It.IsAny())) .Returns(Architecture.AnyCPU).Returns(Architecture.AnyCPU).Returns(Architecture.AnyCPU); - Assert.AreEqual(_defaultArchitecture, _inferHelper.AutoDetectArchitecture(new List() { "AnyCPU1.dll", "AnyCPU2.exe", "AnyCPU3.dll" }, _defaultArchitecture, _sourceArchitectures)); + Assert.AreEqual(_defaultArchitecture, _inferHelper.AutoDetectArchitecture(new List() { "AnyCPU1.dll", "AnyCPU2.exe", "AnyCPU3.dll" }, _defaultArchitecture, out _)); _mockAssemblyHelper.Verify(ah => ah.GetArchitecture(It.IsAny()), Times.Exactly(3)); } @@ -98,7 +95,7 @@ public void AutoDetectArchitectureShouldReturnX86ArchIfOneX86AssemblyAndRestAnyC { _mockAssemblyHelper.SetupSequence(ah => ah.GetArchitecture(It.IsAny())) .Returns(Architecture.AnyCPU).Returns(Architecture.AnyCPU).Returns(Architecture.X86); - Assert.AreEqual(Architecture.X86, _inferHelper.AutoDetectArchitecture(new List() { "AnyCPU1.dll", "AnyCPU2.exe", "x86.dll" }, _defaultArchitecture, _sourceArchitectures)); + Assert.AreEqual(Architecture.X86, _inferHelper.AutoDetectArchitecture(new List() { "AnyCPU1.dll", "AnyCPU2.exe", "x86.dll" }, _defaultArchitecture, out _)); _mockAssemblyHelper.Verify(ah => ah.GetArchitecture(It.IsAny()), Times.Exactly(3)); } @@ -107,7 +104,7 @@ public void AutoDetectArchitectureShouldReturnARMArchIfOneARMAssemblyAndRestAnyC { _mockAssemblyHelper.SetupSequence(ah => ah.GetArchitecture(It.IsAny())) .Returns(Architecture.ARM).Returns(Architecture.ARM).Returns(Architecture.ARM); - Assert.AreEqual(Architecture.ARM, _inferHelper.AutoDetectArchitecture(new List() { "ARM1.dll", "ARM2.dll", "ARM3.dll" }, _defaultArchitecture, _sourceArchitectures)); + Assert.AreEqual(Architecture.ARM, _inferHelper.AutoDetectArchitecture(new List() { "ARM1.dll", "ARM2.dll", "ARM3.dll" }, _defaultArchitecture, out _)); _mockAssemblyHelper.Verify(ah => ah.GetArchitecture(It.IsAny()), Times.Exactly(3)); } @@ -116,7 +113,7 @@ public void AutoDetectArchitectureShouldReturnX64ArchIfOneX64AssemblyAndRestAnyC { _mockAssemblyHelper.SetupSequence(ah => ah.GetArchitecture(It.IsAny())) .Returns(Architecture.AnyCPU).Returns(Architecture.AnyCPU).Returns(Architecture.X64); - Assert.AreEqual(Architecture.X64, _inferHelper.AutoDetectArchitecture(new List() { "x64.dll", "AnyCPU2.exe", "x64.dll" }, _defaultArchitecture, _sourceArchitectures)); + Assert.AreEqual(Architecture.X64, _inferHelper.AutoDetectArchitecture(new List() { "x64.dll", "AnyCPU2.exe", "x64.dll" }, _defaultArchitecture, out _)); _mockAssemblyHelper.Verify(ah => ah.GetArchitecture(It.IsAny()), Times.Exactly(3)); } @@ -125,7 +122,7 @@ public void AutoDetectArchitectureShouldReturnDefaultArchOnConflictArches() { _mockAssemblyHelper.SetupSequence(ah => ah.GetArchitecture(It.IsAny())) .Returns(Architecture.AnyCPU).Returns(Architecture.X64).Returns(Architecture.X86); - Assert.AreEqual(_defaultArchitecture, _inferHelper.AutoDetectArchitecture(new List() { "AnyCPU1.dll", "x64.exe", "x86.dll" }, _defaultArchitecture, _sourceArchitectures)); + Assert.AreEqual(_defaultArchitecture, _inferHelper.AutoDetectArchitecture(new List() { "AnyCPU1.dll", "x64.exe", "x86.dll" }, _defaultArchitecture, out _)); _mockAssemblyHelper.Verify(ah => ah.GetArchitecture(It.IsAny()), Times.Exactly(3)); } @@ -135,11 +132,11 @@ public void AutoDetectArchitectureShouldPoulateSourceArchitectureDictionary() _mockAssemblyHelper.SetupSequence(ah => ah.GetArchitecture(It.IsAny())) .Returns(Architecture.AnyCPU).Returns(Architecture.X64).Returns(Architecture.X86); - Assert.AreEqual(_defaultArchitecture, _inferHelper.AutoDetectArchitecture(new List() { "AnyCPU1.dll", "x64.exe", "x86.dll" }, _defaultArchitecture, _sourceArchitectures)); - Assert.AreEqual(3, _sourceArchitectures.Count); - Assert.AreEqual(Architecture.AnyCPU, _sourceArchitectures["AnyCPU1.dll"]); - Assert.AreEqual(Architecture.X64, _sourceArchitectures["x64.exe"]); - Assert.AreEqual(Architecture.X86, _sourceArchitectures["x86.dll"]); + Assert.AreEqual(_defaultArchitecture, _inferHelper.AutoDetectArchitecture(new List() { "AnyCPU1.dll", "x64.exe", "x86.dll" }, _defaultArchitecture, out var sourceArchitectures)); + Assert.AreEqual(3, sourceArchitectures.Count); + Assert.AreEqual(Architecture.AnyCPU, sourceArchitectures["AnyCPU1.dll"]); + Assert.AreEqual(Architecture.X64, sourceArchitectures["x64.exe"]); + Assert.AreEqual(Architecture.X86, sourceArchitectures["x86.dll"]); _mockAssemblyHelper.Verify(ah => ah.GetArchitecture(It.IsAny()), Times.Exactly(3)); } @@ -149,32 +146,32 @@ public void AutoDetectArchitectureShouldReturnDefaultArchIfthereIsNotDotNetAssem { _mockAssemblyHelper.SetupSequence(ah => ah.GetArchitecture(It.IsAny())) .Returns(Architecture.AnyCPU); - Assert.AreEqual(_defaultArchitecture, _inferHelper.AutoDetectArchitecture(new List() { "AnyCPU1.dll", "NotDotNetAssebly.appx" }, _defaultArchitecture, _sourceArchitectures)); + Assert.AreEqual(_defaultArchitecture, _inferHelper.AutoDetectArchitecture(new List() { "AnyCPU1.dll", "NotDotNetAssebly.appx" }, _defaultArchitecture, out var sourceArchitectures)); _mockAssemblyHelper.Verify(ah => ah.GetArchitecture(It.IsAny()), Times.Exactly(1)); } [TestMethod] public void AutoDetectFrameworkShouldReturnDefaultFrameworkOnNullSources() { - Assert.AreEqual(_defaultFramework, _inferHelper.AutoDetectFramework(null, _sourceFrameworks)); + Assert.AreEqual(_defaultFramework, _inferHelper.AutoDetectFramework(null, out _)); } [TestMethod] public void AutoDetectFrameworkShouldReturnDefaultFrameworkOnEmptySources() { - Assert.AreEqual(_defaultFramework, _inferHelper.AutoDetectFramework(new List(0), _sourceFrameworks)); + Assert.AreEqual(_defaultFramework, _inferHelper.AutoDetectFramework(new List(0), out _)); } [TestMethod] public void AutoDetectFrameworkShouldReturnDefaultFrameworkOnNullItemInSources() { - Assert.AreEqual(_defaultFramework, _inferHelper.AutoDetectFramework(new List() { null }, _sourceFrameworks)); + Assert.AreEqual(_defaultFramework, _inferHelper.AutoDetectFramework(new List() { null }, out _)); } [TestMethod] public void AutoDetectFrameworkShouldReturnDefaultFrameworkOnEmptyItemInSources() { - Assert.AreEqual(_defaultFramework.Name, _inferHelper.AutoDetectFramework(new List() { string.Empty }, _sourceFrameworks).Name); + Assert.AreEqual(_defaultFramework.Name, _inferHelper.AutoDetectFramework(new List() { string.Empty }, out _).Name); } [TestMethod] @@ -232,7 +229,7 @@ public void AutoDetectFrameworkShouldReturnHighestVersionFxOnSameFxName() .Returns(new FrameworkName(_frameworkNet46.Name)) .Returns(new FrameworkName(_frameworkNet47.Name)) .Returns(new FrameworkName(_frameworkNet45.Name)); - Assert.AreEqual(_frameworkNet47.Name, _inferHelper.AutoDetectFramework(new List() { "net46.dll", "net47.exe", "net45.dll" }, _sourceFrameworks).Name); + Assert.AreEqual(_frameworkNet47.Name, _inferHelper.AutoDetectFramework(new List() { "net46.dll", "net47.exe", "net45.dll" }, out _).Name); _mockAssemblyHelper.Verify(ah => ah.GetFrameWork(It.IsAny()), Times.Exactly(3)); } @@ -244,7 +241,7 @@ public void AutoDetectFrameworkShouldPopulatetheDictionaryForAllTheSources() .Returns(new FrameworkName(_frameworkNet47.Name)) .Returns(new FrameworkName(_frameworkNet45.Name)); - Assert.AreEqual(_frameworkNet47.Name, _inferHelper.AutoDetectFramework(new List() { "net46.dll", "net47.exe", "net45.dll" }, _sourceFrameworks).Name); + Assert.AreEqual(_frameworkNet47.Name, _inferHelper.AutoDetectFramework(new List() { "net46.dll", "net47.exe", "net45.dll" }, out _).Name); Assert.AreEqual(3, _sourceFrameworks.Count); Assert.AreEqual(_frameworkNet46.Name, _sourceFrameworks["net46.dll"].Name); @@ -260,7 +257,7 @@ public void AutoDetectFrameworkShouldReturnHighestVersionFxOnEvenManyLowerVersio .Returns(new FrameworkName(_frameworkCore10.Name)) .Returns(new FrameworkName(_frameworkCore11.Name)) .Returns(new FrameworkName(_frameworkCore10.Name)); - Assert.AreEqual(_frameworkCore11.Name, _inferHelper.AutoDetectFramework(new List() { "netcore10_1.dll", "netcore11.dll", "netcore10_2.dll" }, _sourceFrameworks).Name); + Assert.AreEqual(_frameworkCore11.Name, _inferHelper.AutoDetectFramework(new List() { "netcore10_1.dll", "netcore11.dll", "netcore10_2.dll" }, out _).Name); _mockAssemblyHelper.Verify(ah => ah.GetFrameWork(It.IsAny()), Times.Exactly(3)); } @@ -268,7 +265,7 @@ private void SetupAndValidateForSingleAssembly(string assemblyName, Framework fx { _mockAssemblyHelper.Setup(sh => sh.GetFrameWork(assemblyName)) .Returns(new FrameworkName(fx.Name)); - Assert.AreEqual(fx.Name, _inferHelper.AutoDetectFramework(new List() { assemblyName }, _sourceFrameworks).Name); + Assert.AreEqual(fx.Name, _inferHelper.AutoDetectFramework(new List() { assemblyName }, out _).Name); if (verify) { _mockAssemblyHelper.Verify(ah => ah.GetFrameWork(assemblyName)); diff --git a/test/vstest.console.UnitTests/Processors/ListFullyQualifiedTestsArgumentProcessorTests.cs b/test/vstest.console.UnitTests/Processors/ListFullyQualifiedTestsArgumentProcessorTests.cs index bbc4b82e0a..1166d65665 100644 --- a/test/vstest.console.UnitTests/Processors/ListFullyQualifiedTestsArgumentProcessorTests.cs +++ b/test/vstest.console.UnitTests/Processors/ListFullyQualifiedTestsArgumentProcessorTests.cs @@ -155,7 +155,7 @@ public void ExecutorExecuteShouldThrowTestPlatformException() var mockDiscoveryRequest = new Mock(); mockDiscoveryRequest.Setup(dr => dr.DiscoverAsync()).Throws(new TestPlatformException("DummyTestPlatformException")); - mockTestPlatform.Setup(tp => tp.CreateDiscoveryRequest(It.IsAny(), It.IsAny(), It.IsAny())).Returns(mockDiscoveryRequest.Object); + mockTestPlatform.Setup(tp => tp.CreateDiscoveryRequest(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny>())).Returns(mockDiscoveryRequest.Object); ResetAndAddSourceToCommandLineOptions(true); @@ -173,7 +173,7 @@ public void ExecutorExecuteShouldThrowSettingsException() var mockDiscoveryRequest = new Mock(); mockDiscoveryRequest.Setup(dr => dr.DiscoverAsync()).Throws(new SettingsException("DummySettingsException")); - mockTestPlatform.Setup(tp => tp.CreateDiscoveryRequest(It.IsAny(), It.IsAny(), It.IsAny())).Returns(mockDiscoveryRequest.Object); + mockTestPlatform.Setup(tp => tp.CreateDiscoveryRequest(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny>())).Returns(mockDiscoveryRequest.Object); ResetAndAddSourceToCommandLineOptions(true); var testRequestManager = new TestRequestManager(CommandLineOptions.Instance, mockTestPlatform.Object, TestRunResultAggregator.Instance, _mockTestPlatformEventSource.Object, _inferHelper, _mockMetricsPublisherTask, _mockProcessHelper.Object, _mockAttachmentsProcessingManager.Object); @@ -190,7 +190,7 @@ public void ExecutorExecuteShouldThrowInvalidOperationException() var mockDiscoveryRequest = new Mock(); mockDiscoveryRequest.Setup(dr => dr.DiscoverAsync()).Throws(new InvalidOperationException("DummyInvalidOperationException")); - mockTestPlatform.Setup(tp => tp.CreateDiscoveryRequest(It.IsAny(), It.IsAny(), It.IsAny())).Returns(mockDiscoveryRequest.Object); + mockTestPlatform.Setup(tp => tp.CreateDiscoveryRequest(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny>())).Returns(mockDiscoveryRequest.Object); ResetAndAddSourceToCommandLineOptions(true); @@ -209,7 +209,7 @@ public void ExecutorExecuteShouldThrowOtherExceptions() var mockDiscoveryRequest = new Mock(); mockDiscoveryRequest.Setup(dr => dr.DiscoverAsync()).Throws(new Exception("DummyException")); - mockTestPlatform.Setup(tp => tp.CreateDiscoveryRequest(It.IsAny(), It.IsAny(), It.IsAny())).Returns(mockDiscoveryRequest.Object); + mockTestPlatform.Setup(tp => tp.CreateDiscoveryRequest(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny>())).Returns(mockDiscoveryRequest.Object); ResetAndAddSourceToCommandLineOptions(true); @@ -299,7 +299,7 @@ private void RunListFullyQualifiedTestArgumentProcessorWithTraits(Mock dr.DiscoverAsync()).Raises(dr => dr.OnDiscoveredTests += null, new DiscoveredTestsEventArgs(list)); - mockTestPlatform.Setup(tp => tp.CreateDiscoveryRequest(It.IsAny(), It.IsAny(), It.IsAny())).Returns(mockDiscoveryRequest.Object); + mockTestPlatform.Setup(tp => tp.CreateDiscoveryRequest(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny>())).Returns(mockDiscoveryRequest.Object); ResetAndAddSourceToCommandLineOptions(legitPath); var cmdOptions = CommandLineOptions.Instance; @@ -319,7 +319,7 @@ private void RunListFullyQualifiedTestArgumentProcessorExecuteWithMockSetup(Mock new TestCase("Test2", new Uri("http://FooTestUri2"), "Source2") }; mockDiscoveryRequest.Setup(dr => dr.DiscoverAsync()).Raises(dr => dr.OnDiscoveredTests += null, new DiscoveredTestsEventArgs(list)); - mockTestPlatform.Setup(tp => tp.CreateDiscoveryRequest(It.IsAny(), It.IsAny(), It.IsAny())).Returns(mockDiscoveryRequest.Object); + mockTestPlatform.Setup(tp => tp.CreateDiscoveryRequest(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny>())).Returns(mockDiscoveryRequest.Object); ResetAndAddSourceToCommandLineOptions(legitPath); diff --git a/test/vstest.console.UnitTests/Processors/ListTestsArgumentProcessorTests.cs b/test/vstest.console.UnitTests/Processors/ListTestsArgumentProcessorTests.cs index 495799340e..5de8108732 100644 --- a/test/vstest.console.UnitTests/Processors/ListTestsArgumentProcessorTests.cs +++ b/test/vstest.console.UnitTests/Processors/ListTestsArgumentProcessorTests.cs @@ -158,7 +158,7 @@ public void ExecutorExecuteShouldThrowTestPlatformException() var mockDiscoveryRequest = new Mock(); mockDiscoveryRequest.Setup(dr => dr.DiscoverAsync()).Throws(new TestPlatformException("DummyTestPlatformException")); - mockTestPlatform.Setup(tp => tp.CreateDiscoveryRequest(It.IsAny(), It.IsAny(), It.IsAny())).Returns(mockDiscoveryRequest.Object); + mockTestPlatform.Setup(tp => tp.CreateDiscoveryRequest(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny>())).Returns(mockDiscoveryRequest.Object); ResetAndAddSourceToCommandLineOptions(); @@ -175,7 +175,7 @@ public void ExecutorExecuteShouldThrowSettingsException() var mockDiscoveryRequest = new Mock(); mockDiscoveryRequest.Setup(dr => dr.DiscoverAsync()).Throws(new SettingsException("DummySettingsException")); - mockTestPlatform.Setup(tp => tp.CreateDiscoveryRequest(It.IsAny(), It.IsAny(), It.IsAny())).Returns(mockDiscoveryRequest.Object); + mockTestPlatform.Setup(tp => tp.CreateDiscoveryRequest(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny>())).Returns(mockDiscoveryRequest.Object); ResetAndAddSourceToCommandLineOptions(); @@ -192,7 +192,7 @@ public void ExecutorExecuteShouldThrowInvalidOperationException() var mockDiscoveryRequest = new Mock(); mockDiscoveryRequest.Setup(dr => dr.DiscoverAsync()).Throws(new InvalidOperationException("DummyInvalidOperationException")); - mockTestPlatform.Setup(tp => tp.CreateDiscoveryRequest(It.IsAny(), It.IsAny(), It.IsAny())).Returns(mockDiscoveryRequest.Object); + mockTestPlatform.Setup(tp => tp.CreateDiscoveryRequest(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny>())).Returns(mockDiscoveryRequest.Object); ResetAndAddSourceToCommandLineOptions(); @@ -209,7 +209,7 @@ public void ExecutorExecuteShouldThrowOtherExceptions() var mockDiscoveryRequest = new Mock(); mockDiscoveryRequest.Setup(dr => dr.DiscoverAsync()).Throws(new Exception("DummyException")); - mockTestPlatform.Setup(tp => tp.CreateDiscoveryRequest(It.IsAny(), It.IsAny(), It.IsAny())).Returns(mockDiscoveryRequest.Object); + mockTestPlatform.Setup(tp => tp.CreateDiscoveryRequest(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny>())).Returns(mockDiscoveryRequest.Object); ResetAndAddSourceToCommandLineOptions(); @@ -267,7 +267,7 @@ private void RunListTestArgumentProcessorExecuteWithMockSetup(Mock dr.DiscoverAsync()).Raises(dr => dr.OnDiscoveredTests += null, new DiscoveredTestsEventArgs(list)); - mockTestPlatform.Setup(tp => tp.CreateDiscoveryRequest(It.IsAny(), It.IsAny(), It.IsAny())).Returns(mockDiscoveryRequest.Object); + mockTestPlatform.Setup(tp => tp.CreateDiscoveryRequest(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny>())).Returns(mockDiscoveryRequest.Object); ResetAndAddSourceToCommandLineOptions(); diff --git a/test/vstest.console.UnitTests/Processors/RunSpecificTestsArgumentProcessorTests.cs b/test/vstest.console.UnitTests/Processors/RunSpecificTestsArgumentProcessorTests.cs index 19cf550f85..3656cf14df 100644 --- a/test/vstest.console.UnitTests/Processors/RunSpecificTestsArgumentProcessorTests.cs +++ b/test/vstest.console.UnitTests/Processors/RunSpecificTestsArgumentProcessorTests.cs @@ -196,8 +196,8 @@ public void ExecutorExecuteForValidSourceWithTestCaseFilterShouldRunTests() }; mockDiscoveryRequest.Setup(dr => dr.DiscoverAsync()).Raises(dr => dr.OnDiscoveredTests += null, new DiscoveredTestsEventArgs(list)); - mockTestPlatform.Setup(tp => tp.CreateTestRunRequest(It.IsAny(), It.IsAny(), It.IsAny())).Returns(mockTestRunRequest.Object); - mockTestPlatform.Setup(tp => tp.CreateDiscoveryRequest(It.IsAny(), It.IsAny(), It.IsAny())).Returns(mockDiscoveryRequest.Object); + mockTestPlatform.Setup(tp => tp.CreateTestRunRequest(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny>())).Returns(mockTestRunRequest.Object); + mockTestPlatform.Setup(tp => tp.CreateDiscoveryRequest(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny>())).Returns(mockDiscoveryRequest.Object); var testRequestManager = new TestRequestManager(CommandLineOptions.Instance, mockTestPlatform.Object, TestRunResultAggregator.Instance, _mockTestPlatformEventSource.Object, _inferHelper, _mockMetricsPublisherTask, _mockProcessHelper.Object, _mockAttachmentsProcessingManager.Object); var executor = GetExecutor(testRequestManager); @@ -207,7 +207,7 @@ public void ExecutorExecuteForValidSourceWithTestCaseFilterShouldRunTests() ArgumentProcessorResult argumentProcessorResult = executor.Execute(); _mockOutput.Verify(o => o.WriteLine(It.IsAny(), OutputLevel.Warning), Times.Never); - mockTestPlatform.Verify(o => o.CreateDiscoveryRequest(It.IsAny(), It.Is(c => c.TestCaseFilter == "Filter"), It.IsAny()), Times.Once()); + mockTestPlatform.Verify(o => o.CreateDiscoveryRequest(It.IsAny(), It.Is(c => c.TestCaseFilter == "Filter"), It.IsAny(), It.IsAny>()), Times.Once()); Assert.AreEqual(ArgumentProcessorResult.Success, argumentProcessorResult); } @@ -220,8 +220,8 @@ public void ExecutorExecuteShouldThrowTestPlatformExceptionThrownDuringDiscovery var mockDiscoveryRequest = new Mock(); mockDiscoveryRequest.Setup(dr => dr.DiscoverAsync()).Throws(new TestPlatformException("DummyTestPlatformException")); - mockTestPlatform.Setup(tp => tp.CreateTestRunRequest(It.IsAny(), It.IsAny(), It.IsAny())).Returns(mockTestRunRequest.Object); - mockTestPlatform.Setup(tp => tp.CreateDiscoveryRequest(It.IsAny(), It.IsAny(), It.IsAny())).Returns(mockDiscoveryRequest.Object); + mockTestPlatform.Setup(tp => tp.CreateTestRunRequest(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny>())).Returns(mockTestRunRequest.Object); + mockTestPlatform.Setup(tp => tp.CreateDiscoveryRequest(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny>())).Returns(mockDiscoveryRequest.Object); ResetAndAddSourceToCommandLineOptions(); var testRequestManager = new TestRequestManager(CommandLineOptions.Instance, mockTestPlatform.Object, TestRunResultAggregator.Instance, _mockTestPlatformEventSource.Object, _inferHelper, _mockMetricsPublisherTask, _mockProcessHelper.Object, _mockAttachmentsProcessingManager.Object); @@ -238,8 +238,8 @@ public void ExecutorExecuteShouldThrowInvalidOperationExceptionThrownDuringDisco var mockDiscoveryRequest = new Mock(); mockDiscoveryRequest.Setup(dr => dr.DiscoverAsync()).Throws(new InvalidOperationException("DummyInvalidOperationException")); - mockTestPlatform.Setup(tp => tp.CreateTestRunRequest(It.IsAny(), It.IsAny(), It.IsAny())).Returns(mockTestRunRequest.Object); - mockTestPlatform.Setup(tp => tp.CreateDiscoveryRequest(It.IsAny(), It.IsAny(), It.IsAny())).Returns(mockDiscoveryRequest.Object); + mockTestPlatform.Setup(tp => tp.CreateTestRunRequest(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny>())).Returns(mockTestRunRequest.Object); + mockTestPlatform.Setup(tp => tp.CreateDiscoveryRequest(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny>())).Returns(mockDiscoveryRequest.Object); ResetAndAddSourceToCommandLineOptions(); var testRequestManager = new TestRequestManager(CommandLineOptions.Instance, mockTestPlatform.Object, TestRunResultAggregator.Instance, _mockTestPlatformEventSource.Object, _inferHelper, _mockMetricsPublisherTask, _mockProcessHelper.Object, _mockAttachmentsProcessingManager.Object); @@ -256,8 +256,8 @@ public void ExecutorExecuteShouldThrowSettingsExceptionThrownDuringDiscovery() var mockDiscoveryRequest = new Mock(); mockDiscoveryRequest.Setup(dr => dr.DiscoverAsync()).Throws(new SettingsException("DummySettingsException")); - mockTestPlatform.Setup(tp => tp.CreateTestRunRequest(It.IsAny(), It.IsAny(), It.IsAny())).Returns(mockTestRunRequest.Object); - mockTestPlatform.Setup(tp => tp.CreateDiscoveryRequest(It.IsAny(), It.IsAny(), It.IsAny())).Returns(mockDiscoveryRequest.Object); + mockTestPlatform.Setup(tp => tp.CreateTestRunRequest(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny>())).Returns(mockTestRunRequest.Object); + mockTestPlatform.Setup(tp => tp.CreateDiscoveryRequest(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny>())).Returns(mockDiscoveryRequest.Object); ResetAndAddSourceToCommandLineOptions(); var testRequestManager = new TestRequestManager(CommandLineOptions.Instance, mockTestPlatform.Object, TestRunResultAggregator.Instance, _mockTestPlatformEventSource.Object, _inferHelper, _mockMetricsPublisherTask, _mockProcessHelper.Object, _mockAttachmentsProcessingManager.Object); @@ -281,8 +281,8 @@ public void ExecutorExecuteShouldThrowTestPlatformExceptionThrownDuringExecution mockDiscoveryRequest.Setup(dr => dr.DiscoverAsync()).Raises(dr => dr.OnDiscoveredTests += null, new DiscoveredTestsEventArgs(list)); mockTestRunRequest.Setup(dr => dr.ExecuteAsync()).Throws(new TestPlatformException("DummyTestPlatformException")); - mockTestPlatform.Setup(tp => tp.CreateTestRunRequest(It.IsAny(), It.IsAny(), It.IsAny())).Returns(mockTestRunRequest.Object); - mockTestPlatform.Setup(tp => tp.CreateDiscoveryRequest(It.IsAny(), It.IsAny(), It.IsAny())).Returns(mockDiscoveryRequest.Object); + mockTestPlatform.Setup(tp => tp.CreateTestRunRequest(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny>())).Returns(mockTestRunRequest.Object); + mockTestPlatform.Setup(tp => tp.CreateDiscoveryRequest(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny>())).Returns(mockDiscoveryRequest.Object); ResetAndAddSourceToCommandLineOptions(); var testRequestManager = new TestRequestManager(CommandLineOptions.Instance, mockTestPlatform.Object, TestRunResultAggregator.Instance, _mockTestPlatformEventSource.Object, _inferHelper, _mockMetricsPublisherTask, _mockProcessHelper.Object, _mockAttachmentsProcessingManager.Object); @@ -308,8 +308,8 @@ public void ExecutorExecuteShouldThrowSettingsExceptionThrownDuringExecution() mockDiscoveryRequest.Setup(dr => dr.DiscoverAsync()).Raises(dr => dr.OnDiscoveredTests += null, new DiscoveredTestsEventArgs(list)); mockTestRunRequest.Setup(dr => dr.ExecuteAsync()).Throws(new SettingsException("DummySettingsException")); - mockTestPlatform.Setup(tp => tp.CreateTestRunRequest(It.IsAny(), It.IsAny(), It.IsAny())).Returns(mockTestRunRequest.Object); - mockTestPlatform.Setup(tp => tp.CreateDiscoveryRequest(It.IsAny(), It.IsAny(), It.IsAny())).Returns(mockDiscoveryRequest.Object); + mockTestPlatform.Setup(tp => tp.CreateTestRunRequest(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny>())).Returns(mockTestRunRequest.Object); + mockTestPlatform.Setup(tp => tp.CreateDiscoveryRequest(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny>())).Returns(mockDiscoveryRequest.Object); ResetAndAddSourceToCommandLineOptions(); var testRequestManager = new TestRequestManager(CommandLineOptions.Instance, mockTestPlatform.Object, TestRunResultAggregator.Instance, _mockTestPlatformEventSource.Object, _inferHelper, _mockMetricsPublisherTask, _mockProcessHelper.Object, _mockAttachmentsProcessingManager.Object); @@ -335,8 +335,8 @@ public void ExecutorExecuteShouldThrowInvalidOperationExceptionThrownDuringExecu mockDiscoveryRequest.Setup(dr => dr.DiscoverAsync()).Raises(dr => dr.OnDiscoveredTests += null, new DiscoveredTestsEventArgs(list)); mockTestRunRequest.Setup(dr => dr.ExecuteAsync()).Throws(new InvalidOperationException("DummySettingsException")); - mockTestPlatform.Setup(tp => tp.CreateTestRunRequest(It.IsAny(), It.IsAny(), It.IsAny())).Returns(mockTestRunRequest.Object); - mockTestPlatform.Setup(tp => tp.CreateDiscoveryRequest(It.IsAny(), It.IsAny(), It.IsAny())).Returns(mockDiscoveryRequest.Object); + mockTestPlatform.Setup(tp => tp.CreateTestRunRequest(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny>())).Returns(mockTestRunRequest.Object); + mockTestPlatform.Setup(tp => tp.CreateDiscoveryRequest(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny>())).Returns(mockDiscoveryRequest.Object); ResetAndAddSourceToCommandLineOptions(); @@ -360,7 +360,7 @@ public void ExecutorExecuteShouldForValidSourcesAndNoTestsDiscoveredShouldLogWar CommandLineOptions.Instance.TestAdapterPath = new[] { @"C:\Foo" }; mockDiscoveryRequest.Setup(dr => dr.DiscoverAsync()).Raises(dr => dr.OnDiscoveredTests += null, new DiscoveredTestsEventArgs(new List())); - mockTestPlatform.Setup(tp => tp.CreateDiscoveryRequest(It.IsAny(), It.IsAny(), It.IsAny())).Returns(mockDiscoveryRequest.Object); + mockTestPlatform.Setup(tp => tp.CreateDiscoveryRequest(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny>())).Returns(mockDiscoveryRequest.Object); var testRequestManager = new TestRequestManager(CommandLineOptions.Instance, mockTestPlatform.Object, TestRunResultAggregator.Instance, _mockTestPlatformEventSource.Object, _inferHelper, _mockMetricsPublisherTask, _mockProcessHelper.Object, _mockAttachmentsProcessingManager.Object); var executor = GetExecutor(testRequestManager); @@ -381,7 +381,7 @@ public void ExecutorExecuteShouldForValidSourcesAndNoTestsDiscoveredShouldLogApp ResetAndAddSourceToCommandLineOptions(); mockDiscoveryRequest.Setup(dr => dr.DiscoverAsync()).Raises(dr => dr.OnDiscoveredTests += null, new DiscoveredTestsEventArgs(new List())); - mockTestPlatform.Setup(tp => tp.CreateDiscoveryRequest(It.IsAny(), It.IsAny(), It.IsAny())).Returns(mockDiscoveryRequest.Object); + mockTestPlatform.Setup(tp => tp.CreateDiscoveryRequest(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny>())).Returns(mockDiscoveryRequest.Object); var testRequestManager = new TestRequestManager(CommandLineOptions.Instance, mockTestPlatform.Object, TestRunResultAggregator.Instance, _mockTestPlatformEventSource.Object, _inferHelper, _mockMetricsPublisherTask, _mockProcessHelper.Object, _mockAttachmentsProcessingManager.Object); var executor = GetExecutor(testRequestManager); @@ -408,8 +408,8 @@ public void ExecutorExecuteShouldForValidSourcesAndValidSelectedTestsRunsTestsAn }; mockDiscoveryRequest.Setup(dr => dr.DiscoverAsync()).Raises(dr => dr.OnDiscoveredTests += null, new DiscoveredTestsEventArgs(list)); - mockTestPlatform.Setup(tp => tp.CreateTestRunRequest(It.IsAny(), It.IsAny(), It.IsAny())).Returns(mockTestRunRequest.Object); - mockTestPlatform.Setup(tp => tp.CreateDiscoveryRequest(It.IsAny(), It.IsAny(), It.IsAny())).Returns(mockDiscoveryRequest.Object); + mockTestPlatform.Setup(tp => tp.CreateTestRunRequest(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny>())).Returns(mockTestRunRequest.Object); + mockTestPlatform.Setup(tp => tp.CreateDiscoveryRequest(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny>())).Returns(mockDiscoveryRequest.Object); var testRequestManager = new TestRequestManager(CommandLineOptions.Instance, mockTestPlatform.Object, TestRunResultAggregator.Instance, _mockTestPlatformEventSource.Object, _inferHelper, _mockMetricsPublisherTask, _mockProcessHelper.Object, _mockAttachmentsProcessingManager.Object); var executor = GetExecutor(testRequestManager); @@ -436,8 +436,8 @@ public void ExecutorShouldRunTestsWhenTestsAreCommaSeparated() }; mockDiscoveryRequest.Setup(dr => dr.DiscoverAsync()).Raises(dr => dr.OnDiscoveredTests += null, new DiscoveredTestsEventArgs(list)); - mockTestPlatform.Setup(tp => tp.CreateTestRunRequest(It.IsAny(), It.IsAny(), It.IsAny())).Returns(mockTestRunRequest.Object); - mockTestPlatform.Setup(tp => tp.CreateDiscoveryRequest(It.IsAny(), It.IsAny(), It.IsAny())).Returns(mockDiscoveryRequest.Object); + mockTestPlatform.Setup(tp => tp.CreateTestRunRequest(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny>())).Returns(mockTestRunRequest.Object); + mockTestPlatform.Setup(tp => tp.CreateDiscoveryRequest(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny>())).Returns(mockDiscoveryRequest.Object); var testRequestManager = new TestRequestManager(CommandLineOptions.Instance, mockTestPlatform.Object, TestRunResultAggregator.Instance, _mockTestPlatformEventSource.Object, _inferHelper, _mockMetricsPublisherTask, _mockProcessHelper.Object, _mockAttachmentsProcessingManager.Object); var executor = GetExecutor(testRequestManager); @@ -465,8 +465,8 @@ public void ExecutorShouldRunTestsWhenTestsAreFiltered() }; mockDiscoveryRequest.Setup(dr => dr.DiscoverAsync()).Raises(dr => dr.OnDiscoveredTests += null, new DiscoveredTestsEventArgs(list)); - mockTestPlatform.Setup(tp => tp.CreateTestRunRequest(It.IsAny(), It.IsAny(), It.IsAny())).Returns(mockTestRunRequest.Object); - mockTestPlatform.Setup(tp => tp.CreateDiscoveryRequest(It.IsAny(), It.IsAny(), It.IsAny())).Returns(mockDiscoveryRequest.Object); + mockTestPlatform.Setup(tp => tp.CreateTestRunRequest(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny>())).Returns(mockTestRunRequest.Object); + mockTestPlatform.Setup(tp => tp.CreateDiscoveryRequest(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny>())).Returns(mockDiscoveryRequest.Object); var testRequestManager = new TestRequestManager(CommandLineOptions.Instance, mockTestPlatform.Object, TestRunResultAggregator.Instance, _mockTestPlatformEventSource.Object, _inferHelper, _mockMetricsPublisherTask, _mockProcessHelper.Object, _mockAttachmentsProcessingManager.Object); var executor = GetExecutor(testRequestManager); @@ -493,8 +493,8 @@ public void ExecutorShouldWarnWhenTestsAreNotAvailable() }; mockDiscoveryRequest.Setup(dr => dr.DiscoverAsync()).Raises(dr => dr.OnDiscoveredTests += null, new DiscoveredTestsEventArgs(list)); - mockTestPlatform.Setup(tp => tp.CreateTestRunRequest(It.IsAny(), It.IsAny(), It.IsAny())).Returns(mockTestRunRequest.Object); - mockTestPlatform.Setup(tp => tp.CreateDiscoveryRequest(It.IsAny(), It.IsAny(), It.IsAny())).Returns(mockDiscoveryRequest.Object); + mockTestPlatform.Setup(tp => tp.CreateTestRunRequest(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny>())).Returns(mockTestRunRequest.Object); + mockTestPlatform.Setup(tp => tp.CreateDiscoveryRequest(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny>())).Returns(mockDiscoveryRequest.Object); var testRequestManager = new TestRequestManager(CommandLineOptions.Instance, mockTestPlatform.Object, TestRunResultAggregator.Instance, _mockTestPlatformEventSource.Object, _inferHelper, _mockMetricsPublisherTask, _mockProcessHelper.Object, _mockAttachmentsProcessingManager.Object); var executor = GetExecutor(testRequestManager); @@ -522,8 +522,8 @@ public void ExecutorShouldRunTestsWhenTestsAreCommaSeparatedWithEscape() }; mockDiscoveryRequest.Setup(dr => dr.DiscoverAsync()).Raises(dr => dr.OnDiscoveredTests += null, new DiscoveredTestsEventArgs(list)); - mockTestPlatform.Setup(tp => tp.CreateTestRunRequest(It.IsAny(), It.IsAny(), It.IsAny())).Returns(mockTestRunRequest.Object); - mockTestPlatform.Setup(tp => tp.CreateDiscoveryRequest(It.IsAny(), It.IsAny(), It.IsAny())).Returns(mockDiscoveryRequest.Object); + mockTestPlatform.Setup(tp => tp.CreateTestRunRequest(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny>())).Returns(mockTestRunRequest.Object); + mockTestPlatform.Setup(tp => tp.CreateDiscoveryRequest(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny>())).Returns(mockDiscoveryRequest.Object); var testRequestManager = new TestRequestManager(CommandLineOptions.Instance, mockTestPlatform.Object, TestRunResultAggregator.Instance, _mockTestPlatformEventSource.Object, _inferHelper, _mockMetricsPublisherTask, _mockProcessHelper.Object, _mockAttachmentsProcessingManager.Object); var executor = GetExecutor(testRequestManager); @@ -553,8 +553,8 @@ public void ExecutorShouldDisplayWarningIfNoTestsAreExecuted() mockTestRunRequest.Setup(tr => tr.ExecuteAsync()).Returns(1).Raises(tr => tr.OnRunCompletion += null, new TestRunCompleteEventArgs(mockTestRunStats.Object, false, false, null, null, null, new TimeSpan())); - mockTestPlatform.Setup(tp => tp.CreateTestRunRequest(It.IsAny(), It.IsAny(), It.IsAny())).Returns(mockTestRunRequest.Object); - mockTestPlatform.Setup(tp => tp.CreateDiscoveryRequest(It.IsAny(), It.IsAny(), It.IsAny())).Returns(mockDiscoveryRequest.Object); + mockTestPlatform.Setup(tp => tp.CreateTestRunRequest(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny>())).Returns(mockTestRunRequest.Object); + mockTestPlatform.Setup(tp => tp.CreateDiscoveryRequest(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny>())).Returns(mockDiscoveryRequest.Object); ResetAndAddSourceToCommandLineOptions(); var testRequestManager = new TestRequestManager(CommandLineOptions.Instance, mockTestPlatform.Object, TestRunResultAggregator.Instance, _mockTestPlatformEventSource.Object, _inferHelper, _mockMetricsPublisherTask, _mockProcessHelper.Object, _mockAttachmentsProcessingManager.Object); @@ -584,8 +584,8 @@ public void ExecutorShouldNotDisplayWarningIfTestsAreExecuted() mockTestRunRequest.Setup(tr => tr.ExecuteAsync()).Returns(1).Raises(tr => tr.OnRunCompletion += null, new TestRunCompleteEventArgs(testRunStats, false, false, null, null, null, new TimeSpan())); - mockTestPlatform.Setup(tp => tp.CreateTestRunRequest(It.IsAny(), It.IsAny(), It.IsAny())).Returns(mockTestRunRequest.Object); - mockTestPlatform.Setup(tp => tp.CreateDiscoveryRequest(It.IsAny(), It.IsAny(), It.IsAny())).Returns(mockDiscoveryRequest.Object); + mockTestPlatform.Setup(tp => tp.CreateTestRunRequest(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny>())).Returns(mockTestRunRequest.Object); + mockTestPlatform.Setup(tp => tp.CreateDiscoveryRequest(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny>())).Returns(mockDiscoveryRequest.Object); ResetAndAddSourceToCommandLineOptions(); var testRequestManager = new TestRequestManager(CommandLineOptions.Instance, mockTestPlatform.Object, TestRunResultAggregator.Instance, _mockTestPlatformEventSource.Object, _inferHelper, _mockMetricsPublisherTask, _mockProcessHelper.Object, _mockAttachmentsProcessingManager.Object); diff --git a/test/vstest.console.UnitTests/Processors/RunTestsArgumentProcessorTests.cs b/test/vstest.console.UnitTests/Processors/RunTestsArgumentProcessorTests.cs index 073a67c1b3..894de8397b 100644 --- a/test/vstest.console.UnitTests/Processors/RunTestsArgumentProcessorTests.cs +++ b/test/vstest.console.UnitTests/Processors/RunTestsArgumentProcessorTests.cs @@ -156,7 +156,7 @@ public void ExecutorExecuteShouldThrowTestPlatformException() var mockTestRunRequest = new Mock(); mockTestRunRequest.Setup(tr => tr.ExecuteAsync()).Throws(new TestPlatformException("DummyTestPlatformException")); - mockTestPlatform.Setup(tp => tp.CreateTestRunRequest(It.IsAny(), It.IsAny(), It.IsAny())).Returns(mockTestRunRequest.Object); + mockTestPlatform.Setup(tp => tp.CreateTestRunRequest(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny>())).Returns(mockTestRunRequest.Object); ResetAndAddSourceToCommandLineOptions(); var testRequestManager = new TestRequestManager(CommandLineOptions.Instance, mockTestPlatform.Object, TestRunResultAggregator.Instance, _mockTestPlatformEventSource.Object, _inferHelper, _mockMetricsPublisherTask, _mockProcessHelper.Object, _mockAttachmentsProcessingManager.Object); @@ -172,7 +172,7 @@ public void ExecutorExecuteShouldThrowSettingsException() var mockTestRunRequest = new Mock(); mockTestRunRequest.Setup(tr => tr.ExecuteAsync()).Throws(new SettingsException("DummySettingsException")); - mockTestPlatform.Setup(tp => tp.CreateTestRunRequest(It.IsAny(), It.IsAny(), It.IsAny())).Returns(mockTestRunRequest.Object); + mockTestPlatform.Setup(tp => tp.CreateTestRunRequest(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny>())).Returns(mockTestRunRequest.Object); ResetAndAddSourceToCommandLineOptions(); var testRequestManager = new TestRequestManager(CommandLineOptions.Instance, mockTestPlatform.Object, TestRunResultAggregator.Instance, _mockTestPlatformEventSource.Object, _inferHelper, _mockMetricsPublisherTask, _mockProcessHelper.Object, _mockAttachmentsProcessingManager.Object); @@ -188,7 +188,7 @@ public void ExecutorExecuteShouldThrowInvalidOperationException() var mockTestRunRequest = new Mock(); mockTestRunRequest.Setup(tr => tr.ExecuteAsync()).Throws(new InvalidOperationException("DummyInvalidOperationException")); - mockTestPlatform.Setup(tp => tp.CreateTestRunRequest(It.IsAny(), It.IsAny(), It.IsAny())).Returns(mockTestRunRequest.Object); + mockTestPlatform.Setup(tp => tp.CreateTestRunRequest(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny>())).Returns(mockTestRunRequest.Object); ResetAndAddSourceToCommandLineOptions(); var testRequestManager = new TestRequestManager(CommandLineOptions.Instance, mockTestPlatform.Object, TestRunResultAggregator.Instance, _mockTestPlatformEventSource.Object, _inferHelper, _mockMetricsPublisherTask, _mockProcessHelper.Object, _mockAttachmentsProcessingManager.Object); @@ -204,7 +204,7 @@ public void ExecutorExecuteShouldThrowOtherExceptions() var mockTestRunRequest = new Mock(); mockTestRunRequest.Setup(tr => tr.ExecuteAsync()).Throws(new Exception("DummyException")); - mockTestPlatform.Setup(tp => tp.CreateTestRunRequest(It.IsAny(), It.IsAny(), It.IsAny())).Returns(mockTestRunRequest.Object); + mockTestPlatform.Setup(tp => tp.CreateTestRunRequest(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny>())).Returns(mockTestRunRequest.Object); ResetAndAddSourceToCommandLineOptions(); var testRequestManager = new TestRequestManager(CommandLineOptions.Instance, mockTestPlatform.Object, TestRunResultAggregator.Instance, _mockTestPlatformEventSource.Object, _inferHelper, _mockMetricsPublisherTask, _mockProcessHelper.Object, _mockAttachmentsProcessingManager.Object); @@ -260,7 +260,7 @@ private ArgumentProcessorResult RunRunArgumentProcessorExecuteWithMockSetup(ITes var args = new TestRunCompleteEventArgs(mockTestRunStats.Object, false, false, null, null, null, new TimeSpan()); - mockTestPlatform.Setup(tp => tp.CreateTestRunRequest(It.IsAny(), It.IsAny(), It.IsAny())).Returns(testRunRequest); + mockTestPlatform.Setup(tp => tp.CreateTestRunRequest(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny>())).Returns(testRunRequest); ResetAndAddSourceToCommandLineOptions(); diff --git a/test/vstest.console.UnitTests/TestPlatformHelpers/TestRequestManagerTests.cs b/test/vstest.console.UnitTests/TestPlatformHelpers/TestRequestManagerTests.cs index 2fb0998e08..fbf21103a4 100644 --- a/test/vstest.console.UnitTests/TestPlatformHelpers/TestRequestManagerTests.cs +++ b/test/vstest.console.UnitTests/TestPlatformHelpers/TestRequestManagerTests.cs @@ -89,9 +89,9 @@ public TestRequestManagerTests() _mockMetricsPublisherTask, _mockProcessHelper.Object, _mockAttachmentsProcessingManager.Object); - _mockTestPlatform.Setup(tp => tp.CreateDiscoveryRequest(It.IsAny(), It.IsAny(), It.IsAny())) + _mockTestPlatform.Setup(tp => tp.CreateDiscoveryRequest(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny>())) .Returns(_mockDiscoveryRequest.Object); - _mockTestPlatform.Setup(tp => tp.CreateTestRunRequest(It.IsAny(), It.IsAny(), It.IsAny())) + _mockTestPlatform.Setup(tp => tp.CreateTestRunRequest(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny>())) .Returns(_mockRunRequest.Object); _mockAssemblyMetadataProvider.Setup(a => a.GetArchitecture(It.IsAny())) .Returns(Architecture.X86); @@ -165,7 +165,7 @@ public void DiscoverTestsShouldReadTheBatchSizeFromSettingsAndSetItForDiscoveryC DiscoveryCriteria actualDiscoveryCriteria = null; var mockDiscoveryRequest = new Mock(); - _mockTestPlatform.Setup(mt => mt.CreateDiscoveryRequest(It.IsAny(), It.IsAny(), It.IsAny())).Callback( + _mockTestPlatform.Setup(mt => mt.CreateDiscoveryRequest(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny>())).Callback( (IRequestData requestData, DiscoveryCriteria discoveryCriteria, TestPlatformOptions options) => actualDiscoveryCriteria = discoveryCriteria).Returns(mockDiscoveryRequest.Object); _testRequestManager.DiscoverTests(payload, new Mock().Object, _protocolConfig); @@ -184,7 +184,7 @@ public void DiscoverTestsShouldCallTestPlatformAndSucceed() var createDiscoveryRequestCalled = 0; DiscoveryCriteria actualDiscoveryCriteria = null; var mockDiscoveryRequest = new Mock(); - _mockTestPlatform.Setup(mt => mt.CreateDiscoveryRequest(It.IsAny(), It.IsAny(), It.IsAny())).Callback( + _mockTestPlatform.Setup(mt => mt.CreateDiscoveryRequest(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny>())).Callback( (IRequestData requestData, DiscoveryCriteria discoveryCriteria, TestPlatformOptions options) => { createDiscoveryRequestCalled++; @@ -238,7 +238,7 @@ public void DiscoverTestsShouldPassSameProtocolConfigInRequestData() IRequestData actualRequestData = null; var mockDiscoveryRequest = new Mock(); - _mockTestPlatform.Setup(mt => mt.CreateDiscoveryRequest(It.IsAny(), It.IsAny(), It.IsAny())).Callback( + _mockTestPlatform.Setup(mt => mt.CreateDiscoveryRequest(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny>())).Callback( (IRequestData requestData, DiscoveryCriteria discoveryCriteria, TestPlatformOptions options) => actualRequestData = requestData).Returns(mockDiscoveryRequest.Object); var mockDiscoveryRegistrar = new Mock(); @@ -289,7 +289,7 @@ public void DiscoverTestsShouldCollectMetrics() IRequestData actualRequestData = null; var mockDiscoveryRequest = new Mock(); - _mockTestPlatform.Setup(mt => mt.CreateDiscoveryRequest(It.IsAny(), It.IsAny(), It.IsAny())).Callback( + _mockTestPlatform.Setup(mt => mt.CreateDiscoveryRequest(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny>())).Callback( (IRequestData requestData, DiscoveryCriteria discoveryCriteria, TestPlatformOptions options) => actualRequestData = requestData).Returns(mockDiscoveryRequest.Object); _testRequestManager = new TestRequestManager( @@ -338,7 +338,7 @@ public void DiscoverTestsShouldCollectTargetDeviceLocalMachineIfTargetDeviceStri IRequestData actualRequestData = null; var mockDiscoveryRequest = new Mock(); - _mockTestPlatform.Setup(mt => mt.CreateDiscoveryRequest(It.IsAny(), It.IsAny(), It.IsAny())).Callback( + _mockTestPlatform.Setup(mt => mt.CreateDiscoveryRequest(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny>())).Callback( (IRequestData requestData, DiscoveryCriteria discoveryCriteria, TestPlatformOptions options) => actualRequestData = requestData).Returns(mockDiscoveryRequest.Object); _testRequestManager = new TestRequestManager( @@ -381,7 +381,7 @@ public void DiscoverTestsShouldCollectTargetDeviceIfTargetDeviceIsDevice() IRequestData actualRequestData = null; var mockDiscoveryRequest = new Mock(); - _mockTestPlatform.Setup(mt => mt.CreateDiscoveryRequest(It.IsAny(), It.IsAny(), It.IsAny())).Callback( + _mockTestPlatform.Setup(mt => mt.CreateDiscoveryRequest(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny>())).Callback( (IRequestData requestData, DiscoveryCriteria discoveryCriteria, TestPlatformOptions options) => actualRequestData = requestData).Returns(mockDiscoveryRequest.Object); _testRequestManager = new TestRequestManager( @@ -424,7 +424,7 @@ public void DiscoverTestsShouldCollectTargetDeviceIfTargetDeviceIsEmulator() IRequestData actualRequestData = null; var mockDiscoveryRequest = new Mock(); - _mockTestPlatform.Setup(mt => mt.CreateDiscoveryRequest(It.IsAny(), It.IsAny(), It.IsAny())).Callback( + _mockTestPlatform.Setup(mt => mt.CreateDiscoveryRequest(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny>())).Callback( (IRequestData requestData, DiscoveryCriteria discoveryCriteria, TestPlatformOptions options) => actualRequestData = requestData).Returns(mockDiscoveryRequest.Object); _testRequestManager = new TestRequestManager( @@ -467,7 +467,7 @@ public void DiscoverTestsShouldCollectCommands() IRequestData actualRequestData = null; var mockDiscoveryRequest = new Mock(); - _mockTestPlatform.Setup(mt => mt.CreateDiscoveryRequest(It.IsAny(), It.IsAny(), It.IsAny())).Callback( + _mockTestPlatform.Setup(mt => mt.CreateDiscoveryRequest(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny>())).Callback( (IRequestData requestData, DiscoveryCriteria discoveryCriteria, TestPlatformOptions options) => actualRequestData = requestData).Returns(mockDiscoveryRequest.Object); _testRequestManager = new TestRequestManager( @@ -522,7 +522,7 @@ public void DiscoverTestsShouldCollectTestSettings() IRequestData actualRequestData = null; var mockDiscoveryRequest = new Mock(); - _mockTestPlatform.Setup(mt => mt.CreateDiscoveryRequest(It.IsAny(), It.IsAny(), It.IsAny())).Callback( + _mockTestPlatform.Setup(mt => mt.CreateDiscoveryRequest(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny>())).Callback( (IRequestData requestData, DiscoveryCriteria discoveryCriteria, TestPlatformOptions options) => actualRequestData = requestData).Returns(mockDiscoveryRequest.Object); _testRequestManager = new TestRequestManager( @@ -569,7 +569,7 @@ public void DiscoverTestsShouldCollectVsmdiFile() IRequestData actualRequestData = null; var mockDiscoveryRequest = new Mock(); - _mockTestPlatform.Setup(mt => mt.CreateDiscoveryRequest(It.IsAny(), It.IsAny(), It.IsAny())).Callback( + _mockTestPlatform.Setup(mt => mt.CreateDiscoveryRequest(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny>())).Callback( (IRequestData requestData, DiscoveryCriteria discoveryCriteria, TestPlatformOptions options) => actualRequestData = requestData).Returns(mockDiscoveryRequest.Object); _testRequestManager = new TestRequestManager( @@ -616,7 +616,7 @@ public void DiscoverTestsShouldCollectTestRunConfigFile() IRequestData actualRequestData = null; var mockDiscoveryRequest = new Mock(); - _mockTestPlatform.Setup(mt => mt.CreateDiscoveryRequest(It.IsAny(), It.IsAny(), It.IsAny())).Callback( + _mockTestPlatform.Setup(mt => mt.CreateDiscoveryRequest(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny>())).Callback( (IRequestData requestData, DiscoveryCriteria discoveryCriteria, TestPlatformOptions options) => actualRequestData = requestData).Returns(mockDiscoveryRequest.Object); _testRequestManager = new TestRequestManager( @@ -662,7 +662,7 @@ public void DiscoverTestsShouldUpdateFrameworkAndPlatformIfNotSpecifiedInDesignM .Returns(new FrameworkName(Constants.DotNetFramework46)); DiscoveryCriteria actualDiscoveryCriteria = null; var mockDiscoveryRequest = new Mock(); - _mockTestPlatform.Setup(mt => mt.CreateDiscoveryRequest(It.IsAny(), It.IsAny(), It.IsAny())).Callback( + _mockTestPlatform.Setup(mt => mt.CreateDiscoveryRequest(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny>())).Callback( (IRequestData requestData, DiscoveryCriteria discoveryCriteria, TestPlatformOptions options) => actualDiscoveryCriteria = discoveryCriteria).Returns(mockDiscoveryRequest.Object); _testRequestManager.DiscoverTests(payload, new Mock().Object, _protocolConfig); @@ -696,7 +696,7 @@ public void DiscoverTestsShouldNotUpdateFrameworkAndPlatformIfSpecifiedInDesignM .Returns(new FrameworkName(Constants.DotNetFramework451)); DiscoveryCriteria actualDiscoveryCriteria = null; var mockDiscoveryRequest = new Mock(); - _mockTestPlatform.Setup(mt => mt.CreateDiscoveryRequest(It.IsAny(), It.IsAny(), It.IsAny())).Callback( + _mockTestPlatform.Setup(mt => mt.CreateDiscoveryRequest(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny>())).Callback( (IRequestData requestData, DiscoveryCriteria discoveryCriteria, TestPlatformOptions options) => actualDiscoveryCriteria = discoveryCriteria).Returns(mockDiscoveryRequest.Object); _testRequestManager.DiscoverTests(payload, new Mock().Object, _protocolConfig); @@ -728,7 +728,7 @@ public void DiscoverTestsShouldUpdateFrameworkAndPlatformInCommandLineScenariosI .Returns(new FrameworkName(Constants.DotNetFramework46)); DiscoveryCriteria actualDiscoveryCriteria = null; var mockDiscoveryRequest = new Mock(); - _mockTestPlatform.Setup(mt => mt.CreateDiscoveryRequest(It.IsAny(), It.IsAny(), It.IsAny())).Callback( + _mockTestPlatform.Setup(mt => mt.CreateDiscoveryRequest(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny>())).Callback( (IRequestData requestData, DiscoveryCriteria discoveryCriteria, TestPlatformOptions options) => actualDiscoveryCriteria = discoveryCriteria).Returns(mockDiscoveryRequest.Object); _testRequestManager.DiscoverTests(payload, new Mock().Object, _protocolConfig); @@ -762,7 +762,7 @@ public void DiscoverTestsShouldNotUpdateFrameworkAndPlatformInCommandLineScenari DiscoveryCriteria actualDiscoveryCriteria = null; var mockDiscoveryRequest = new Mock(); _mockTestPlatform - .Setup(mt => mt.CreateDiscoveryRequest(It.IsAny(), It.IsAny(), It.IsAny())) + .Setup(mt => mt.CreateDiscoveryRequest(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny>())) .Callback( (IRequestData requestData, DiscoveryCriteria discoveryCriteria, TestPlatformOptions options) => actualDiscoveryCriteria = discoveryCriteria).Returns(mockDiscoveryRequest.Object); @@ -841,7 +841,7 @@ public void RunTestsShouldReadTheBatchSizeFromSettingsAndSetItForTestRunCriteria TestRunCriteria actualTestRunCriteria = null; var mockDiscoveryRequest = new Mock(); - _mockTestPlatform.Setup(mt => mt.CreateTestRunRequest(It.IsAny(), It.IsAny(), It.IsAny())).Callback( + _mockTestPlatform.Setup(mt => mt.CreateTestRunRequest(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny>())).Callback( (IRequestData requestData, TestRunCriteria runCriteria, TestPlatformOptions options) => actualTestRunCriteria = runCriteria).Returns(mockDiscoveryRequest.Object); _testRequestManager.RunTests(payload, new Mock().Object, new Mock().Object, _protocolConfig); @@ -865,7 +865,7 @@ public void RunTestsShouldNotThrowForFramework35() TestRunCriteria actualTestRunCriteria = null; var mockDiscoveryRequest = new Mock(); - _mockTestPlatform.Setup(mt => mt.CreateTestRunRequest(It.IsAny(), It.IsAny(), It.IsAny())).Callback( + _mockTestPlatform.Setup(mt => mt.CreateTestRunRequest(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny>())).Callback( (IRequestData requestData, TestRunCriteria runCriteria, TestPlatformOptions options) => actualTestRunCriteria = runCriteria).Returns(mockDiscoveryRequest.Object); _mockAssemblyMetadataProvider.Setup(a => a.GetFrameWork(It.IsAny())).Returns(new FrameworkName(Constants.DotNetFramework35)); @@ -890,7 +890,7 @@ public void RunTestsShouldPassSameProtocolConfigInRequestData() var mockProtocolConfig = new ProtocolConfig { Version = 6 }; IRequestData actualRequestData = null; var mockDiscoveryRequest = new Mock(); - _mockTestPlatform.Setup(mt => mt.CreateTestRunRequest(It.IsAny(), It.IsAny(), It.IsAny())).Callback( + _mockTestPlatform.Setup(mt => mt.CreateTestRunRequest(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny>())).Callback( (IRequestData requestData, TestRunCriteria runCriteria, TestPlatformOptions options) => actualRequestData = requestData).Returns(mockDiscoveryRequest.Object); // Act. @@ -914,7 +914,7 @@ public void RunTestsShouldCollectCommands() var mockProtocolConfig = new ProtocolConfig { Version = 6 }; IRequestData actualRequestData = null; var mockDiscoveryRequest = new Mock(); - _mockTestPlatform.Setup(mt => mt.CreateTestRunRequest(It.IsAny(), It.IsAny(), It.IsAny())).Callback( + _mockTestPlatform.Setup(mt => mt.CreateTestRunRequest(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny>())).Callback( (IRequestData requestData, TestRunCriteria runCriteria, TestPlatformOptions options) => actualRequestData = requestData).Returns(mockDiscoveryRequest.Object); _testRequestManager = new TestRequestManager( @@ -978,7 +978,7 @@ public void RunTestsShouldCollectTelemetryForLegacySettings() var mockProtocolConfig = new ProtocolConfig { Version = 6 }; IRequestData actualRequestData = null; var mockDiscoveryRequest = new Mock(); - _mockTestPlatform.Setup(mt => mt.CreateTestRunRequest(It.IsAny(), It.IsAny(), It.IsAny())).Callback( + _mockTestPlatform.Setup(mt => mt.CreateTestRunRequest(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny>())).Callback( (IRequestData requestData, TestRunCriteria runCriteria, TestPlatformOptions options) => actualRequestData = requestData).Returns(mockDiscoveryRequest.Object); _testRequestManager = new TestRequestManager( @@ -1025,7 +1025,7 @@ public void RunTestsShouldCollectTelemetryForTestSettingsEmbeddedInsideRunSettin var mockProtocolConfig = new ProtocolConfig { Version = 6 }; IRequestData actualRequestData = null; var mockDiscoveryRequest = new Mock(); - _mockTestPlatform.Setup(mt => mt.CreateTestRunRequest(It.IsAny(), It.IsAny(), It.IsAny())).Callback( + _mockTestPlatform.Setup(mt => mt.CreateTestRunRequest(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny>())).Callback( (IRequestData requestData, TestRunCriteria runCriteria, TestPlatformOptions options) => actualRequestData = requestData).Returns(mockDiscoveryRequest.Object); _testRequestManager = new TestRequestManager( @@ -1070,7 +1070,7 @@ public void RunTestsShouldCollectMetrics() var mockProtocolConfig = new ProtocolConfig { Version = 6 }; IRequestData actualRequestData = null; var mockDiscoveryRequest = new Mock(); - _mockTestPlatform.Setup(mt => mt.CreateTestRunRequest(It.IsAny(), It.IsAny(), It.IsAny())).Callback( + _mockTestPlatform.Setup(mt => mt.CreateTestRunRequest(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny>())).Callback( (IRequestData requestData, TestRunCriteria runCriteria, TestPlatformOptions options) => actualRequestData = requestData).Returns(mockDiscoveryRequest.Object); _testRequestManager = new TestRequestManager( @@ -1109,7 +1109,7 @@ public void RunTestsWithSourcesShouldCallTestPlatformAndSucceed() var createRunRequestCalled = 0; TestRunCriteria observedCriteria = null; var mockRunRequest = new Mock(); - _mockTestPlatform.Setup(mt => mt.CreateTestRunRequest(It.IsAny(), It.IsAny(), It.IsAny())).Callback( + _mockTestPlatform.Setup(mt => mt.CreateTestRunRequest(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny>())).Callback( (IRequestData requestData, TestRunCriteria runCriteria, TestPlatformOptions options) => { createRunRequestCalled++; @@ -1166,7 +1166,7 @@ public void RunTestsMultipleCallsShouldNotRunInParallel() }; var mockRunRequest = new Mock(); - _mockTestPlatform.Setup(mt => mt.CreateTestRunRequest(It.IsAny(), It.IsAny(), It.IsAny())) + _mockTestPlatform.Setup(mt => mt.CreateTestRunRequest(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny>())) .Returns(mockRunRequest.Object); var mockRunEventsRegistrar1 = new Mock(); @@ -1305,11 +1305,11 @@ public void DiscoverTestsShouldUpdateDesignModeAndCollectSourceInformation(bool var designmode = $"{designModeValue}"; _mockTestPlatform.Verify( - tp => tp.CreateDiscoveryRequest(It.IsAny(), It.Is(dc => dc.RunSettings.Contains(designmode)), It.IsAny())); + tp => tp.CreateDiscoveryRequest(It.IsAny(), It.Is(dc => dc.RunSettings.Contains(designmode)), It.IsAny(), It.IsAny>())); var collectSourceInformation = $"{designModeValue}"; _mockTestPlatform.Verify( - tp => tp.CreateDiscoveryRequest(It.IsAny(), It.Is(dc => dc.RunSettings.Contains(collectSourceInformation)), It.IsAny())); + tp => tp.CreateDiscoveryRequest(It.IsAny(), It.Is(dc => dc.RunSettings.Contains(collectSourceInformation)), It.IsAny(), It.IsAny>())); } [TestMethod] @@ -1323,7 +1323,7 @@ public void DiscoverTestsShouldNotUpdateDesignModeIfUserHasSetDesignModeInRunSet var designmode = "False"; _mockTestPlatform.Verify( - tp => tp.CreateDiscoveryRequest(It.IsAny(), It.Is(dc => dc.RunSettings.Contains(designmode)), It.IsAny())); + tp => tp.CreateDiscoveryRequest(It.IsAny(), It.Is(dc => dc.RunSettings.Contains(designmode)), It.IsAny(), It.IsAny>())); } [DataTestMethod] @@ -1343,7 +1343,7 @@ public void RunTestsShouldUpdateDesignModeIfRunnerIsInDesignMode(bool designMode _testRequestManager.RunTests(payload, new Mock().Object, new Mock().Object, _protocolConfig); var designmode = $"{designModeValue}"; - _mockTestPlatform.Verify(tp => tp.CreateTestRunRequest(It.IsAny(), It.Is(rc => rc.TestRunSettings.Contains(designmode)), It.IsAny())); + _mockTestPlatform.Verify(tp => tp.CreateTestRunRequest(It.IsAny(), It.Is(rc => rc.TestRunSettings.Contains(designmode)), It.IsAny(), It.IsAny>())); } [DataTestMethod] @@ -1358,7 +1358,7 @@ public void DiscoverTestsShouldNotUpdateCollectSourceInformationIfUserHasSetItIn var collectSourceInformation = $"{val}"; _mockTestPlatform.Verify( - tp => tp.CreateDiscoveryRequest(It.IsAny(), It.Is(dc => dc.RunSettings.Contains(collectSourceInformation)), It.IsAny())); + tp => tp.CreateDiscoveryRequest(It.IsAny(), It.Is(dc => dc.RunSettings.Contains(collectSourceInformation)), It.IsAny(), It.IsAny>())); } [TestMethod] @@ -1382,7 +1382,7 @@ public void RunTestsShouldShouldUpdateFrameworkAndPlatformIfNotSpecifiedInDesign .Returns(new FrameworkName(Constants.DotNetFramework46)); TestRunCriteria actualTestRunCriteria = null; var mockTestRunRequest = new Mock(); - _mockTestPlatform.Setup(mt => mt.CreateTestRunRequest(It.IsAny(), It.IsAny(), It.IsAny())).Callback( + _mockTestPlatform.Setup(mt => mt.CreateTestRunRequest(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny>())).Callback( (IRequestData requestData, TestRunCriteria runCriteria, TestPlatformOptions options) => actualTestRunCriteria = runCriteria).Returns(mockTestRunRequest.Object); _testRequestManager.RunTests(payload, new Mock().Object, new Mock().Object, _protocolConfig); @@ -1418,7 +1418,7 @@ public void RunTestsShouldNotUpdateFrameworkAndPlatformIfSpecifiedInDesignModeBu .Returns(new FrameworkName(Constants.DotNetFramework451)); TestRunCriteria actualTestRunCriteria = null; var mockTestRunRequest = new Mock(); - _mockTestPlatform.Setup(mt => mt.CreateTestRunRequest(It.IsAny(), It.IsAny(), It.IsAny())).Callback( + _mockTestPlatform.Setup(mt => mt.CreateTestRunRequest(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny>())).Callback( (IRequestData requestData, TestRunCriteria runCriteria, TestPlatformOptions options) => actualTestRunCriteria = runCriteria).Returns(mockTestRunRequest.Object); _testRequestManager.RunTests(payload, new Mock().Object, new Mock().Object, _protocolConfig); @@ -1456,7 +1456,7 @@ public void RunTestsShouldNotUpdatePlatformIfSpecifiedInDesignModeButInferred(st .Returns(new FrameworkName(Constants.DotNetFramework451)); TestRunCriteria actualTestRunCriteria = null; var mockTestRunRequest = new Mock(); - _mockTestPlatform.Setup(mt => mt.CreateTestRunRequest(It.IsAny(), It.IsAny(), It.IsAny())).Callback( + _mockTestPlatform.Setup(mt => mt.CreateTestRunRequest(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny>())).Callback( (IRequestData requestData, TestRunCriteria runCriteria, TestPlatformOptions options) => actualTestRunCriteria = runCriteria).Returns(mockTestRunRequest.Object); _testRequestManager.RunTests(payload, new Mock().Object, new Mock().Object, _protocolConfig); @@ -1488,7 +1488,7 @@ public void RunTestsShouldUpdateFrameworkAndPlatformInCommandLineScenarios() .Returns(new FrameworkName(Constants.DotNetFramework46)); TestRunCriteria actualTestRunCriteria = null; var mockTestRunRequest = new Mock(); - _mockTestPlatform.Setup(mt => mt.CreateTestRunRequest(It.IsAny(), It.IsAny(), It.IsAny())).Callback( + _mockTestPlatform.Setup(mt => mt.CreateTestRunRequest(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny>())).Callback( (IRequestData requestData, TestRunCriteria runCriteria, TestPlatformOptions options) => actualTestRunCriteria = runCriteria).Returns(mockTestRunRequest.Object); _testRequestManager.RunTests(payload, new Mock().Object, new Mock().Object, _protocolConfig); @@ -1523,7 +1523,7 @@ public void RunTestsShouldNotpdateFrameworkAndPlatformInCommandLineScenariosIfSp .Returns(new FrameworkName(Constants.DotNetFramework46)); TestRunCriteria actualTestRunCriteria = null; var mockTestRunRequest = new Mock(); - _mockTestPlatform.Setup(mt => mt.CreateTestRunRequest(It.IsAny(), It.IsAny(), It.IsAny())).Callback( + _mockTestPlatform.Setup(mt => mt.CreateTestRunRequest(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny>())).Callback( (IRequestData requestData, TestRunCriteria runCriteria, TestPlatformOptions options) => actualTestRunCriteria = runCriteria).Returns(mockTestRunRequest.Object); _testRequestManager.RunTests(payload, new Mock().Object, new Mock().Object, _protocolConfig); @@ -1563,7 +1563,7 @@ public void RunTestsWithTestCasesShouldUpdateFrameworkAndPlatformIfNotSpecifiedI .Returns(new FrameworkName(Constants.DotNetFramework46)); TestRunCriteria actualTestRunCriteria = null; var mockTestRunRequest = new Mock(); - _mockTestPlatform.Setup(mt => mt.CreateTestRunRequest(It.IsAny(), It.IsAny(), It.IsAny())).Callback( + _mockTestPlatform.Setup(mt => mt.CreateTestRunRequest(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny>())).Callback( (IRequestData requestData, TestRunCriteria runCriteria, TestPlatformOptions options) => actualTestRunCriteria = runCriteria).Returns(mockTestRunRequest.Object); _testRequestManager.RunTests(payload, new Mock().Object, new Mock().Object, _protocolConfig); @@ -1698,7 +1698,7 @@ public void RunTestsShouldAddConsoleLoggerInRunSettingsInNonDesignMode() _commandLineOptions.IsDesignMode = false; TestRunCriteria actualTestRunCriteria = null; var mockTestRunRequest = new Mock(); - _mockTestPlatform.Setup(mt => mt.CreateTestRunRequest(It.IsAny(), It.IsAny(), It.IsAny())).Callback( + _mockTestPlatform.Setup(mt => mt.CreateTestRunRequest(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny>())).Callback( (IRequestData requestData, TestRunCriteria runCriteria, TestPlatformOptions options) => actualTestRunCriteria = runCriteria).Returns(mockTestRunRequest.Object); _testRequestManager.RunTests(payload, new Mock().Object, new Mock().Object, _protocolConfig); @@ -1737,7 +1737,7 @@ public void RunTestsShouldAddConsoleLoggerInRunSettingsIfDesignModeSetFalseInRun _commandLineOptions.IsDesignMode = true; TestRunCriteria actualTestRunCriteria = null; var mockTestRunRequest = new Mock(); - _mockTestPlatform.Setup(mt => mt.CreateTestRunRequest(It.IsAny(), It.IsAny(), It.IsAny())).Callback( + _mockTestPlatform.Setup(mt => mt.CreateTestRunRequest(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny>())).Callback( (IRequestData requestData, TestRunCriteria runCriteria, TestPlatformOptions options) => actualTestRunCriteria = runCriteria).Returns(mockTestRunRequest.Object); _testRequestManager.RunTests(payload, new Mock().Object, new Mock().Object, _protocolConfig); @@ -1777,7 +1777,7 @@ public void DiscoverTestsShouldAddConsoleLoggerInRunSettingsIfDesignModeSetFalse DiscoveryCriteria actualDiscoveryCriteria = null; var mockDiscoveryRequest = new Mock(); _mockTestPlatform - .Setup(mt => mt.CreateDiscoveryRequest(It.IsAny(), It.IsAny(), It.IsAny())) + .Setup(mt => mt.CreateDiscoveryRequest(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny>())) .Callback( (IRequestData requestData, DiscoveryCriteria discoveryCriteria, TestPlatformOptions options) => actualDiscoveryCriteria = discoveryCriteria).Returns(mockDiscoveryRequest.Object); @@ -1811,7 +1811,7 @@ public void RunTestsShouldNotAddConsoleLoggerInRunSettingsInDesignMode() _commandLineOptions.IsDesignMode = false; TestRunCriteria actualTestRunCriteria = null; var mockTestRunRequest = new Mock(); - _mockTestPlatform.Setup(mt => mt.CreateTestRunRequest(It.IsAny(), It.IsAny(), It.IsAny())).Callback( + _mockTestPlatform.Setup(mt => mt.CreateTestRunRequest(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny>())).Callback( (IRequestData requestData, TestRunCriteria runCriteria, TestPlatformOptions options) => actualTestRunCriteria = runCriteria).Returns(mockTestRunRequest.Object); _testRequestManager.RunTests(payload, new Mock().Object, new Mock().Object, _protocolConfig); @@ -1835,7 +1835,7 @@ public void DiscoverTestsShouldAddConsoleLoggerInRunSettingsInNonDesignMode() DiscoveryCriteria actualDiscoveryCriteria = null; var mockDiscoveryRequest = new Mock(); _mockTestPlatform - .Setup(mt => mt.CreateDiscoveryRequest(It.IsAny(), It.IsAny(), It.IsAny())) + .Setup(mt => mt.CreateDiscoveryRequest(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny>())) .Callback( (IRequestData requestData, DiscoveryCriteria discoveryCriteria, TestPlatformOptions options) => actualDiscoveryCriteria = discoveryCriteria).Returns(mockDiscoveryRequest.Object); @@ -1867,7 +1867,7 @@ public void DiscoverTestsShouldNotAddConsoleLoggerInRunSettingsInDesignMode() DiscoveryCriteria actualDiscoveryCriteria = null; var mockDiscoveryRequest = new Mock(); _mockTestPlatform - .Setup(mt => mt.CreateDiscoveryRequest(It.IsAny(), It.IsAny(), It.IsAny())) + .Setup(mt => mt.CreateDiscoveryRequest(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny>())) .Callback( (IRequestData requestData, DiscoveryCriteria discoveryCriteria, TestPlatformOptions options) => actualDiscoveryCriteria = discoveryCriteria).Returns(mockDiscoveryRequest.Object); @@ -1909,7 +1909,7 @@ public void RunTestsShouldOverrideOnlyAssemblyNameIfConsoleLoggerAlreadyPresentI _commandLineOptions.IsDesignMode = false; TestRunCriteria actualTestRunCriteria = null; var mockTestRunRequest = new Mock(); - _mockTestPlatform.Setup(mt => mt.CreateTestRunRequest(It.IsAny(), It.IsAny(), It.IsAny())).Callback( + _mockTestPlatform.Setup(mt => mt.CreateTestRunRequest(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny>())).Callback( (IRequestData requestData, TestRunCriteria runCriteria, TestPlatformOptions options) => actualTestRunCriteria = runCriteria).Returns(mockTestRunRequest.Object); _testRequestManager.RunTests(payload, new Mock().Object, new Mock().Object, _protocolConfig); @@ -1958,7 +1958,7 @@ public void DiscoverTestsShouldOverrideOnlyAssemblyNameIfConsoleLoggerAlreadyPre DiscoveryCriteria actualDiscoveryCriteria = null; var mockDiscoveryRequest = new Mock(); _mockTestPlatform - .Setup(mt => mt.CreateDiscoveryRequest(It.IsAny(), It.IsAny(), It.IsAny())) + .Setup(mt => mt.CreateDiscoveryRequest(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny>())) .Callback( (IRequestData requestData, DiscoveryCriteria discoveryCriteria, TestPlatformOptions options) => actualDiscoveryCriteria = discoveryCriteria).Returns(mockDiscoveryRequest.Object); @@ -2010,7 +2010,7 @@ public void RunTestsShouldOverrideOnlyAssemblyNameIfConsoleLoggerAlreadyPresentI _commandLineOptions.IsDesignMode = false; TestRunCriteria actualTestRunCriteria = null; var mockTestRunRequest = new Mock(); - _mockTestPlatform.Setup(mt => mt.CreateTestRunRequest(It.IsAny(), It.IsAny(), It.IsAny())).Callback( + _mockTestPlatform.Setup(mt => mt.CreateTestRunRequest(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny>())).Callback( (IRequestData requestData, TestRunCriteria runCriteria, TestPlatformOptions options) => actualTestRunCriteria = runCriteria).Returns(mockTestRunRequest.Object); _testRequestManager.RunTests(payload, new Mock().Object, new Mock().Object, _protocolConfig); @@ -2059,7 +2059,7 @@ public void DiscoverTestsShouldOverrideOnlyAssemblyNameIfConsoleLoggerAlreadyPre DiscoveryCriteria actualDiscoveryCriteria = null; var mockDiscoveryRequest = new Mock(); _mockTestPlatform - .Setup(mt => mt.CreateDiscoveryRequest(It.IsAny(), It.IsAny(), It.IsAny())) + .Setup(mt => mt.CreateDiscoveryRequest(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny>())) .Callback( (IRequestData requestData, DiscoveryCriteria discoveryCriteria, TestPlatformOptions options) => actualDiscoveryCriteria = discoveryCriteria).Returns(mockDiscoveryRequest.Object); @@ -2188,7 +2188,8 @@ public void StartTestSessionShouldPassCorrectTelemetryOptedInOptionToTestPlatfor tp => tp.StartTestSession( It.IsAny(), It.IsAny(), - It.IsAny())) + It.IsAny(), + It.IsAny>())) .Returns(true) .Callback( (IRequestData rd, StartTestSessionCriteria _, ITestSessionEventsHandler _) => Assert.IsTrue(rd.IsTelemetryOptedIn)); @@ -2234,7 +2235,8 @@ public void StartTestSessionShouldUpdateSettings() tp => tp.StartTestSession( It.IsAny(), It.IsAny(), - It.IsAny())) + It.IsAny(), + It.IsAny>())) .Returns(true) .Callback( (IRequestData _, StartTestSessionCriteria criteria, ITestSessionEventsHandler _) => @@ -2304,7 +2306,8 @@ public void StartTestSessionShouldBeSuccessful() tp => tp.StartTestSession( It.IsAny(), It.IsAny(), - It.IsAny())) + It.IsAny(), + It.IsAny>())) .Returns(true); _testRequestManager.StartTestSession( @@ -2496,7 +2499,7 @@ private void RunTestsIfThrowsExceptionShouldThrowOut(Exception exception) var createRunRequestCalled = 0; TestRunCriteria observedCriteria = null; var mockRunRequest = new Mock(); - _mockTestPlatform.Setup(mt => mt.CreateTestRunRequest(It.IsAny(), It.IsAny(), It.IsAny())).Callback( + _mockTestPlatform.Setup(mt => mt.CreateTestRunRequest(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny>())).Callback( (IRequestData requestData, TestRunCriteria runCriteria, TestPlatformOptions options) => { createRunRequestCalled++; @@ -2521,7 +2524,7 @@ private void DiscoverTestsIfThrowsExceptionShouldThrowOut(Exception exception) DiscoveryCriteria observedCriteria = null; var mockDiscoveryRequest = new Mock(); - _mockTestPlatform.Setup(mt => mt.CreateDiscoveryRequest(It.IsAny(), It.IsAny(), It.IsAny())).Callback( + _mockTestPlatform.Setup(mt => mt.CreateDiscoveryRequest(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny>())).Callback( (IRequestData requestData, DiscoveryCriteria discoveryCriteria, TestPlatformOptions options) => observedCriteria = discoveryCriteria).Returns(mockDiscoveryRequest.Object); mockDiscoveryRequest.Setup(mr => mr.DiscoverAsync()).Throws(exception); From a4873253e81b4c6f61a73e64e052e57912d5de34 Mon Sep 17 00:00:00 2001 From: nohwnd Date: Wed, 2 Mar 2022 16:49:40 +0100 Subject: [PATCH 014/112] Fix some unit tests --- .../Parallel/ParallelOperationManager.cs | 6 +- .../Parallel/ParallelProxyDiscoveryManager.cs | 2 +- .../TestEngine.cs | 152 +++++----- .../ParallelProxyDiscoveryManagerTests.cs | 195 ++++++++----- .../ParallelProxyExecutionManagerTests.cs | 266 ++++++++++-------- .../TestEngineTests.cs | 125 ++++---- 6 files changed, 404 insertions(+), 342 deletions(-) diff --git a/src/Microsoft.TestPlatform.CrossPlatEngine/Client/Parallel/ParallelOperationManager.cs b/src/Microsoft.TestPlatform.CrossPlatEngine/Client/Parallel/ParallelOperationManager.cs index 1e00d5c2c1..c37d9fb2aa 100644 --- a/src/Microsoft.TestPlatform.CrossPlatEngine/Client/Parallel/ParallelOperationManager.cs +++ b/src/Microsoft.TestPlatform.CrossPlatEngine/Client/Parallel/ParallelOperationManager.cs @@ -75,7 +75,7 @@ private bool RunWorkInParallel() throw new InvalidOperationException($"{nameof(_eventHandler)} was not provided."); if (_getEventHandler == null) - throw new InvalidOperationException($"{nameof(_eventHandler)} was not provided."); + throw new InvalidOperationException($"{nameof(_getEventHandler)} was not provided."); if (_runWorkload == null) throw new InvalidOperationException($"{nameof(_runWorkload)} was not provided."); @@ -125,14 +125,14 @@ private bool RunWorkInParallel() return workToRun.Count > 0; } - public bool RunNextWork(TManager completedManager) + public bool RunNextWork(TManager completedManager!!) { lock (_lock) { var completedSlot = _managerSlots.Where(s => ReferenceEquals(completedManager, s.Manager)).ToList(); if (!completedSlot.Any()) { - // yikes, we should have found it there + throw new InvalidOperationException("The provided manager was not found in any slot."); } var slot = completedSlot.Single(); diff --git a/src/Microsoft.TestPlatform.CrossPlatEngine/Client/Parallel/ParallelProxyDiscoveryManager.cs b/src/Microsoft.TestPlatform.CrossPlatEngine/Client/Parallel/ParallelProxyDiscoveryManager.cs index 8f0a19f37c..fc63454529 100644 --- a/src/Microsoft.TestPlatform.CrossPlatEngine/Client/Parallel/ParallelProxyDiscoveryManager.cs +++ b/src/Microsoft.TestPlatform.CrossPlatEngine/Client/Parallel/ParallelProxyDiscoveryManager.cs @@ -73,7 +73,7 @@ public void Initialize(bool skipDefaultAdapters) } /// - public void DiscoverTests(DiscoveryCriteria discoveryCriteria, ITestDiscoveryEventsHandler2 eventHandler) + public void DiscoverTests(DiscoveryCriteria discoveryCriteria!!, ITestDiscoveryEventsHandler2 eventHandler!!) { var workloads = SplitToWorkloads(discoveryCriteria, _sourceToTestHostProviderMap); _availableTestSources = workloads.Count; diff --git a/src/Microsoft.TestPlatform.CrossPlatEngine/TestEngine.cs b/src/Microsoft.TestPlatform.CrossPlatEngine/TestEngine.cs index cf2adf39d1..503601f922 100644 --- a/src/Microsoft.TestPlatform.CrossPlatEngine/TestEngine.cs +++ b/src/Microsoft.TestPlatform.CrossPlatEngine/TestEngine.cs @@ -210,40 +210,59 @@ public IProxyExecutionManager GetExecutionManager( new TestHostManagerFactory(newRequestData)); } + // This creates a single non-parallel execution manager, based requestData, isDataCollectorEnabled and the + // overall testRunCriteria. The overall testRunCriteria are split to smaller pieces (e.g. each source from the overall + // testRunCriteria) so we can run them in parallel, and those are then passed to those non-parallel execution managers. + // + // The function below grabs most of the parameter via closure from the local context, + // but gets the runtime provider later, because that is specific info to the source (or sources) it will be running. + // This creator does not get those smaller pieces of testRunCriteria, those come later when we call a method on + // the non-parallel execution manager we create here. E.g. StartTests(). + Func proxyExecutionManagerCreator = runtimeProviderInfo => + CreateNonParallelExecutionManager(requestData, testRunCriteria, isDataCollectorEnabled, runtimeProviderInfo); + + var executionManager = new ParallelProxyExecutionManager(requestData, proxyExecutionManagerCreator, parallelLevel, testHostProviders); + + EqtTrace.Verbose($"TestEngine.GetExecutionManager: Chosen execution manager '{executionManager.GetType().AssemblyQualifiedName}' ParallelLevel '{parallelLevel}'."); + + return executionManager; + } + + // This is internal so tests can use it. + internal IProxyExecutionManager CreateNonParallelExecutionManager(IRequestData requestData, TestRunCriteria testRunCriteria, bool isDataCollectorEnabled, TestRuntimeProviderInfo runtimeProviderInfo) + { // SetupChannel ProxyExecutionManager with data collection if data collectors are // specified in run settings. - Func proxyExecutionManagerCreator = runtimeProviderInfo => + // Create a new host manager, to be associated with individual + // ProxyExecutionManager(&POM) + var sources = runtimeProviderInfo.SourceDetails.Select(r => r.Source).ToList(); + var hostManager = _testHostProviderManager.GetTestHostManagerByRunConfiguration(runtimeProviderInfo.RunSettings, sources); + ThrowExceptionIfTestHostManagerIsNull(hostManager, runtimeProviderInfo.RunSettings); + hostManager.Initialize(TestSessionMessageLogger.Instance, runtimeProviderInfo.RunSettings); + + if (testRunCriteria.TestHostLauncher != null) { - // Create a new host manager, to be associated with individual - // ProxyExecutionManager(&POM) - var sources = runtimeProviderInfo.SourceDetails.Select(r => r.Source).ToList(); - var hostManager = _testHostProviderManager.GetTestHostManagerByRunConfiguration(runtimeProviderInfo.RunSettings, sources); - ThrowExceptionIfTestHostManagerIsNull(hostManager, runtimeProviderInfo.RunSettings); - hostManager.Initialize(TestSessionMessageLogger.Instance, runtimeProviderInfo.RunSettings); + hostManager.SetCustomLauncher(testRunCriteria.TestHostLauncher); + } - if (testRunCriteria.TestHostLauncher != null) - { - hostManager.SetCustomLauncher(testRunCriteria.TestHostLauncher); - } + var requestSender = new TestRequestSender(requestData.ProtocolConfig, hostManager); - var requestSender = new TestRequestSender(requestData.ProtocolConfig, hostManager); + if (testRunCriteria.TestSessionInfo != null) + { + // This function is used to either take a pre-existing proxy operation manager from + // the test pool or to create a new proxy operation manager on the spot. + Func + proxyOperationManagerCreator = ( + string source, + ProxyExecutionManager proxyExecutionManager) => + { + var proxyOperationManager = TestSessionPool.Instance.TryTakeProxy( + testRunCriteria.TestSessionInfo, + source, + runtimeProviderInfo.RunSettings); - if (testRunCriteria.TestSessionInfo != null) - { - // This function is used to either take a pre-existing proxy operation manager from - // the test pool or to create a new proxy operation manager on the spot. - Func - proxyOperationManagerCreator = ( - string source, - ProxyExecutionManager proxyExecutionManager) => + if (proxyOperationManager == null) { - var proxyOperationManager = TestSessionPool.Instance.TryTakeProxy( - testRunCriteria.TestSessionInfo, - source, - runtimeProviderInfo.RunSettings); - - if (proxyOperationManager == null) - { // If the proxy creation process based on test session info failed, then // we'll proceed with the normal creation process as if no test session // info was passed in in the first place. @@ -252,46 +271,39 @@ public IProxyExecutionManager GetExecutionManager( // regarding the test session pool operation and consistency. EqtTrace.Warning("ProxyExecutionManager creation with test session failed."); - proxyOperationManager = new ProxyOperationManager( - requestData, - requestSender, - hostManager, - proxyExecutionManager); - } - - return proxyOperationManager; - }; - - // In case we have an active test session, data collection needs were - // already taken care of when first creating the session. As a consequence - // we always return this proxy instead of choosing between the vanilla - // execution proxy and the one with data collection enabled. - return new ProxyExecutionManager( - testRunCriteria.TestSessionInfo, - proxyOperationManagerCreator, - testRunCriteria.DebugEnabledForTestSession); - } - - return isDataCollectorEnabled - ? new ProxyExecutionManagerWithDataCollection( - requestData, - requestSender, - hostManager, - new ProxyDataCollectionManager( - requestData, - runtimeProviderInfo.RunSettings, - GetSourcesFromTestRunCriteria(testRunCriteria))) - : new ProxyExecutionManager( - requestData, - requestSender, - hostManager); - }; + proxyOperationManager = new ProxyOperationManager( + requestData, + requestSender, + hostManager, + proxyExecutionManager); + } - var executionManager = new ParallelProxyExecutionManager(requestData, proxyExecutionManagerCreator, parallelLevel, testHostProviders); + return proxyOperationManager; + }; - EqtTrace.Verbose($"TestEngine.GetExecutionManager: Chosen execution manager '{executionManager.GetType().AssemblyQualifiedName}' ParallelLevel '{parallelLevel}'."); + // In case we have an active test session, data collection needs were + // already taken care of when first creating the session. As a consequence + // we always return this proxy instead of choosing between the vanilla + // execution proxy and the one with data collection enabled. + return new ProxyExecutionManager( + testRunCriteria.TestSessionInfo, + proxyOperationManagerCreator, + testRunCriteria.DebugEnabledForTestSession); + } - return executionManager; + return isDataCollectorEnabled + ? new ProxyExecutionManagerWithDataCollection( + requestData, + requestSender, + hostManager, + new ProxyDataCollectionManager( + requestData, + runtimeProviderInfo.RunSettings, + sources)) + : new ProxyExecutionManager( + requestData, + requestSender, + hostManager); } /// @@ -563,18 +575,6 @@ private IRequestData GetRequestData(bool isTelemetryOptedIn) }; } - /// - /// Gets test sources from test run criteria. - /// - /// - /// The test sources. - private IEnumerable GetSourcesFromTestRunCriteria(TestRunCriteria testRunCriteria) - { - return testRunCriteria.HasSpecificTests - ? TestSourcesUtility.GetSources(testRunCriteria.Tests) - : testRunCriteria.Sources; - } - private static void ThrowExceptionIfTestHostManagerIsNull(ITestRuntimeProvider testHostManager, string settingsXml) { if (testHostManager == null) diff --git a/test/Microsoft.TestPlatform.CrossPlatEngine.UnitTests/Client/Parallel/ParallelProxyDiscoveryManagerTests.cs b/test/Microsoft.TestPlatform.CrossPlatEngine.UnitTests/Client/Parallel/ParallelProxyDiscoveryManagerTests.cs index e9e714d4b3..ece360a1b1 100644 --- a/test/Microsoft.TestPlatform.CrossPlatEngine.UnitTests/Client/Parallel/ParallelProxyDiscoveryManagerTests.cs +++ b/test/Microsoft.TestPlatform.CrossPlatEngine.UnitTests/Client/Parallel/ParallelProxyDiscoveryManagerTests.cs @@ -27,13 +27,15 @@ namespace TestPlatform.CrossPlatEngine.UnitTests.Client; [TestClass] public class ParallelProxyDiscoveryManagerTests { - private const int TaskTimeout = 15 * 1000; // In milliseconds. - private readonly List> _createdMockManagers; - private readonly Func _proxyManagerFunc; - private readonly Mock _mockHandler; + private const int Timeout3Seconds = 3 * 1000; + private readonly Queue> _preCreatedMockManagers; + private readonly List> _usedMockManagers; + private readonly Func _createMockManager; + private readonly Mock _mockEventHandler; private readonly List _sources = new() { "1.dll", "2.dll" }; - private readonly DiscoveryCriteria _testDiscoveryCriteria; - private bool _proxyManagerFuncCalled; + private readonly DiscoveryCriteria _discoveryCriteriaWith2Sources; + private readonly List _runtimeProviders; + private int _createMockManagerCalled; private readonly List _processedSources; private readonly ManualResetEventSlim _discoveryCompleted; private readonly Mock _mockRequestData; @@ -41,61 +43,104 @@ public class ParallelProxyDiscoveryManagerTests public ParallelProxyDiscoveryManagerTests() { _processedSources = new List(); - _createdMockManagers = new List>(); - _proxyManagerFunc = _ => + _preCreatedMockManagers = new Queue>( + new List>{ + // have at least as many of them as you have test dlls + // they will be dequeued when we "create" a non-parallel + // manager. The setup adds callback for handler to complete + // the discovery. + new Mock(), + new Mock(), + new Mock(), + new Mock(), + new Mock(), + }); + _usedMockManagers = new List>(); + _createMockManager = _ => { - _proxyManagerFuncCalled = true; - var manager = new Mock(); - _createdMockManagers.Add(manager); + // We create the manager at the last possible + // moment now, not when we create the parallel proxy manager class + // so rather than creating the class here, and adding the mock setup + // that allows the tests to complete. We instead pre-create a bunch of managers + // and then grab and setup the ones we need, and only assert on the used ones. + _createMockManagerCalled++; + var manager = _preCreatedMockManagers.Dequeue(); + _usedMockManagers.Add(manager); return manager.Object; }; - _mockHandler = new Mock(); - _testDiscoveryCriteria = new DiscoveryCriteria(_sources, 100, null); + _mockEventHandler = new Mock(); + _discoveryCriteriaWith2Sources = new DiscoveryCriteria(_sources, 100, null); + _runtimeProviders = new List { + new TestRuntimeProviderInfo(typeof(ITestRuntimeProvider), false, "", new List + { + new SourceDetail{ Source = _sources[0], Architecture = Architecture.X86, Framework = Framework.DefaultFramework }, + new SourceDetail{ Source = _sources[1], Architecture = Architecture.X86, Framework = Framework.DefaultFramework } + }) + }; + + // This event is Set by callback from _mockEventHandler in SetupDiscoveryManager _discoveryCompleted = new ManualResetEventSlim(false); _mockRequestData = new Mock(); _mockRequestData.Setup(rd => rd.MetricsCollection).Returns(new NoOpMetricsCollection()); } [TestMethod] - public void InitializeShouldCallAllConcurrentManagersOnce() + public void CreatingAndInitializingProxyExecutionManagerDoesNothingUntilThereIsActualWorkToDo() { InvokeAndVerifyInitialize(3); } [TestMethod] - public void InitializeShouldCallAllConcurrentManagersWithFalseFlagIfSkipDefaultAdaptersIsFalse() + public void CreatingAndInitializingProxyExecutionManagerDoesNothingUntilThereIsActualWorkToDoButItKeepsSkipDefaultAdaptersValueFalse() { - InvokeAndVerifyInitialize(3, false); + InvokeAndVerifyInitialize(3, skipDefaultAdapters: false); } [TestMethod] - public void InitializeShouldCallAllConcurrentManagersWithTrueFlagIfSkipDefaultAdaptersIsTrue() + public void CreatingAndInitializingProxyExecutionManagerDoesNothingUntilThereIsActualWorkToDoButItKeepsSkipDefaultAdaptersValueTrue() { - InvokeAndVerifyInitialize(3, true); + InvokeAndVerifyInitialize(3, skipDefaultAdapters: true); } [TestMethod] public void AbortShouldCallAllConcurrentManagersOnce() { - var parallelDiscoveryManager = new ParallelProxyDiscoveryManager(_mockRequestData.Object, _proxyManagerFunc, 4, new List()); + var parallelDiscoveryManager = new ParallelProxyDiscoveryManager(_mockRequestData.Object, _createMockManager, parallelLevel: 1000, _runtimeProviders); + + // Starting parallel discovery will create 2 proxy managers, which we will then promptly abort. + parallelDiscoveryManager.DiscoverTests(_discoveryCriteriaWith2Sources, new Mock().Object); parallelDiscoveryManager.Abort(); - Assert.AreEqual(4, _createdMockManagers.Count, "Number of Concurrent Managers created should be 4"); - _createdMockManagers.ForEach(dm => dm.Verify(m => m.Abort(), Times.Once)); + Assert.AreEqual(2, _usedMockManagers.Count, "Number of Concurrent Managers created should be equal to the number of sources that should run"); + _usedMockManagers.ForEach(dm => dm.Verify(m => m.Abort(), Times.Once)); } [TestMethod] public void DiscoverTestsShouldProcessAllSources() { // Testcase filter should be passed to all parallel discovery criteria. - _testDiscoveryCriteria.TestCaseFilter = "Name~Test"; - var parallelDiscoveryManager = SetupDiscoveryManager(_proxyManagerFunc, 2, false); + _discoveryCriteriaWith2Sources.TestCaseFilter = "Name~Test"; + var parallelDiscoveryManager = SetupDiscoveryManager(_createMockManager, 2, false); - Task.Run(() => parallelDiscoveryManager.DiscoverTests(_testDiscoveryCriteria, _mockHandler.Object)); + var task = Task.Run(() => parallelDiscoveryManager.DiscoverTests(_discoveryCriteriaWith2Sources, _mockEventHandler.Object)); + var discoveryCompleted = _discoveryCompleted.Wait(Timeout3Seconds); - Assert.IsTrue(_discoveryCompleted.Wait(TaskTimeout), "Test discovery not completed."); + if (task.IsCompleted) + { + // If the work is done, either there is output, + // or an exception that we want to "receive" to + // fail our test. + task.GetAwaiter().GetResult(); + } + else + { + // We don't want to await the result because we + // completed or timed out on the event above. + } + + Assert.IsTrue(discoveryCompleted, "Test discovery not completed."); Assert.AreEqual(_sources.Count, _processedSources.Count, "All Sources must be processed."); AssertMissingAndDuplicateSources(_processedSources); } @@ -103,10 +148,10 @@ public void DiscoverTestsShouldProcessAllSources() [TestMethod] public void HandlePartialDiscoveryCompleteShouldReturnTrueIfDiscoveryWasAbortedWithEventHandler() { - var parallelDiscoveryManager = new ParallelProxyDiscoveryManager(_mockRequestData.Object, _proxyManagerFunc, 1, new List()); + var parallelDiscoveryManager = new ParallelProxyDiscoveryManager(_mockRequestData.Object, _createMockManager, 1, new List()); var proxyDiscovermanager = new ProxyDiscoveryManager(_mockRequestData.Object, new Mock().Object, new Mock().Object); - parallelDiscoveryManager.Abort(_mockHandler.Object); + parallelDiscoveryManager.Abort(_mockEventHandler.Object); bool isPartialDiscoveryComplete = parallelDiscoveryManager.HandlePartialDiscoveryComplete(proxyDiscovermanager, 20, new List(), isAborted: false); Assert.IsTrue(isPartialDiscoveryComplete); @@ -115,7 +160,7 @@ public void HandlePartialDiscoveryCompleteShouldReturnTrueIfDiscoveryWasAbortedW [TestMethod] public void HandlePartialDiscoveryCompleteShouldReturnTrueIfDiscoveryWasAborted() { - var parallelDiscoveryManager = new ParallelProxyDiscoveryManager(_mockRequestData.Object, _proxyManagerFunc, 1, new List()); + var parallelDiscoveryManager = new ParallelProxyDiscoveryManager(_mockRequestData.Object, _createMockManager, 1, new List()); var proxyDiscovermanager = new ProxyDiscoveryManager(_mockRequestData.Object, new Mock().Object, new Mock().Object); parallelDiscoveryManager.Abort(); @@ -129,16 +174,16 @@ public void DiscoveryTestsShouldStopDiscoveryIfAbortionWasRequested() { // Since the hosts are aborted, total aggregated tests sent across will be -1 var discoveryManagerMock = new Mock(); - _createdMockManagers.Add(discoveryManagerMock); + _preCreatedMockManagers.Enqueue(discoveryManagerMock); var parallelDiscoveryManager = SetupDiscoveryManager(_ => discoveryManagerMock.Object, 1, true, totalTests: -1); Task.Run(() => { - parallelDiscoveryManager.DiscoverTests(_testDiscoveryCriteria, _mockHandler.Object); + parallelDiscoveryManager.DiscoverTests(_discoveryCriteriaWith2Sources, _mockEventHandler.Object); parallelDiscoveryManager.Abort(); }); - Assert.IsTrue(_discoveryCompleted.Wait(TaskTimeout), "Test discovery not completed."); + Assert.IsTrue(_discoveryCompleted.Wait(Timeout3Seconds), "Test discovery not completed."); Assert.AreEqual(1, _processedSources.Count, "One source should be processed."); } @@ -147,16 +192,16 @@ public void DiscoveryTestsShouldStopDiscoveryIfAbortionWithEventHandlerWasReques { // Since the hosts are aborted, total aggregated tests sent across will be -1 var discoveryManagerMock = new Mock(); - _createdMockManagers.Add(discoveryManagerMock); + _preCreatedMockManagers.Enqueue(discoveryManagerMock); var parallelDiscoveryManager = SetupDiscoveryManager(_ => discoveryManagerMock.Object, 1, true, totalTests: -1); Task.Run(() => { - parallelDiscoveryManager.DiscoverTests(_testDiscoveryCriteria, _mockHandler.Object); - parallelDiscoveryManager.Abort(_mockHandler.Object); + parallelDiscoveryManager.DiscoverTests(_discoveryCriteriaWith2Sources, _mockEventHandler.Object); + parallelDiscoveryManager.Abort(_mockEventHandler.Object); }); - Assert.IsTrue(_discoveryCompleted.Wait(TaskTimeout), "Test discovery not completed."); + Assert.IsTrue(_discoveryCompleted.Wait(Timeout3Seconds), "Test discovery not completed."); Assert.AreEqual(1, _processedSources.Count, "One source should be processed."); } @@ -165,17 +210,18 @@ public void DiscoveryTestsShouldProcessAllSourceIfOneDiscoveryManagerIsStarved() { // Ensure that second discovery manager never starts. Expect 10 total tests. // Override DiscoveryComplete since overall aborted should be true - var parallelDiscoveryManager = SetupDiscoveryManager(_proxyManagerFunc, 2, false, totalTests: 10); - _createdMockManagers[1].Reset(); - _createdMockManagers[1].Setup(dm => dm.DiscoverTests(It.IsAny(), It.IsAny())) + var parallelDiscoveryManager = SetupDiscoveryManager(_createMockManager, 2, false, totalTests: 10); + var secondMockManager = _preCreatedMockManagers.ToArray()[1]; + secondMockManager.Reset(); + secondMockManager.Setup(dm => dm.DiscoverTests(It.IsAny(), It.IsAny())) .Throws(); - _mockHandler.Setup(mh => mh.HandleDiscoveryComplete(It.IsAny(), null)) + _mockEventHandler.Setup(mh => mh.HandleDiscoveryComplete(It.IsAny(), null)) .Callback>((t, l) => _discoveryCompleted.Set()); - Task.Run(() => parallelDiscoveryManager.DiscoverTests(_testDiscoveryCriteria, _mockHandler.Object)); + Task.Run(() => parallelDiscoveryManager.DiscoverTests(_discoveryCriteriaWith2Sources, _mockEventHandler.Object)); // Processed sources should be 1 since the 2nd source is never discovered - Assert.IsTrue(_discoveryCompleted.Wait(TaskTimeout), "Test discovery not completed."); + Assert.IsTrue(_discoveryCompleted.Wait(Timeout3Seconds), "Test discovery not completed."); Assert.AreEqual(1, _processedSources.Count, "All Sources must be processed."); } @@ -184,18 +230,19 @@ public void DiscoveryTestsShouldCatchExceptionAndHandleLogMessageOfError() { // Ensure that second discovery manager never starts. Expect 10 total tests. // Override DiscoveryComplete since overall aborted should be true - var parallelDiscoveryManager = SetupDiscoveryManager(_proxyManagerFunc, 2, false, totalTests: 10); - _createdMockManagers[1].Reset(); - _createdMockManagers[1].Setup(dm => dm.DiscoverTests(It.IsAny(), It.IsAny())) + var parallelDiscoveryManager = SetupDiscoveryManager(_createMockManager, 2, false, totalTests: 10); + var secondMockManager = _preCreatedMockManagers.ToArray()[1]; + secondMockManager.Reset(); + secondMockManager.Setup(dm => dm.DiscoverTests(It.IsAny(), It.IsAny())) .Throws(); - _mockHandler.Setup(mh => mh.HandleDiscoveryComplete(It.IsAny(), null)) + _mockEventHandler.Setup(mh => mh.HandleDiscoveryComplete(It.IsAny(), null)) .Callback>((t, l) => _discoveryCompleted.Set()); - Task.Run(() => parallelDiscoveryManager.DiscoverTests(_testDiscoveryCriteria, _mockHandler.Object)); + Task.Run(() => parallelDiscoveryManager.DiscoverTests(_discoveryCriteriaWith2Sources, _mockEventHandler.Object)); // Processed sources should be 1 since the 2nd source is never discovered - Assert.IsTrue(_discoveryCompleted.Wait(TaskTimeout), "Test discovery not completed."); - _mockHandler.Verify(s => s.HandleLogMessage(TestMessageLevel.Error, It.IsAny()), Times.Once); + Assert.IsTrue(_discoveryCompleted.Wait(Timeout3Seconds), "Test discovery not completed."); + _mockEventHandler.Verify(s => s.HandleLogMessage(TestMessageLevel.Error, It.IsAny()), Times.Once); } [TestMethod] @@ -203,39 +250,48 @@ public void DiscoveryTestsShouldCatchExceptionAndHandleRawMessageOfTestMessage() { // Ensure that second discovery manager never starts. Expect 10 total tests. // Override DiscoveryComplete since overall aborted should be true - var parallelDiscoveryManager = SetupDiscoveryManager(_proxyManagerFunc, 2, false, totalTests: 10); - _createdMockManagers[1].Reset(); - _createdMockManagers[1].Setup(dm => dm.DiscoverTests(It.IsAny(), It.IsAny())) + var parallelDiscoveryManager = SetupDiscoveryManager(_createMockManager, 2, false, totalTests: 10); + var secondMockManager = _preCreatedMockManagers.ToArray()[1]; + secondMockManager.Reset(); + secondMockManager.Setup(dm => dm.DiscoverTests(It.IsAny(), It.IsAny())) .Throws(); - _mockHandler.Setup(mh => mh.HandleDiscoveryComplete(It.IsAny(), null)) + _mockEventHandler.Setup(mh => mh.HandleDiscoveryComplete(It.IsAny(), null)) .Callback>((t, l) => _discoveryCompleted.Set()); - Task.Run(() => parallelDiscoveryManager.DiscoverTests(_testDiscoveryCriteria, _mockHandler.Object)); + Task.Run(() => parallelDiscoveryManager.DiscoverTests(_discoveryCriteriaWith2Sources, _mockEventHandler.Object)); // Processed sources should be 1 since the 2nd source is never discovered - Assert.IsTrue(_discoveryCompleted.Wait(TaskTimeout), "Test discovery not completed."); - _mockHandler.Verify(s => s.HandleRawMessage(It.Is(str => str.Contains(MessageType.TestMessage)))); + Assert.IsTrue(_discoveryCompleted.Wait(Timeout3Seconds), "Test discovery not completed."); + _mockEventHandler.Verify(s => s.HandleRawMessage(It.Is(str => str.Contains(MessageType.TestMessage)))); } [TestMethod] public void HandlePartialDiscoveryCompleteShouldCreateANewProxyDiscoveryManagerIfIsAbortedIsTrue() { - _proxyManagerFuncCalled = false; - var parallelDiscoveryManager = new ParallelProxyDiscoveryManager(_mockRequestData.Object, _proxyManagerFunc, 1, new List()); - var proxyDiscovermanager = new ProxyDiscoveryManager(_mockRequestData.Object, new Mock().Object, new Mock().Object); + var parallelDiscoveryManager = new ParallelProxyDiscoveryManager(_mockRequestData.Object, _createMockManager, 1, _runtimeProviders); + + // Trigger discover tests, this will create a manager by calling the _createMockManager func + // which dequeues it to _usedMockManagers. + parallelDiscoveryManager.DiscoverTests(_discoveryCriteriaWith2Sources, _mockEventHandler.Object); + var completedManager = _usedMockManagers[0]; - parallelDiscoveryManager.HandlePartialDiscoveryComplete(proxyDiscovermanager, 20, new List(), isAborted: true); + // act + // Tell the manager that completedManager finished work, and that it should progress to next work + parallelDiscoveryManager.HandlePartialDiscoveryComplete(completedManager.Object, 20, new List(), isAborted: true); - Assert.IsTrue(_proxyManagerFuncCalled); + // assert + // We created 2 managers 1 for the original work and another one + // when we called HandlePartialDiscoveryComplete and it moved on to the next piece of work. + Assert.AreEqual(2, _createMockManagerCalled); } private IParallelProxyDiscoveryManager SetupDiscoveryManager(Func getProxyManager, int parallelLevel, bool abortDiscovery, int totalTests = 20) { - var parallelDiscoveryManager = new ParallelProxyDiscoveryManager(_mockRequestData.Object, getProxyManager, parallelLevel, new List()); + var parallelDiscoveryManager = new ParallelProxyDiscoveryManager(_mockRequestData.Object, getProxyManager, parallelLevel, _runtimeProviders); SetupDiscoveryTests(_processedSources, abortDiscovery); // Setup a complete handler for parallel discovery manager - _mockHandler.Setup(mh => mh.HandleDiscoveryComplete(It.IsAny(), null)) + _mockEventHandler.Setup(mh => mh.HandleDiscoveryComplete(It.IsAny(), null)) .Callback>( (discoveryCompleteEventArgs, lastChunk) => _discoveryCompleted.Set()); @@ -245,7 +301,10 @@ private IParallelProxyDiscoveryManager SetupDiscoveryManager(Func processedSources, bool isAbort) { var syncObject = new object(); - foreach (var manager in _createdMockManagers) + // This setups callbacks for the handler the we pass through. + // We pick up those managers in the _createMockManager func, + // and return them. + foreach (var manager in _preCreatedMockManagers.ToArray()) { manager.Setup(m => m.DiscoverTests(It.IsAny(), It.IsAny())). Callback( @@ -258,7 +317,7 @@ private void SetupDiscoveryTests(List processedSources, bool isAbort) Task.Delay(100).Wait(); - Assert.AreEqual(_testDiscoveryCriteria.TestCaseFilter, criteria.TestCaseFilter); + Assert.AreEqual(_discoveryCriteriaWith2Sources.TestCaseFilter, criteria.TestCaseFilter); handler.HandleDiscoveryComplete(isAbort ? new DiscoveryCompleteEventArgs(-1, isAbort) : new DiscoveryCompleteEventArgs(10, isAbort), null); }); } @@ -287,15 +346,15 @@ private void AssertMissingAndDuplicateSources(List processedSources) } } - private void InvokeAndVerifyInitialize(int concurrentManagersCount, bool skipDefaultAdapters = false) + private void InvokeAndVerifyInitialize(int maxParallelLevel, bool skipDefaultAdapters = false) { - var parallelDiscoveryManager = new ParallelProxyDiscoveryManager(_mockRequestData.Object, _proxyManagerFunc, concurrentManagersCount, new List()); + var parallelDiscoveryManager = new ParallelProxyDiscoveryManager(_mockRequestData.Object, _createMockManager, maxParallelLevel, new List()); // Action parallelDiscoveryManager.Initialize(skipDefaultAdapters); // Verify - Assert.AreEqual(concurrentManagersCount, _createdMockManagers.Count, $"Number of Concurrent Managers created should be {concurrentManagersCount}"); - _createdMockManagers.ForEach(dm => dm.Verify(m => m.Initialize(skipDefaultAdapters), Times.Once)); + Assert.AreEqual(0, _usedMockManagers.Count, $"No managers are pre-created until there is work for them."); + _usedMockManagers.ForEach(dm => dm.Verify(m => m.Initialize(skipDefaultAdapters), Times.Once)); } } diff --git a/test/Microsoft.TestPlatform.CrossPlatEngine.UnitTests/Client/Parallel/ParallelProxyExecutionManagerTests.cs b/test/Microsoft.TestPlatform.CrossPlatEngine.UnitTests/Client/Parallel/ParallelProxyExecutionManagerTests.cs index 791091dec6..ac39fcc5fc 100644 --- a/test/Microsoft.TestPlatform.CrossPlatEngine.UnitTests/Client/Parallel/ParallelProxyExecutionManagerTests.cs +++ b/test/Microsoft.TestPlatform.CrossPlatEngine.UnitTests/Client/Parallel/ParallelProxyExecutionManagerTests.cs @@ -13,7 +13,6 @@ using Microsoft.VisualStudio.TestPlatform.CommunicationUtilities.ObjectModel; using Microsoft.VisualStudio.TestPlatform.CrossPlatEngine.Client; using Microsoft.VisualStudio.TestPlatform.CrossPlatEngine.Client.Parallel; -using Microsoft.VisualStudio.TestPlatform.CrossPlatEngine.DataCollection; using Microsoft.VisualStudio.TestPlatform.CrossPlatEngine.DataCollection.Interfaces; using Microsoft.VisualStudio.TestPlatform.ObjectModel; using Microsoft.VisualStudio.TestPlatform.ObjectModel.Client; @@ -31,11 +30,11 @@ namespace TestPlatform.CrossPlatEngine.UnitTests.Client; [TestClass] public class ParallelProxyExecutionManagerTests { - private static readonly int TaskTimeout = 15 * 1000; // In milliseconds + private static readonly int Timeout3Seconds = 3 * 1000; // In milliseconds - private readonly List> _createdMockManagers; - private Func _proxyManagerFunc; - private readonly Mock _mockHandler; + private readonly List> _usedMockManagers; + private readonly Func _createMockManager; + private readonly Mock _mockEvenHandler; private Mock _mockTestHostManager; private Mock _mockRequestSender; @@ -43,89 +42,112 @@ public class ParallelProxyExecutionManagerTests private Mock _mockDataCollectionManager; private readonly List _sources; private readonly List _processedSources; - private readonly TestRunCriteria _testRunCriteriaWithSources; + private readonly TestRunCriteria _testRunCriteriaWith2Sources; + private readonly List _runtimeProviders; private readonly List _testCases; private readonly List _processedTestCases; - private readonly TestRunCriteria _testRunCriteriaWithTests; + private readonly TestRunCriteria _testRunCriteriaWithTestsFrom3Dlls; - private bool _proxyManagerFuncCalled; + private int _createMockManagerCalled; private readonly ManualResetEventSlim _executionCompleted; + private readonly Queue> _preCreatedMockManagers; private readonly Mock _mockRequestData; public ParallelProxyExecutionManagerTests() { _executionCompleted = new ManualResetEventSlim(false); - _createdMockManagers = new List>(); - _proxyManagerFunc = _ => + _preCreatedMockManagers = new Queue>( + new List> + { + new Mock(), + new Mock(), + new Mock(), + new Mock(), + }); + _usedMockManagers = new List>(); + _createMockManager = _ => { - _proxyManagerFuncCalled = true; - var manager = new Mock(); - _createdMockManagers.Add(manager); + _createMockManagerCalled++; + var manager = _preCreatedMockManagers.Dequeue(); + _usedMockManagers.Add(manager); return manager.Object; }; - _mockHandler = new Mock(); + _mockEvenHandler = new Mock(); // Configure sources _sources = new List() { "1.dll", "2.dll" }; _processedSources = new List(); - _testRunCriteriaWithSources = new TestRunCriteria(_sources, 100, false, string.Empty, TimeSpan.MaxValue, null, "Name~Test", new FilterOptions() { FilterRegEx = @"^[^\s\(]+" }); + _testRunCriteriaWith2Sources = new TestRunCriteria(_sources, 100, false, string.Empty, TimeSpan.MaxValue, null, "Name~Test", new FilterOptions() { FilterRegEx = @"^[^\s\(]+" }); + _runtimeProviders = new List { + new TestRuntimeProviderInfo(typeof(ITestRuntimeProvider), false, "", new List + { + new SourceDetail{ Source = "1.dll", Architecture = Architecture.X86, Framework = Framework.DefaultFramework }, + new SourceDetail{ Source = "2.dll", Architecture = Architecture.X86, Framework = Framework.DefaultFramework }, + // For testcases on the bottom. + new SourceDetail{ Source = "3.dll", Architecture = Architecture.X86, Framework = Framework.DefaultFramework }, + }) + }; // Configure testcases _testCases = CreateTestCases(); _processedTestCases = new List(); - _testRunCriteriaWithTests = new TestRunCriteria(_testCases, 100); + _testRunCriteriaWithTestsFrom3Dlls = new TestRunCriteria(_testCases, 100); _mockRequestData = new Mock(); _mockRequestData.Setup(rd => rd.MetricsCollection).Returns(new NoOpMetricsCollection()); } [TestMethod] - public void InitializeShouldCallAllConcurrentManagersOnce() + public void NoManagersArePreCreatedUntilThereIsWorkForThem() { InvokeAndVerifyInitialize(3); } [TestMethod] - public void InitializeShouldCallAllConcurrentManagersWithFalseFlagIfSkipDefaultAdaptersIsFalse() + public void NoManagersArePreCreatedUntilThereIsWorkForThemButSkipDefaultAdaptersValueFalseIsKept() { - InvokeAndVerifyInitialize(3, false); + InvokeAndVerifyInitialize(3, skipDefaultAdapters: false); } [TestMethod] - public void InitializeShouldCallAllConcurrentManagersWithTrueFlagIfSkipDefaultAdaptersIsTrue() + public void NoManagersArePreCreatedUntilThereIsWorkForThemButSkipDefaultAdaptersValueTrueIsKept() { - InvokeAndVerifyInitialize(3, true); + InvokeAndVerifyInitialize(3, skipDefaultAdapters: true); } [TestMethod] public void AbortShouldCallAllConcurrentManagersOnce() { - var parallelExecutionManager = new ParallelProxyExecutionManager(_mockRequestData.Object, _proxyManagerFunc, 4, new List()); + var parallelExecutionManager = new ParallelProxyExecutionManager(_mockRequestData.Object, _createMockManager, parallelLevel: 1000, _runtimeProviders); + // Starting parallel run will create 2 proxy managers, which we will then promptly abort. + parallelExecutionManager.StartTestRun(_testRunCriteriaWith2Sources, new Mock().Object); parallelExecutionManager.Abort(It.IsAny()); - Assert.AreEqual(4, _createdMockManagers.Count, "Number of Concurrent Managers created should be 4"); - _createdMockManagers.ForEach(em => em.Verify(m => m.Abort(It.IsAny()), Times.Once)); + Assert.AreEqual(2, _usedMockManagers.Count, "Number of Concurrent Managers created should be equal to the amount of dlls that run"); + _usedMockManagers.ForEach(em => em.Verify(m => m.Abort(It.IsAny()), Times.Once)); } [TestMethod] public void CancelShouldCallAllConcurrentManagersOnce() { - var parallelExecutionManager = new ParallelProxyExecutionManager(_mockRequestData.Object, _proxyManagerFunc, 4, new List()); + var parallelExecutionManager = new ParallelProxyExecutionManager(_mockRequestData.Object, _createMockManager, 4, _runtimeProviders); + // Starting parallel run will create 2 proxy managers, which we will then promptly cancel. + parallelExecutionManager.StartTestRun(_testRunCriteriaWith2Sources, new Mock().Object); parallelExecutionManager.Cancel(It.IsAny()); - Assert.AreEqual(4, _createdMockManagers.Count, "Number of Concurrent Managers created should be 4"); - _createdMockManagers.ForEach(em => em.Verify(m => m.Cancel(It.IsAny()), Times.Once)); + Assert.AreEqual(2, _usedMockManagers.Count, "Number of Concurrent Managers created should be equal to the amount of dlls that run"); + _usedMockManagers.ForEach(em => em.Verify(m => m.Cancel(It.IsAny()), Times.Once)); } [TestMethod] public void StartTestRunShouldProcessAllSources() { - var parallelExecutionManager = SetupExecutionManager(_proxyManagerFunc, 2); + var parallelExecutionManager = SetupExecutionManager(_createMockManager, 2); - parallelExecutionManager.StartTestRun(_testRunCriteriaWithSources, _mockHandler.Object); + parallelExecutionManager.StartTestRun(_testRunCriteriaWith2Sources, _mockEvenHandler.Object); - Assert.IsTrue(_executionCompleted.Wait(TaskTimeout), "Test run not completed."); + Assert.IsTrue(_executionCompleted.Wait(Timeout3Seconds), "Test run not completed."); Assert.AreEqual(_sources.Count, _processedSources.Count, "All Sources must be processed."); AssertMissingAndDuplicateSources(_processedSources); } @@ -135,11 +157,11 @@ public void StartTestRunShouldProcessAllSources() [TestMethod] public void StartTestRunShouldProcessAllTestCases() { - var parallelExecutionManager = SetupExecutionManager(_proxyManagerFunc, 3, setupTestCases: true); + var parallelExecutionManager = SetupExecutionManager(_createMockManager, 3, setupTestCases: true); - parallelExecutionManager.StartTestRun(_testRunCriteriaWithTests, _mockHandler.Object); + parallelExecutionManager.StartTestRun(_testRunCriteriaWithTestsFrom3Dlls, _mockEvenHandler.Object); - Assert.IsTrue(_executionCompleted.Wait(TaskTimeout), "Test run not completed."); + Assert.IsTrue(_executionCompleted.Wait(Timeout3Seconds), "Test run not completed."); Assert.AreEqual(_testCases.Count, _processedTestCases.Count, "All Tests must be processed."); AssertMissingAndDuplicateTestCases(_testCases, _processedTestCases); } @@ -147,11 +169,11 @@ public void StartTestRunShouldProcessAllTestCases() [TestMethod] public void StartTestRunWithSourcesShouldNotSendCompleteUntilAllSourcesAreProcessed() { - var parallelExecutionManager = SetupExecutionManager(_proxyManagerFunc, 2); + var parallelExecutionManager = SetupExecutionManager(_createMockManager, 2); - Task.Run(() => parallelExecutionManager.StartTestRun(_testRunCriteriaWithSources, _mockHandler.Object)); + Task.Run(() => parallelExecutionManager.StartTestRun(_testRunCriteriaWith2Sources, _mockEvenHandler.Object)); - Assert.IsTrue(_executionCompleted.Wait(TaskTimeout), "Test run not completed."); + Assert.IsTrue(_executionCompleted.Wait(Timeout3Seconds), "Test run not completed."); Assert.AreEqual(_sources.Count, _processedSources.Count, "All Sources must be processed."); AssertMissingAndDuplicateSources(_processedSources); } @@ -159,43 +181,26 @@ public void StartTestRunWithSourcesShouldNotSendCompleteUntilAllSourcesAreProces [TestMethod] public void HandlePartialRunCompleteShouldCreateNewProxyExecutionManagerIfDataCollectionEnabled() { - var completeArgs = new TestRunCompleteEventArgs(null, true, true, null, null, null, TimeSpan.Zero); + var completeArgs = new TestRunCompleteEventArgs(null, isCanceled: false, isAborted: false, null, null, null, TimeSpan.Zero); _mockTestHostManager = new Mock(); _mockRequestSender = new Mock(); _mockDataCollectionManager = new Mock(); - var proxyDataCollectionManager = new ProxyExecutionManagerWithDataCollection(_mockRequestData.Object, _mockRequestSender.Object, _mockTestHostManager.Object, _mockDataCollectionManager.Object); - var parallelExecutionManager = SetupExecutionManager(_proxyManagerFunc, 2, setupTestCases: true); + var parallelExecutionManager = SetupExecutionManager(_createMockManager, 2, setupTestCases: true); - _proxyManagerFuncCalled = false; - parallelExecutionManager.HandlePartialRunComplete(proxyDataCollectionManager, completeArgs, null, null, null); - Assert.IsTrue(_proxyManagerFuncCalled); - } + // Trigger discover tests, this will create a manager by calling the _createMockManager func + // which dequeues it to _usedMockManagers. + parallelExecutionManager.StartTestRun(_testRunCriteriaWith2Sources, _mockEvenHandler.Object); + var completedManager = _usedMockManagers[0]; + + // act + // Tell the manager that completedManager finished work, and that it should progress to next work + parallelExecutionManager.HandlePartialRunComplete(completedManager.Object, completeArgs, null, null, null); - //[TestMethod] - //public void HandlePartialRunCompleteShouldCreateNewProxyExecutionManagerIfDataCollectionEnabledAndCreatorWithDataCollection() - //{ - // var completeArgs = new TestRunCompleteEventArgs(null, true, true, null, null, null, TimeSpan.Zero); - // _mockTestHostManager = new Mock(); - // _mockRequestSender = new Mock(); - // _mockDataCollectionManager = new Mock(); - // var proxyDataCollectionManager = new ProxyExecutionManagerWithDataCollection(_mockRequestData.Object, _mockRequestSender.Object, _mockTestHostManager.Object, _mockDataCollectionManager.Object); - // var managers = new List>(); - // _proxyManagerFunc = _ => - // { - // _proxyManagerFuncCalled = true; - // var manager = new Mock(_mockRequestData.Object, _mockRequestSender.Object, _mockTestHostManager.Object, _mockDataCollectionManager.Object); - // managers.Add(manager); - // return manager.Object; - // }; - // var parallelExecutionManager = SetupExecutionManager(_proxyManagerFunc, 2, setupTestCases: true); - - // _proxyManagerFuncCalled = false; - // parallelExecutionManager.HandlePartialRunComplete(proxyDataCollectionManager, completeArgs, null, null, null); - // Assert.IsTrue(_proxyManagerFuncCalled); - - // var handler = parallelExecutionManager.GetHandlerForGivenManager(managers.Last().Object); - // Assert.IsTrue(handler is ParallelDataCollectionEventsHandler); - //} + // assert + // We created 2 managers 1 for the original work and another one + // when we called HandlePartialDiscoveryComplete and it moved on to the next piece of work. + Assert.AreEqual(2, _createMockManagerCalled); + } [TestMethod] public void HandlePartialRunCompleteShouldCreateNewProxyExecutionManagerIfIsAbortedIsTrue() @@ -203,22 +208,40 @@ public void HandlePartialRunCompleteShouldCreateNewProxyExecutionManagerIfIsAbor var completeArgs = new TestRunCompleteEventArgs(null, true, true, null, null, null, TimeSpan.Zero); _mockTestHostManager = new Mock(); _mockRequestSender = new Mock(); - var parallelExecutionManager = SetupExecutionManager(_proxyManagerFunc, 2, setupTestCases: true); + var parallelExecutionManager = SetupExecutionManager(_createMockManager, 2, setupTestCases: true); - _proxyManagerFuncCalled = false; - var proxyExecutionManagerManager = new ProxyExecutionManager(_mockRequestData.Object, _mockRequestSender.Object, _mockTestHostManager.Object); - parallelExecutionManager.HandlePartialRunComplete(proxyExecutionManagerManager, completeArgs, null, null, null); - Assert.IsTrue(_proxyManagerFuncCalled); + // Trigger discover tests, this will create a manager by calling the _createMockManager func + // which dequeues it to _usedMockManagers. + parallelExecutionManager.StartTestRun(_testRunCriteriaWith2Sources, _mockEvenHandler.Object); + var completedManager = _usedMockManagers[0]; + + // act + // Tell the manager that completedManager finished work, and that it should progress to next work + parallelExecutionManager.HandlePartialRunComplete(completedManager.Object, completeArgs, null, null, null); + + // assert + // We created 2 managers 1 for the original work and another one + // when we called HandlePartialDiscoveryComplete and it moved on to the next piece of work. + Assert.AreEqual(2, _createMockManagerCalled); } [TestMethod] public void StartTestRunWithTestsShouldNotSendCompleteUntilAllTestsAreProcessed() { - var parallelExecutionManager = SetupExecutionManager(_proxyManagerFunc, 3, setupTestCases: true); + var parallelExecutionManager = SetupExecutionManager(_createMockManager, 3, setupTestCases: true); - Task.Run(() => parallelExecutionManager.StartTestRun(_testRunCriteriaWithTests, _mockHandler.Object)); + var task = Task.Run(() => parallelExecutionManager.StartTestRun(_testRunCriteriaWithTestsFrom3Dlls, _mockEvenHandler.Object)); - Assert.IsTrue(_executionCompleted.Wait(TaskTimeout), "Test run not completed."); + bool executionCompleted = _executionCompleted.Wait(Timeout3Seconds); + + if (task.IsCompleted) + { + // Receive any exception if some happened + // Don't await if not completed, to avoid hanging the test. + task.GetAwaiter().GetResult(); + } + + Assert.IsTrue(executionCompleted, "Test run not completed."); Assert.AreEqual(_testCases.Count, _processedTestCases.Count, "All Tests must be processed."); AssertMissingAndDuplicateTestCases(_testCases, _processedTestCases); } @@ -227,14 +250,14 @@ public void StartTestRunWithTestsShouldNotSendCompleteUntilAllTestsAreProcessed( public void StartTestRunShouldNotProcessAllSourcesOnExecutionCancelsForAnySource() { var executionManagerMock = new Mock(); - var parallelExecutionManager = new ParallelProxyExecutionManager(_mockRequestData.Object, _ => executionManagerMock.Object, 1, new List()); - _createdMockManagers.Add(executionManagerMock); + var parallelExecutionManager = new ParallelProxyExecutionManager(_mockRequestData.Object, _ => executionManagerMock.Object, 1, _runtimeProviders); + _preCreatedMockManagers.Enqueue(executionManagerMock); SetupMockManagers(_processedSources, isCanceled: true, isAborted: false); SetupHandleTestRunComplete(_executionCompleted); - Task.Run(() => parallelExecutionManager.StartTestRun(_testRunCriteriaWithSources, _mockHandler.Object)); + Task.Run(() => parallelExecutionManager.StartTestRun(_testRunCriteriaWith2Sources, _mockEvenHandler.Object)); - Assert.IsTrue(_executionCompleted.Wait(TaskTimeout), "Test run not completed."); + Assert.IsTrue(_executionCompleted.Wait(Timeout3Seconds), "Test run not completed."); Assert.AreEqual(1, _processedSources.Count, "Abort should stop all sources execution."); } @@ -242,15 +265,15 @@ public void StartTestRunShouldNotProcessAllSourcesOnExecutionCancelsForAnySource public void StartTestRunShouldNotProcessAllSourcesOnExecutionAborted() { var executionManagerMock = new Mock(); - var parallelExecutionManager = new ParallelProxyExecutionManager(_mockRequestData.Object, _ => executionManagerMock.Object, 1, new List()); - _createdMockManagers.Add(executionManagerMock); + var parallelExecutionManager = new ParallelProxyExecutionManager(_mockRequestData.Object, _ => executionManagerMock.Object, 1, _runtimeProviders); + _preCreatedMockManagers.Enqueue(executionManagerMock); SetupMockManagers(_processedSources, isCanceled: false, isAborted: false); SetupHandleTestRunComplete(_executionCompleted); parallelExecutionManager.Abort(It.IsAny()); - Task.Run(() => parallelExecutionManager.StartTestRun(_testRunCriteriaWithSources, _mockHandler.Object)); + Task.Run(() => parallelExecutionManager.StartTestRun(_testRunCriteriaWith2Sources, _mockEvenHandler.Object)); - Assert.IsTrue(_executionCompleted.Wait(TaskTimeout), "Test run not completed."); + Assert.IsTrue(_executionCompleted.Wait(Timeout3Seconds), "Test run not completed."); Assert.AreEqual(1, _processedSources.Count, "Abort should stop all sources execution."); } @@ -258,14 +281,14 @@ public void StartTestRunShouldNotProcessAllSourcesOnExecutionAborted() public void StartTestRunShouldProcessAllSourcesOnExecutionAbortsForAnySource() { var executionManagerMock = new Mock(); - var parallelExecutionManager = new ParallelProxyExecutionManager(_mockRequestData.Object, _ => executionManagerMock.Object, 1, new List()); - _createdMockManagers.Add(executionManagerMock); + var parallelExecutionManager = new ParallelProxyExecutionManager(_mockRequestData.Object, _ => executionManagerMock.Object, 1, _runtimeProviders); + _preCreatedMockManagers.Enqueue(executionManagerMock); SetupMockManagers(_processedSources, isCanceled: false, isAborted: true); SetupHandleTestRunComplete(_executionCompleted); - Task.Run(() => parallelExecutionManager.StartTestRun(_testRunCriteriaWithSources, _mockHandler.Object)); + Task.Run(() => parallelExecutionManager.StartTestRun(_testRunCriteriaWith2Sources, _mockEvenHandler.Object)); - Assert.IsTrue(_executionCompleted.Wait(TaskTimeout), "Test run not completed."); + Assert.IsTrue(_executionCompleted.Wait(Timeout3Seconds), "Test run not completed."); Assert.AreEqual(2, _processedSources.Count, "Abort should stop all sources execution."); } @@ -273,53 +296,56 @@ public void StartTestRunShouldProcessAllSourcesOnExecutionAbortsForAnySource() public void StartTestRunShouldProcessAllSourceIfOneDiscoveryManagerIsStarved() { // Ensure that second discovery manager never starts. Expect 10 total tests. - var parallelExecutionManager = SetupExecutionManager(_proxyManagerFunc, 2); - _createdMockManagers[1].Reset(); - _createdMockManagers[1].Setup(em => em.StartTestRun(It.IsAny(), It.IsAny())) + var parallelExecutionManager = SetupExecutionManager(_createMockManager, 2); + var mockManagers = _preCreatedMockManagers.ToArray(); + mockManagers[1].Reset(); + mockManagers[1].Setup(em => em.StartTestRun(It.IsAny(), It.IsAny())) .Throws(); - Task.Run(() => parallelExecutionManager.StartTestRun(_testRunCriteriaWithSources, _mockHandler.Object)); + Task.Run(() => parallelExecutionManager.StartTestRun(_testRunCriteriaWith2Sources, _mockEvenHandler.Object)); // Processed sources should be 1 since the 2nd source is never discovered - Assert.IsTrue(_executionCompleted.Wait(TaskTimeout), "Test run not completed."); + Assert.IsTrue(_executionCompleted.Wait(Timeout3Seconds), "Test run not completed."); Assert.AreEqual(1, _processedSources.Count, "All Sources must be processed."); } [TestMethod] public void StartTestRunShouldCatchExceptionAndHandleLogMessageOfError() { - var parallelExecutionManager = SetupExecutionManager(_proxyManagerFunc, 2); - _createdMockManagers[1].Reset(); - _createdMockManagers[1].Setup(em => em.StartTestRun(It.IsAny(), It.IsAny())) + var parallelExecutionManager = SetupExecutionManager(_createMockManager, 2); + var mockManagers = _preCreatedMockManagers.ToArray(); + mockManagers[1].Reset(); + mockManagers[1].Setup(em => em.StartTestRun(It.IsAny(), It.IsAny())) .Throws(); - Task.Run(() => parallelExecutionManager.StartTestRun(_testRunCriteriaWithSources, _mockHandler.Object)); + Task.Run(() => parallelExecutionManager.StartTestRun(_testRunCriteriaWith2Sources, _mockEvenHandler.Object)); - Assert.IsTrue(_executionCompleted.Wait(TaskTimeout), "Test run not completed."); - _mockHandler.Verify(s => s.HandleLogMessage(TestMessageLevel.Error, It.IsAny()), Times.Once); + Assert.IsTrue(_executionCompleted.Wait(Timeout3Seconds), "Test run not completed."); + _mockEvenHandler.Verify(s => s.HandleLogMessage(TestMessageLevel.Error, It.IsAny()), Times.Once); } [TestMethod] public void StartTestRunShouldCatchExceptionAndHandleRawMessageOfTestMessage() { - var parallelExecutionManager = SetupExecutionManager(_proxyManagerFunc, 2); - _createdMockManagers[1].Reset(); - _createdMockManagers[1].Setup(em => em.StartTestRun(It.IsAny(), It.IsAny())) + var parallelExecutionManager = SetupExecutionManager(_createMockManager, 2); + var mockManagers = _preCreatedMockManagers.ToArray(); + mockManagers[1].Reset(); + mockManagers[1].Setup(em => em.StartTestRun(It.IsAny(), It.IsAny())) .Throws(); - Task.Run(() => parallelExecutionManager.StartTestRun(_testRunCriteriaWithSources, _mockHandler.Object)); + Task.Run(() => parallelExecutionManager.StartTestRun(_testRunCriteriaWith2Sources, _mockEvenHandler.Object)); - Assert.IsTrue(_executionCompleted.Wait(TaskTimeout), "Test run not completed."); - _mockHandler.Verify(s => s.HandleRawMessage(It.Is(str => str.Contains(MessageType.TestMessage)))); + Assert.IsTrue(_executionCompleted.Wait(Timeout3Seconds), "Test run not completed."); + _mockEvenHandler.Verify(s => s.HandleRawMessage(It.Is(str => str.Contains(MessageType.TestMessage)))); } [TestMethod] public void StartTestRunShouldAggregateRunData() { - var parallelExecutionManager = new ParallelProxyExecutionManager(_mockRequestData.Object, _proxyManagerFunc, 2, new List()); + var parallelExecutionManager = new ParallelProxyExecutionManager(_mockRequestData.Object, _createMockManager, 2, _runtimeProviders); var syncObject = new object(); - foreach (var manager in _createdMockManagers) + foreach (var manager in _preCreatedMockManagers) { manager.Setup(m => m.StartTestRun(It.IsAny(), It.IsAny())). Callback( @@ -359,7 +385,7 @@ public void StartTestRunShouldAggregateRunData() } Exception assertException = null; - _mockHandler.Setup(m => m.HandleTestRunComplete( + _mockEvenHandler.Setup(m => m.HandleTestRunComplete( It.IsAny(), It.IsAny(), It.IsAny>(), @@ -396,9 +422,9 @@ public void StartTestRunShouldAggregateRunData() } }); - Task.Run(() => parallelExecutionManager.StartTestRun(_testRunCriteriaWithSources, _mockHandler.Object)); + Task.Run(() => parallelExecutionManager.StartTestRun(_testRunCriteriaWith2Sources, _mockEvenHandler.Object)); - Assert.IsTrue(_executionCompleted.Wait(TaskTimeout), "Test run not completed."); + Assert.IsTrue(_executionCompleted.Wait(Timeout3Seconds), "Test run not completed."); Assert.IsNull(assertException, assertException?.ToString()); Assert.AreEqual(_sources.Count, _processedSources.Count, "All Sources must be processed."); @@ -412,11 +438,11 @@ private ParallelProxyExecutionManager SetupExecutionManager(Func proxyManagerFunc, int parallelLevel, bool setupTestCases) { - var parallelExecutionManager = new ParallelProxyExecutionManager(_mockRequestData.Object, proxyManagerFunc, parallelLevel, new List()); + var parallelExecutionManager = new ParallelProxyExecutionManager(_mockRequestData.Object, proxyManagerFunc, parallelLevel, _runtimeProviders); if (setupTestCases) { - SetupMockManagersForTestCase(_processedTestCases, _testRunCriteriaWithTests); + SetupMockManagersForTestCase(_processedTestCases, _testRunCriteriaWithTestsFrom3Dlls); } else { @@ -429,7 +455,7 @@ private ParallelProxyExecutionManager SetupExecutionManager(Func mh.HandleTestRunComplete(It.IsAny(), + _mockEvenHandler.Setup(mh => mh.HandleTestRunComplete(It.IsAny(), It.IsAny(), It.IsAny>(), It.IsAny>())) @@ -496,7 +522,7 @@ private static void AssertMissingAndDuplicateTestCases(List tests, Lis private void SetupMockManagersForTestCase(List processedTestCases, TestRunCriteria testRunCriteria) { var syncObject = new object(); - foreach (var manager in _createdMockManagers) + foreach (var manager in _preCreatedMockManagers) { manager.Setup(m => m.StartTestRun(It.IsAny(), It.IsAny())). Callback( @@ -531,7 +557,7 @@ private static List CreateTestCases() private void SetupMockManagers(List processedSources, bool isCanceled = false, bool isAborted = false) { var syncObject = new object(); - foreach (var manager in _createdMockManagers) + foreach (var manager in _preCreatedMockManagers) { manager.Setup(m => m.StartTestRun(It.IsAny(), It.IsAny())). Callback( @@ -543,20 +569,18 @@ private void SetupMockManagers(List processedSources, bool isCanceled = } Task.Delay(100).Wait(); - // Duplicated testRunCriteria should match the actual one. - Assert.AreEqual(_testRunCriteriaWithSources, criteria, "Mismatch in testRunCriteria"); handler.HandleTestRunComplete(CreateTestRunCompleteArgs(isCanceled, isAborted), null, null, null); }); } } - private void InvokeAndVerifyInitialize(int concurrentManagersCount, bool skipDefaultAdapters = false) + private void InvokeAndVerifyInitialize(int parallelLevel, bool skipDefaultAdapters = false) { - var parallelExecutionManager = new ParallelProxyExecutionManager(_mockRequestData.Object, _proxyManagerFunc, concurrentManagersCount, new List()); + var parallelExecutionManager = new ParallelProxyExecutionManager(_mockRequestData.Object, _createMockManager, parallelLevel, _runtimeProviders); parallelExecutionManager.Initialize(skipDefaultAdapters); - Assert.AreEqual(concurrentManagersCount, _createdMockManagers.Count, $"Number of Concurrent Managers created should be {concurrentManagersCount}"); - _createdMockManagers.ForEach(em => em.Verify(m => m.Initialize(skipDefaultAdapters), Times.Once)); + Assert.AreEqual(0, _usedMockManagers.Count, $"No concurrent managers should be pre-created, until there is work for them"); + _usedMockManagers.ForEach(em => em.Verify(m => m.Initialize(skipDefaultAdapters), Times.Once)); } } diff --git a/test/Microsoft.TestPlatform.CrossPlatEngine.UnitTests/TestEngineTests.cs b/test/Microsoft.TestPlatform.CrossPlatEngine.UnitTests/TestEngineTests.cs index b34d05791a..4639dd4dc9 100644 --- a/test/Microsoft.TestPlatform.CrossPlatEngine.UnitTests/TestEngineTests.cs +++ b/test/Microsoft.TestPlatform.CrossPlatEngine.UnitTests/TestEngineTests.cs @@ -28,7 +28,7 @@ namespace TestPlatform.CrossPlatEngine.UnitTests; [TestClass] public class TestEngineTests { - private ITestEngine _testEngine; + private TestableTestEngine _testEngine; private readonly Mock _mockProcessHelper; private readonly ProtocolConfig _protocolConfig = new() { Version = 1 }; private readonly Mock _mockRequestData; @@ -352,28 +352,6 @@ public void GetExecutionManagerShouldReturnParallelExecutionManagerIfHostIsNotSh Assert.IsInstanceOfType(_testEngine.GetExecutionManager(_mockRequestData.Object, testRunCriteria, sourceToSourceDetailMap), typeof(ParallelProxyExecutionManager)); } - [TestMethod] - public void GetExecutionManagerShouldReturnExecutionManagerWithDataCollectionIfDataCollectionIsEnabled() - { - var settingXml = - @" - - - - - - "; - var testRunCriteria = new TestRunCriteria(new List { "1.dll" }, 100, false, settingXml); - var sourceToSourceDetailMap = new Dictionary - { - ["1.dll"] = new SourceDetail { Source = "1.dll", Architecture = Architecture.X86, Framework = Framework.DefaultFramework }, - }; - var result = _testEngine.GetExecutionManager(_mockRequestData.Object, testRunCriteria, sourceToSourceDetailMap); - - Assert.IsNotNull(result); - Assert.IsInstanceOfType(result, typeof(ProxyExecutionManagerWithDataCollection)); - } - [TestMethod] public void GetExecutionManagerShouldNotReturnInProcessProxyexecutionManagerIfInIsolationIsTrue() { @@ -580,56 +558,6 @@ public void GetExtensionManagerShouldCollectMetrics() _mockMetricsCollection.Verify(mc => mc.Add(TelemetryDataConstants.ParallelEnabledDuringExecution, It.IsAny()), Times.Once); } - [TestMethod] - public void ProxyDataCollectionManagerShouldBeInitialzedWithCorrectTestSourcesWhenTestRunCriteriaContainsSourceList() - { - var settingXml = - @" - - - - - - "; - - var testRunCriteria = new TestRunCriteria(new List { "1.dll", "2.dll" }, 100, false, settingXml); - var sourceToSourceDetailMap = new Dictionary - { - ["1.dll"] = new SourceDetail { Source = "1.dll", Architecture = Architecture.X86, Framework = Framework.DefaultFramework }, - ["2.dll"] = new SourceDetail { Source = "2.dll", Architecture = Architecture.X86, Framework = Framework.DefaultFramework }, - }; - - var executionManager = _testEngine.GetExecutionManager(_mockRequestData.Object, testRunCriteria, sourceToSourceDetailMap); - - Assert.IsInstanceOfType(executionManager, typeof(ProxyExecutionManagerWithDataCollection)); - Assert.IsTrue(((ProxyExecutionManagerWithDataCollection)executionManager).ProxyDataCollectionManager.Sources.Contains("1.dll")); - } - - [TestMethod] - public void ProxyDataCollectionManagerShouldBeInitialzedWithCorrectTestSourcesWhenTestRunCriteriaContainsTestCaseList() - { - var settingXml = - @" - - - - - - "; - - var testCaseList = new List { new TestCase("x.y.z", new Uri("uri://dummy"), "x.dll") }; - var testRunCriteria = new TestRunCriteria(testCaseList, 100, false, settingXml); - var sourceToSourceDetailMap = new Dictionary - { - ["x.dll"] = new SourceDetail { Source = "x.dll", Architecture = Architecture.X86, Framework = Framework.DefaultFramework }, - }; - - var executionManager = _testEngine.GetExecutionManager(_mockRequestData.Object, testRunCriteria, sourceToSourceDetailMap); - - Assert.IsInstanceOfType(executionManager, typeof(ProxyExecutionManagerWithDataCollection)); - Assert.IsTrue(((ProxyExecutionManagerWithDataCollection)executionManager).ProxyDataCollectionManager.Sources.Contains("x.dll")); - } - /// /// GetLoggerManager should return a non null instance. /// @@ -1093,4 +1021,55 @@ public void GetTestSessionManagerShouldReturnNotNullIfRunSettingsHasTestSettings Assert.IsNotNull(testSessionManager); } + + [TestMethod] + public void CreatingNonParallelExecutionManagerShouldReturnExecutionManagerWithDataCollectionIfDataCollectionIsEnabled() + { + var settingXml = + @" + + + + + + "; + var testRunCriteria = new TestRunCriteria(new List { "1.dll" }, 100, false, settingXml); + + var runtimeProviderInfo = new TestRuntimeProviderInfo(typeof(ITestRuntimeProvider), false, settingXml, + new List { new SourceDetail { Source = "1.dll", Architecture = Architecture.X86, Framework = Framework.DefaultFramework } }); + var nonParallelExecutionManager = _testEngine.CreateNonParallelExecutionManager(_mockRequestData.Object, testRunCriteria, true, runtimeProviderInfo); + + Assert.IsNotNull(nonParallelExecutionManager); + Assert.IsInstanceOfType(nonParallelExecutionManager, typeof(ProxyExecutionManagerWithDataCollection)); + } + + + [TestMethod] + public void CreatedNonParallelExecutionManagerShouldBeInitialzedWithCorrectTestSourcesWhenTestRunCriteriaContainsSourceList() + { + // Test run criteria are NOT used to get sources or settings + // those are taken from the runtimeProviderInfo, because we've split the + // test run criteria into smaller pieces to run them on each non-parallel execution manager. + var testRunCriteria = new TestRunCriteria(new List { "none.dll" }, 100, false, testSettings: null); + + var settingXml = + @" + + + + + + "; + + var runtimeProviderInfo = new TestRuntimeProviderInfo(typeof(ITestRuntimeProvider), false, settingXml, + new List { + new SourceDetail { Source = "1.dll", Architecture = Architecture.X86, Framework = Framework.DefaultFramework }, + new SourceDetail { Source = "2.dll", Architecture = Architecture.X86, Framework = Framework.DefaultFramework } + }); + var nonParallelExecutionManager = _testEngine.CreateNonParallelExecutionManager(_mockRequestData.Object, testRunCriteria, true, runtimeProviderInfo); + + Assert.IsInstanceOfType(nonParallelExecutionManager, typeof(ProxyExecutionManagerWithDataCollection)); + Assert.IsTrue(((ProxyExecutionManagerWithDataCollection)nonParallelExecutionManager).ProxyDataCollectionManager.Sources.Contains("1.dll")); + Assert.IsTrue(((ProxyExecutionManagerWithDataCollection)nonParallelExecutionManager).ProxyDataCollectionManager.Sources.Contains("2.dll")); + } } From 88ae4ad2bd69345a983c52ead155f0e923b9ca71 Mon Sep 17 00:00:00 2001 From: nohwnd Date: Wed, 2 Mar 2022 17:36:32 +0100 Subject: [PATCH 015/112] Unit tests fixed --- src/vstest.console/CommandLine/InferHelper.cs | 9 +- .../TestPlatformHelpers/TestRequestManager.cs | 5 + .../EventLogDataCollectorTests.cs | 1 - .../Utilities/MetadataReaderHelperTests.cs | 2 +- .../CommandLine/InferHelperTests.cs | 19 ++- .../TestRequestManagerTests.cs | 125 ++++++++++-------- 6 files changed, 94 insertions(+), 67 deletions(-) diff --git a/src/vstest.console/CommandLine/InferHelper.cs b/src/vstest.console/CommandLine/InferHelper.cs index 482e1afab1..08a4e5347d 100644 --- a/src/vstest.console/CommandLine/InferHelper.cs +++ b/src/vstest.console/CommandLine/InferHelper.cs @@ -35,7 +35,13 @@ public Architecture AutoDetectArchitecture(IList sources, Architecture d // Set the default for all sources. foreach (var source in sources) { - sourceToPlatformMap.Add(source, defaultArchitecture); + // TODO: Add default architecture to runtime providers info, or something and that will allow us to have test + // cases without any sources. Otherwise change test AutoDetectArchitectureShouldReturnDefaultArchitectureOnNullItemInSources + // because this condition is making that test happy. + if (source != null) + { + sourceToPlatformMap.Add(source, defaultArchitecture); + } } try @@ -70,6 +76,7 @@ public Architecture AutoDetectArchitecture(IList sources, Architecture d // This source has no associated architecture so it does not help use determine a common architecture for // all the sources, so we continue to next one. + sourceToPlatformMap[source] = defaultArchitecture; continue; } diff --git a/src/vstest.console/TestPlatformHelpers/TestRequestManager.cs b/src/vstest.console/TestPlatformHelpers/TestRequestManager.cs index ea751aaca6..f292e92254 100644 --- a/src/vstest.console/TestPlatformHelpers/TestRequestManager.cs +++ b/src/vstest.console/TestPlatformHelpers/TestRequestManager.cs @@ -158,6 +158,8 @@ public void DiscoverTests( // TODO: Normalize rest of the data on the request as well discoveryPayload.Sources = discoveryPayload.Sources?.Distinct().ToList() ?? new List(); + discoveryPayload.RunSettings ??= ""; + var runsettings = discoveryPayload.RunSettings; if (discoveryPayload.TestPlatformOptions != null) @@ -454,6 +456,9 @@ public void StartTestSession( _telemetryOptedIn = payload.TestPlatformOptions.CollectMetrics; } + payload.Sources ??= new List(); + payload.RunSettings ??= ""; + if (UpdateRunSettingsIfRequired( payload.RunSettings, payload.Sources, diff --git a/test/DataCollectors/Microsoft.TestPlatform.Extensions.EventLogCollector.UnitTests/EventLogDataCollectorTests.cs b/test/DataCollectors/Microsoft.TestPlatform.Extensions.EventLogCollector.UnitTests/EventLogDataCollectorTests.cs index 3096009d27..30cf5a9fbf 100644 --- a/test/DataCollectors/Microsoft.TestPlatform.Extensions.EventLogCollector.UnitTests/EventLogDataCollectorTests.cs +++ b/test/DataCollectors/Microsoft.TestPlatform.Extensions.EventLogCollector.UnitTests/EventLogDataCollectorTests.cs @@ -217,7 +217,6 @@ public void InitializeShouldSubscribeToDataCollectionEvents() { var testableDataCollectionEvents = new TestableDataCollectionEvents(); _eventLogDataCollector.Initialize(null, testableDataCollectionEvents, _mockDataCollectionSink, _mockDataCollectionLogger.Object, _dataCollectionEnvironmentContext); - Assert.AreEqual(1, testableDataCollectionEvents.GetTestHostLaunchedInvocationList().Length); Assert.AreEqual(1, testableDataCollectionEvents.GetTestCaseStartInvocationList().Length); Assert.AreEqual(1, testableDataCollectionEvents.GetTestCaseEndInvocationList().Length); Assert.AreEqual(1, testableDataCollectionEvents.GetTestSessionEndInvocationList().Length); diff --git a/test/Microsoft.TestPlatform.Common.UnitTests/Utilities/MetadataReaderHelperTests.cs b/test/Microsoft.TestPlatform.Common.UnitTests/Utilities/MetadataReaderHelperTests.cs index b7cc7d837a..61cfae3fef 100644 --- a/test/Microsoft.TestPlatform.Common.UnitTests/Utilities/MetadataReaderHelperTests.cs +++ b/test/Microsoft.TestPlatform.Common.UnitTests/Utilities/MetadataReaderHelperTests.cs @@ -22,7 +22,7 @@ public class MetadataReaderHelperTests public void MetadataReaderHelper_GetCollectorExtensionTypes() { string testAssetsPath = GetTestAssetsFolder(); - var dataCollectorFilePath = Directory.GetFiles(testAssetsPath, "AttachmentProcessorDataCollector.dll", SearchOption.AllDirectories).Where(x => x.Contains("bin")).Single(); + var dataCollectorFilePath = Directory.GetFiles(testAssetsPath, "AttachmentProcessorDataCollector.dll", SearchOption.AllDirectories).Where(x => x.Contains("bin")).First(); var types = _metadataReaderHelper.DiscoverTestExtensionTypesV2Attribute(Assembly.LoadFile(dataCollectorFilePath), dataCollectorFilePath); Assert.IsTrue(types.Any(), $"File {dataCollectorFilePath}"); Assert.IsTrue(types[0].AssemblyQualifiedName.StartsWith("AttachmentProcessorDataCollector.SampleDataCollectorV2"), $"File {dataCollectorFilePath}"); diff --git a/test/vstest.console.UnitTests/CommandLine/InferHelperTests.cs b/test/vstest.console.UnitTests/CommandLine/InferHelperTests.cs index b69f909f09..b150b04f30 100644 --- a/test/vstest.console.UnitTests/CommandLine/InferHelperTests.cs +++ b/test/vstest.console.UnitTests/CommandLine/InferHelperTests.cs @@ -27,7 +27,6 @@ public class InferHelperTests private readonly Framework _frameworkNet47 = Framework.FromString(".NETFramework,Version=4.7"); private readonly Framework _frameworkCore10 = Framework.FromString(".NETCoreApp,Version=1.0"); private readonly Framework _frameworkCore11 = Framework.FromString(".NETCoreApp,Version=1.1"); - private readonly IDictionary _sourceFrameworks; public InferHelperTests() { @@ -75,10 +74,10 @@ public void AutoDetectArchitectureShouldReturnCorrectDefaultArchForNotDotNetAsse } [TestMethod] - public void AutoDetectArchitectureShouldSetAnyCpuArchForNotDotNetAssembly() + public void AutoDetectArchitectureShouldSetDefaultArchForNotDotNetAssembly() { _inferHelper.AutoDetectArchitecture(new List() { "NotDotNetAssebly.appx" }, _defaultArchitecture, out var sourceArchitectures); - Assert.AreEqual(Architecture.AnyCPU, sourceArchitectures["NotDotNetAssebly.appx"]); + Assert.AreEqual(_defaultArchitecture, sourceArchitectures["NotDotNetAssebly.appx"]); } [TestMethod] @@ -113,7 +112,7 @@ public void AutoDetectArchitectureShouldReturnX64ArchIfOneX64AssemblyAndRestAnyC { _mockAssemblyHelper.SetupSequence(ah => ah.GetArchitecture(It.IsAny())) .Returns(Architecture.AnyCPU).Returns(Architecture.AnyCPU).Returns(Architecture.X64); - Assert.AreEqual(Architecture.X64, _inferHelper.AutoDetectArchitecture(new List() { "x64.dll", "AnyCPU2.exe", "x64.dll" }, _defaultArchitecture, out _)); + Assert.AreEqual(Architecture.X64, _inferHelper.AutoDetectArchitecture(new List() { "x64.dll", "AnyCPU2.exe", "x64-2.dll" }, _defaultArchitecture, out _)); _mockAssemblyHelper.Verify(ah => ah.GetArchitecture(It.IsAny()), Times.Exactly(3)); } @@ -134,7 +133,7 @@ public void AutoDetectArchitectureShouldPoulateSourceArchitectureDictionary() Assert.AreEqual(_defaultArchitecture, _inferHelper.AutoDetectArchitecture(new List() { "AnyCPU1.dll", "x64.exe", "x86.dll" }, _defaultArchitecture, out var sourceArchitectures)); Assert.AreEqual(3, sourceArchitectures.Count); - Assert.AreEqual(Architecture.AnyCPU, sourceArchitectures["AnyCPU1.dll"]); + Assert.AreEqual(_defaultArchitecture, sourceArchitectures["AnyCPU1.dll"]); Assert.AreEqual(Architecture.X64, sourceArchitectures["x64.exe"]); Assert.AreEqual(Architecture.X86, sourceArchitectures["x86.dll"]); @@ -241,12 +240,12 @@ public void AutoDetectFrameworkShouldPopulatetheDictionaryForAllTheSources() .Returns(new FrameworkName(_frameworkNet47.Name)) .Returns(new FrameworkName(_frameworkNet45.Name)); - Assert.AreEqual(_frameworkNet47.Name, _inferHelper.AutoDetectFramework(new List() { "net46.dll", "net47.exe", "net45.dll" }, out _).Name); + Assert.AreEqual(_frameworkNet47.Name, _inferHelper.AutoDetectFramework(new List() { "net46.dll", "net47.exe", "net45.dll" }, out var sourceFrameworks).Name); - Assert.AreEqual(3, _sourceFrameworks.Count); - Assert.AreEqual(_frameworkNet46.Name, _sourceFrameworks["net46.dll"].Name); - Assert.AreEqual(_frameworkNet47.Name, _sourceFrameworks["net47.exe"].Name); - Assert.AreEqual(_frameworkNet45.Name, _sourceFrameworks["net45.dll"].Name); + Assert.AreEqual(3, sourceFrameworks.Count); + Assert.AreEqual(_frameworkNet46.Name, sourceFrameworks["net46.dll"].Name); + Assert.AreEqual(_frameworkNet47.Name, sourceFrameworks["net47.exe"].Name); + Assert.AreEqual(_frameworkNet45.Name, sourceFrameworks["net45.dll"].Name); _mockAssemblyHelper.Verify(ah => ah.GetFrameWork(It.IsAny()), Times.Exactly(3)); } diff --git a/test/vstest.console.UnitTests/TestPlatformHelpers/TestRequestManagerTests.cs b/test/vstest.console.UnitTests/TestPlatformHelpers/TestRequestManagerTests.cs index fbf21103a4..169e67f267 100644 --- a/test/vstest.console.UnitTests/TestPlatformHelpers/TestRequestManagerTests.cs +++ b/test/vstest.console.UnitTests/TestPlatformHelpers/TestRequestManagerTests.cs @@ -166,7 +166,7 @@ public void DiscoverTestsShouldReadTheBatchSizeFromSettingsAndSetItForDiscoveryC DiscoveryCriteria actualDiscoveryCriteria = null; var mockDiscoveryRequest = new Mock(); _mockTestPlatform.Setup(mt => mt.CreateDiscoveryRequest(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny>())).Callback( - (IRequestData requestData, DiscoveryCriteria discoveryCriteria, TestPlatformOptions options) => actualDiscoveryCriteria = discoveryCriteria).Returns(mockDiscoveryRequest.Object); + (IRequestData requestData, DiscoveryCriteria discoveryCriteria, TestPlatformOptions options, Dictionary sourceToSourceDetailMap) => actualDiscoveryCriteria = discoveryCriteria).Returns(mockDiscoveryRequest.Object); _testRequestManager.DiscoverTests(payload, new Mock().Object, _protocolConfig); Assert.AreEqual(15, actualDiscoveryCriteria.FrequencyOfDiscoveredTestsEvent); @@ -185,7 +185,7 @@ public void DiscoverTestsShouldCallTestPlatformAndSucceed() DiscoveryCriteria actualDiscoveryCriteria = null; var mockDiscoveryRequest = new Mock(); _mockTestPlatform.Setup(mt => mt.CreateDiscoveryRequest(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny>())).Callback( - (IRequestData requestData, DiscoveryCriteria discoveryCriteria, TestPlatformOptions options) => + (IRequestData requestData, DiscoveryCriteria discoveryCriteria, TestPlatformOptions options, Dictionary sourceToSourceDetailMap) => { createDiscoveryRequestCalled++; actualDiscoveryCriteria = discoveryCriteria; @@ -239,7 +239,7 @@ public void DiscoverTestsShouldPassSameProtocolConfigInRequestData() IRequestData actualRequestData = null; var mockDiscoveryRequest = new Mock(); _mockTestPlatform.Setup(mt => mt.CreateDiscoveryRequest(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny>())).Callback( - (IRequestData requestData, DiscoveryCriteria discoveryCriteria, TestPlatformOptions options) => actualRequestData = requestData).Returns(mockDiscoveryRequest.Object); + (IRequestData requestData, DiscoveryCriteria discoveryCriteria, TestPlatformOptions options, Dictionary sourceToSourceDetailMap) => actualRequestData = requestData).Returns(mockDiscoveryRequest.Object); var mockDiscoveryRegistrar = new Mock(); @@ -290,7 +290,7 @@ public void DiscoverTestsShouldCollectMetrics() IRequestData actualRequestData = null; var mockDiscoveryRequest = new Mock(); _mockTestPlatform.Setup(mt => mt.CreateDiscoveryRequest(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny>())).Callback( - (IRequestData requestData, DiscoveryCriteria discoveryCriteria, TestPlatformOptions options) => actualRequestData = requestData).Returns(mockDiscoveryRequest.Object); + (IRequestData requestData, DiscoveryCriteria discoveryCriteria, TestPlatformOptions options, Dictionary sourceToSourceDetailMap) => actualRequestData = requestData).Returns(mockDiscoveryRequest.Object); _testRequestManager = new TestRequestManager( CommandLineOptions.Instance, @@ -339,7 +339,7 @@ public void DiscoverTestsShouldCollectTargetDeviceLocalMachineIfTargetDeviceStri IRequestData actualRequestData = null; var mockDiscoveryRequest = new Mock(); _mockTestPlatform.Setup(mt => mt.CreateDiscoveryRequest(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny>())).Callback( - (IRequestData requestData, DiscoveryCriteria discoveryCriteria, TestPlatformOptions options) => actualRequestData = requestData).Returns(mockDiscoveryRequest.Object); + (IRequestData requestData, DiscoveryCriteria discoveryCriteria, TestPlatformOptions options, Dictionary sourceToSourceDetailMap) => actualRequestData = requestData).Returns(mockDiscoveryRequest.Object); _testRequestManager = new TestRequestManager( CommandLineOptions.Instance, @@ -382,7 +382,7 @@ public void DiscoverTestsShouldCollectTargetDeviceIfTargetDeviceIsDevice() IRequestData actualRequestData = null; var mockDiscoveryRequest = new Mock(); _mockTestPlatform.Setup(mt => mt.CreateDiscoveryRequest(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny>())).Callback( - (IRequestData requestData, DiscoveryCriteria discoveryCriteria, TestPlatformOptions options) => actualRequestData = requestData).Returns(mockDiscoveryRequest.Object); + (IRequestData requestData, DiscoveryCriteria discoveryCriteria, TestPlatformOptions options, Dictionary sourceToSourceDetailMap) => actualRequestData = requestData).Returns(mockDiscoveryRequest.Object); _testRequestManager = new TestRequestManager( CommandLineOptions.Instance, @@ -425,7 +425,7 @@ public void DiscoverTestsShouldCollectTargetDeviceIfTargetDeviceIsEmulator() IRequestData actualRequestData = null; var mockDiscoveryRequest = new Mock(); _mockTestPlatform.Setup(mt => mt.CreateDiscoveryRequest(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny>())).Callback( - (IRequestData requestData, DiscoveryCriteria discoveryCriteria, TestPlatformOptions options) => actualRequestData = requestData).Returns(mockDiscoveryRequest.Object); + (IRequestData requestData, DiscoveryCriteria discoveryCriteria, TestPlatformOptions options, Dictionary sourceToSourceDetailMap) => actualRequestData = requestData).Returns(mockDiscoveryRequest.Object); _testRequestManager = new TestRequestManager( CommandLineOptions.Instance, @@ -468,7 +468,7 @@ public void DiscoverTestsShouldCollectCommands() IRequestData actualRequestData = null; var mockDiscoveryRequest = new Mock(); _mockTestPlatform.Setup(mt => mt.CreateDiscoveryRequest(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny>())).Callback( - (IRequestData requestData, DiscoveryCriteria discoveryCriteria, TestPlatformOptions options) => actualRequestData = requestData).Returns(mockDiscoveryRequest.Object); + (IRequestData requestData, DiscoveryCriteria discoveryCriteria, TestPlatformOptions options, Dictionary sourceToSourceDetailMap) => actualRequestData = requestData).Returns(mockDiscoveryRequest.Object); _testRequestManager = new TestRequestManager( CommandLineOptions.Instance, @@ -523,7 +523,7 @@ public void DiscoverTestsShouldCollectTestSettings() IRequestData actualRequestData = null; var mockDiscoveryRequest = new Mock(); _mockTestPlatform.Setup(mt => mt.CreateDiscoveryRequest(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny>())).Callback( - (IRequestData requestData, DiscoveryCriteria discoveryCriteria, TestPlatformOptions options) => actualRequestData = requestData).Returns(mockDiscoveryRequest.Object); + (IRequestData requestData, DiscoveryCriteria discoveryCriteria, TestPlatformOptions options, Dictionary sourceToSourceDetailMap) => actualRequestData = requestData).Returns(mockDiscoveryRequest.Object); _testRequestManager = new TestRequestManager( CommandLineOptions.Instance, @@ -570,7 +570,7 @@ public void DiscoverTestsShouldCollectVsmdiFile() IRequestData actualRequestData = null; var mockDiscoveryRequest = new Mock(); _mockTestPlatform.Setup(mt => mt.CreateDiscoveryRequest(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny>())).Callback( - (IRequestData requestData, DiscoveryCriteria discoveryCriteria, TestPlatformOptions options) => actualRequestData = requestData).Returns(mockDiscoveryRequest.Object); + (IRequestData requestData, DiscoveryCriteria discoveryCriteria, TestPlatformOptions options, Dictionary sourceToSourceDetailMap) => actualRequestData = requestData).Returns(mockDiscoveryRequest.Object); _testRequestManager = new TestRequestManager( CommandLineOptions.Instance, @@ -617,7 +617,7 @@ public void DiscoverTestsShouldCollectTestRunConfigFile() IRequestData actualRequestData = null; var mockDiscoveryRequest = new Mock(); _mockTestPlatform.Setup(mt => mt.CreateDiscoveryRequest(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny>())).Callback( - (IRequestData requestData, DiscoveryCriteria discoveryCriteria, TestPlatformOptions options) => actualRequestData = requestData).Returns(mockDiscoveryRequest.Object); + (IRequestData requestData, DiscoveryCriteria discoveryCriteria, TestPlatformOptions options, Dictionary sourceToSourceDetailMap) => actualRequestData = requestData).Returns(mockDiscoveryRequest.Object); _testRequestManager = new TestRequestManager( CommandLineOptions.Instance, @@ -663,7 +663,7 @@ public void DiscoverTestsShouldUpdateFrameworkAndPlatformIfNotSpecifiedInDesignM DiscoveryCriteria actualDiscoveryCriteria = null; var mockDiscoveryRequest = new Mock(); _mockTestPlatform.Setup(mt => mt.CreateDiscoveryRequest(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny>())).Callback( - (IRequestData requestData, DiscoveryCriteria discoveryCriteria, TestPlatformOptions options) => actualDiscoveryCriteria = discoveryCriteria).Returns(mockDiscoveryRequest.Object); + (IRequestData requestData, DiscoveryCriteria discoveryCriteria, TestPlatformOptions options, Dictionary sourceToSourceDetailMap) => actualDiscoveryCriteria = discoveryCriteria).Returns(mockDiscoveryRequest.Object); _testRequestManager.DiscoverTests(payload, new Mock().Object, _protocolConfig); @@ -697,7 +697,7 @@ public void DiscoverTestsShouldNotUpdateFrameworkAndPlatformIfSpecifiedInDesignM DiscoveryCriteria actualDiscoveryCriteria = null; var mockDiscoveryRequest = new Mock(); _mockTestPlatform.Setup(mt => mt.CreateDiscoveryRequest(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny>())).Callback( - (IRequestData requestData, DiscoveryCriteria discoveryCriteria, TestPlatformOptions options) => actualDiscoveryCriteria = discoveryCriteria).Returns(mockDiscoveryRequest.Object); + (IRequestData requestData, DiscoveryCriteria discoveryCriteria, TestPlatformOptions options, Dictionary sourceToSourceDetailMap) => actualDiscoveryCriteria = discoveryCriteria).Returns(mockDiscoveryRequest.Object); _testRequestManager.DiscoverTests(payload, new Mock().Object, _protocolConfig); @@ -729,7 +729,7 @@ public void DiscoverTestsShouldUpdateFrameworkAndPlatformInCommandLineScenariosI DiscoveryCriteria actualDiscoveryCriteria = null; var mockDiscoveryRequest = new Mock(); _mockTestPlatform.Setup(mt => mt.CreateDiscoveryRequest(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny>())).Callback( - (IRequestData requestData, DiscoveryCriteria discoveryCriteria, TestPlatformOptions options) => actualDiscoveryCriteria = discoveryCriteria).Returns(mockDiscoveryRequest.Object); + (IRequestData requestData, DiscoveryCriteria discoveryCriteria, TestPlatformOptions options, Dictionary sourceToSourceDetailMap) => actualDiscoveryCriteria = discoveryCriteria).Returns(mockDiscoveryRequest.Object); _testRequestManager.DiscoverTests(payload, new Mock().Object, _protocolConfig); _mockAssemblyMetadataProvider.Verify(a => a.GetArchitecture(It.IsAny())); @@ -740,7 +740,7 @@ public void DiscoverTestsShouldUpdateFrameworkAndPlatformInCommandLineScenariosI } [TestMethod] - public void DiscoverTestsShouldNotUpdateFrameworkAndPlatformInCommandLineScenariosIfSpecifiedButInferred() + public void DiscoverTestsShouldNotInferAndUpdateFrameworkAndPlatformInCommandLineScenariosIfSpecified() { var payload = new DiscoveryRequestPayload() { @@ -753,6 +753,8 @@ public void DiscoverTestsShouldNotUpdateFrameworkAndPlatformInCommandLineScenari " }; _commandLineOptions.IsDesignMode = false; + + // specified architecture _commandLineOptions.TargetFrameworkVersion = Framework.DefaultFramework; _commandLineOptions.TargetArchitecture = Architecture.X86; _mockAssemblyMetadataProvider.Setup(a => a.GetArchitecture(It.IsAny())) @@ -764,13 +766,16 @@ public void DiscoverTestsShouldNotUpdateFrameworkAndPlatformInCommandLineScenari _mockTestPlatform .Setup(mt => mt.CreateDiscoveryRequest(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny>())) .Callback( - (IRequestData requestData, DiscoveryCriteria discoveryCriteria, TestPlatformOptions options) => actualDiscoveryCriteria = discoveryCriteria).Returns(mockDiscoveryRequest.Object); + (IRequestData requestData, DiscoveryCriteria discoveryCriteria, TestPlatformOptions options, Dictionary sourceToSourceDetailMap) => actualDiscoveryCriteria = discoveryCriteria).Returns(mockDiscoveryRequest.Object); _testRequestManager.DiscoverTests(payload, new Mock().Object, _protocolConfig); - _mockAssemblyMetadataProvider.Verify(a => a.GetArchitecture(It.IsAny()), Times.Once); - _mockAssemblyMetadataProvider.Verify(a => a.GetFrameWork(It.IsAny()), Times.Once); + // we don't infer the settings + _mockAssemblyMetadataProvider.Verify(a => a.GetArchitecture(It.IsAny()), Times.Never); + _mockAssemblyMetadataProvider.Verify(a => a.GetFrameWork(It.IsAny()), Times.Never); + + // we don't update the settings Assert.IsFalse(actualDiscoveryCriteria.RunSettings.Contains(Constants.DotNetFramework46)); Assert.IsFalse(actualDiscoveryCriteria.RunSettings.Contains(nameof(Architecture.ARM))); } @@ -842,7 +847,7 @@ public void RunTestsShouldReadTheBatchSizeFromSettingsAndSetItForTestRunCriteria TestRunCriteria actualTestRunCriteria = null; var mockDiscoveryRequest = new Mock(); _mockTestPlatform.Setup(mt => mt.CreateTestRunRequest(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny>())).Callback( - (IRequestData requestData, TestRunCriteria runCriteria, TestPlatformOptions options) => actualTestRunCriteria = runCriteria).Returns(mockDiscoveryRequest.Object); + (IRequestData requestData, TestRunCriteria runCriteria, TestPlatformOptions options, Dictionary sourceToSourceDetailMap) => actualTestRunCriteria = runCriteria).Returns(mockDiscoveryRequest.Object); _testRequestManager.RunTests(payload, new Mock().Object, new Mock().Object, _protocolConfig); Assert.AreEqual(15, actualTestRunCriteria.FrequencyOfRunStatsChangeEvent); @@ -866,7 +871,7 @@ public void RunTestsShouldNotThrowForFramework35() TestRunCriteria actualTestRunCriteria = null; var mockDiscoveryRequest = new Mock(); _mockTestPlatform.Setup(mt => mt.CreateTestRunRequest(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny>())).Callback( - (IRequestData requestData, TestRunCriteria runCriteria, TestPlatformOptions options) => actualTestRunCriteria = runCriteria).Returns(mockDiscoveryRequest.Object); + (IRequestData requestData, TestRunCriteria runCriteria, TestPlatformOptions options, Dictionary sourceToSourceDetailMap) => actualTestRunCriteria = runCriteria).Returns(mockDiscoveryRequest.Object); _mockAssemblyMetadataProvider.Setup(a => a.GetFrameWork(It.IsAny())).Returns(new FrameworkName(Constants.DotNetFramework35)); var mockRunEventsRegistrar = new Mock(); @@ -891,7 +896,7 @@ public void RunTestsShouldPassSameProtocolConfigInRequestData() IRequestData actualRequestData = null; var mockDiscoveryRequest = new Mock(); _mockTestPlatform.Setup(mt => mt.CreateTestRunRequest(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny>())).Callback( - (IRequestData requestData, TestRunCriteria runCriteria, TestPlatformOptions options) => actualRequestData = requestData).Returns(mockDiscoveryRequest.Object); + (IRequestData requestData, TestRunCriteria runCriteria, TestPlatformOptions options, Dictionary sourceToSourceDetailMap) => actualRequestData = requestData).Returns(mockDiscoveryRequest.Object); // Act. _testRequestManager.RunTests(payload, new Mock().Object, new Mock().Object, mockProtocolConfig); @@ -915,7 +920,7 @@ public void RunTestsShouldCollectCommands() IRequestData actualRequestData = null; var mockDiscoveryRequest = new Mock(); _mockTestPlatform.Setup(mt => mt.CreateTestRunRequest(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny>())).Callback( - (IRequestData requestData, TestRunCriteria runCriteria, TestPlatformOptions options) => actualRequestData = requestData).Returns(mockDiscoveryRequest.Object); + (IRequestData requestData, TestRunCriteria runCriteria, TestPlatformOptions options, Dictionary sourceToSourceDetailMap) => actualRequestData = requestData).Returns(mockDiscoveryRequest.Object); _testRequestManager = new TestRequestManager( CommandLineOptions.Instance, @@ -979,7 +984,7 @@ public void RunTestsShouldCollectTelemetryForLegacySettings() IRequestData actualRequestData = null; var mockDiscoveryRequest = new Mock(); _mockTestPlatform.Setup(mt => mt.CreateTestRunRequest(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny>())).Callback( - (IRequestData requestData, TestRunCriteria runCriteria, TestPlatformOptions options) => actualRequestData = requestData).Returns(mockDiscoveryRequest.Object); + (IRequestData requestData, TestRunCriteria runCriteria, TestPlatformOptions options, Dictionary sourceToSourceDetailMap) => actualRequestData = requestData).Returns(mockDiscoveryRequest.Object); _testRequestManager = new TestRequestManager( CommandLineOptions.Instance, @@ -1026,7 +1031,7 @@ public void RunTestsShouldCollectTelemetryForTestSettingsEmbeddedInsideRunSettin IRequestData actualRequestData = null; var mockDiscoveryRequest = new Mock(); _mockTestPlatform.Setup(mt => mt.CreateTestRunRequest(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny>())).Callback( - (IRequestData requestData, TestRunCriteria runCriteria, TestPlatformOptions options) => actualRequestData = requestData).Returns(mockDiscoveryRequest.Object); + (IRequestData requestData, TestRunCriteria runCriteria, TestPlatformOptions options, Dictionary sourceToSourceDetailMap) => actualRequestData = requestData).Returns(mockDiscoveryRequest.Object); _testRequestManager = new TestRequestManager( CommandLineOptions.Instance, @@ -1071,7 +1076,7 @@ public void RunTestsShouldCollectMetrics() IRequestData actualRequestData = null; var mockDiscoveryRequest = new Mock(); _mockTestPlatform.Setup(mt => mt.CreateTestRunRequest(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny>())).Callback( - (IRequestData requestData, TestRunCriteria runCriteria, TestPlatformOptions options) => actualRequestData = requestData).Returns(mockDiscoveryRequest.Object); + (IRequestData requestData, TestRunCriteria runCriteria, TestPlatformOptions options, Dictionary sourceToSourceDetailMap) => actualRequestData = requestData).Returns(mockDiscoveryRequest.Object); _testRequestManager = new TestRequestManager( CommandLineOptions.Instance, @@ -1110,7 +1115,7 @@ public void RunTestsWithSourcesShouldCallTestPlatformAndSucceed() TestRunCriteria observedCriteria = null; var mockRunRequest = new Mock(); _mockTestPlatform.Setup(mt => mt.CreateTestRunRequest(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny>())).Callback( - (IRequestData requestData, TestRunCriteria runCriteria, TestPlatformOptions options) => + (IRequestData requestData, TestRunCriteria runCriteria, TestPlatformOptions options, Dictionary sourceToSourceDetailMap) => { createRunRequestCalled++; observedCriteria = runCriteria; @@ -1383,7 +1388,7 @@ public void RunTestsShouldShouldUpdateFrameworkAndPlatformIfNotSpecifiedInDesign TestRunCriteria actualTestRunCriteria = null; var mockTestRunRequest = new Mock(); _mockTestPlatform.Setup(mt => mt.CreateTestRunRequest(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny>())).Callback( - (IRequestData requestData, TestRunCriteria runCriteria, TestPlatformOptions options) => actualTestRunCriteria = runCriteria).Returns(mockTestRunRequest.Object); + (IRequestData requestData, TestRunCriteria runCriteria, TestPlatformOptions options, Dictionary sourceToSourceDetailMap) => actualTestRunCriteria = runCriteria).Returns(mockTestRunRequest.Object); _testRequestManager.RunTests(payload, new Mock().Object, new Mock().Object, _protocolConfig); @@ -1396,11 +1401,12 @@ public void RunTestsShouldShouldUpdateFrameworkAndPlatformIfNotSpecifiedInDesign } [TestMethod] - public void RunTestsShouldNotUpdateFrameworkAndPlatformIfSpecifiedInDesignModeButInferred() + public void RunTestsShouldNotUpdateFrameworkAndPlatformIfSpecifiedInDesignMode() { var payload = new TestRunRequestPayload() { Sources = new List() { "a.dll" }, + // specify architecture and framework RunSettings = $@" @@ -1419,13 +1425,15 @@ public void RunTestsShouldNotUpdateFrameworkAndPlatformIfSpecifiedInDesignModeBu TestRunCriteria actualTestRunCriteria = null; var mockTestRunRequest = new Mock(); _mockTestPlatform.Setup(mt => mt.CreateTestRunRequest(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny>())).Callback( - (IRequestData requestData, TestRunCriteria runCriteria, TestPlatformOptions options) => actualTestRunCriteria = runCriteria).Returns(mockTestRunRequest.Object); + (IRequestData requestData, TestRunCriteria runCriteria, TestPlatformOptions options, Dictionary sourceToSourceDetailMap) => actualTestRunCriteria = runCriteria).Returns(mockTestRunRequest.Object); _testRequestManager.RunTests(payload, new Mock().Object, new Mock().Object, _protocolConfig); - _mockAssemblyMetadataProvider.Verify(a => a.GetArchitecture(It.IsAny()), Times.Once); - _mockAssemblyMetadataProvider.Verify(a => a.GetFrameWork(It.IsAny()), Times.Once); + // don't infer them + _mockAssemblyMetadataProvider.Verify(a => a.GetArchitecture(It.IsAny()), Times.Never); + _mockAssemblyMetadataProvider.Verify(a => a.GetFrameWork(It.IsAny()), Times.Never); + // don't update them Assert.IsTrue(actualTestRunCriteria.TestRunSettings.Contains(Constants.DotNetFramework46)); Assert.IsTrue(actualTestRunCriteria.TestRunSettings.Contains(nameof(Architecture.ARM))); } @@ -1435,11 +1443,12 @@ public void RunTestsShouldNotUpdateFrameworkAndPlatformIfSpecifiedInDesignModeBu [DataRow("X86")] [DataRow("ARM")] [DataRow("aRm")] - public void RunTestsShouldNotUpdatePlatformIfSpecifiedInDesignModeButInferred(string targetPlatform) + public void RunTestsShouldNotUpdatePlatformIfSpecifiedInDesignMode(string targetPlatform) { var payload = new TestRunRequestPayload() { Sources = new List() { "a.dll" }, + // Specify platform RunSettings = $@" @@ -1457,13 +1466,16 @@ public void RunTestsShouldNotUpdatePlatformIfSpecifiedInDesignModeButInferred(st TestRunCriteria actualTestRunCriteria = null; var mockTestRunRequest = new Mock(); _mockTestPlatform.Setup(mt => mt.CreateTestRunRequest(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny>())).Callback( - (IRequestData requestData, TestRunCriteria runCriteria, TestPlatformOptions options) => actualTestRunCriteria = runCriteria).Returns(mockTestRunRequest.Object); + (IRequestData requestData, TestRunCriteria runCriteria, TestPlatformOptions options, Dictionary sourceToSourceDetailMap) => actualTestRunCriteria = runCriteria).Returns(mockTestRunRequest.Object); _testRequestManager.RunTests(payload, new Mock().Object, new Mock().Object, _protocolConfig); - _mockAssemblyMetadataProvider.Verify(a => a.GetArchitecture(It.IsAny()), Times.Once); + // don't infer it + _mockAssemblyMetadataProvider.Verify(a => a.GetArchitecture(It.IsAny()), Times.Never); + // but infer framework _mockAssemblyMetadataProvider.Verify(a => a.GetFrameWork(It.IsAny()), Times.Once); + // don't update it Assert.IsTrue(actualTestRunCriteria.TestRunSettings.Contains(targetPlatform)); } @@ -1489,7 +1501,7 @@ public void RunTestsShouldUpdateFrameworkAndPlatformInCommandLineScenarios() TestRunCriteria actualTestRunCriteria = null; var mockTestRunRequest = new Mock(); _mockTestPlatform.Setup(mt => mt.CreateTestRunRequest(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny>())).Callback( - (IRequestData requestData, TestRunCriteria runCriteria, TestPlatformOptions options) => actualTestRunCriteria = runCriteria).Returns(mockTestRunRequest.Object); + (IRequestData requestData, TestRunCriteria runCriteria, TestPlatformOptions options, Dictionary sourceToSourceDetailMap) => actualTestRunCriteria = runCriteria).Returns(mockTestRunRequest.Object); _testRequestManager.RunTests(payload, new Mock().Object, new Mock().Object, _protocolConfig); @@ -1501,7 +1513,7 @@ public void RunTestsShouldUpdateFrameworkAndPlatformInCommandLineScenarios() } [TestMethod] - public void RunTestsShouldNotpdateFrameworkAndPlatformInCommandLineScenariosIfSpecifiedButInferred() + public void RunTestsShouldNotpdateFrameworkAndPlatformInRunsettingsIfSpecifiedByCommandLine() { var payload = new TestRunRequestPayload() { @@ -1515,8 +1527,11 @@ public void RunTestsShouldNotpdateFrameworkAndPlatformInCommandLineScenariosIfSp }; _commandLineOptions.IsDesignMode = false; + + // specify architecture and framework _commandLineOptions.TargetArchitecture = Architecture.X86; _commandLineOptions.TargetFrameworkVersion = Framework.DefaultFramework; + _mockAssemblyMetadataProvider.Setup(a => a.GetArchitecture(It.IsAny())) .Returns(Architecture.ARM); _mockAssemblyMetadataProvider.Setup(a => a.GetFrameWork(It.IsAny())) @@ -1524,13 +1539,15 @@ public void RunTestsShouldNotpdateFrameworkAndPlatformInCommandLineScenariosIfSp TestRunCriteria actualTestRunCriteria = null; var mockTestRunRequest = new Mock(); _mockTestPlatform.Setup(mt => mt.CreateTestRunRequest(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny>())).Callback( - (IRequestData requestData, TestRunCriteria runCriteria, TestPlatformOptions options) => actualTestRunCriteria = runCriteria).Returns(mockTestRunRequest.Object); + (IRequestData requestData, TestRunCriteria runCriteria, TestPlatformOptions options, Dictionary sourceToSourceDetailMap) => actualTestRunCriteria = runCriteria).Returns(mockTestRunRequest.Object); _testRequestManager.RunTests(payload, new Mock().Object, new Mock().Object, _protocolConfig); - _mockAssemblyMetadataProvider.Verify(a => a.GetArchitecture(It.IsAny()), Times.Once); - _mockAssemblyMetadataProvider.Verify(a => a.GetFrameWork(It.IsAny()), Times.Once); + // don't infer them + _mockAssemblyMetadataProvider.Verify(a => a.GetArchitecture(It.IsAny()), Times.Never); + _mockAssemblyMetadataProvider.Verify(a => a.GetFrameWork(It.IsAny()), Times.Never); + // don't update them Assert.IsFalse(actualTestRunCriteria.TestRunSettings.Contains(Constants.DotNetFramework46)); Assert.IsFalse(actualTestRunCriteria.TestRunSettings.Contains(nameof(Architecture.ARM))); } @@ -1564,7 +1581,7 @@ public void RunTestsWithTestCasesShouldUpdateFrameworkAndPlatformIfNotSpecifiedI TestRunCriteria actualTestRunCriteria = null; var mockTestRunRequest = new Mock(); _mockTestPlatform.Setup(mt => mt.CreateTestRunRequest(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny>())).Callback( - (IRequestData requestData, TestRunCriteria runCriteria, TestPlatformOptions options) => actualTestRunCriteria = runCriteria).Returns(mockTestRunRequest.Object); + (IRequestData requestData, TestRunCriteria runCriteria, TestPlatformOptions options, Dictionary sourceToSourceDetailMap) => actualTestRunCriteria = runCriteria).Returns(mockTestRunRequest.Object); _testRequestManager.RunTests(payload, new Mock().Object, new Mock().Object, _protocolConfig); @@ -1699,7 +1716,7 @@ public void RunTestsShouldAddConsoleLoggerInRunSettingsInNonDesignMode() TestRunCriteria actualTestRunCriteria = null; var mockTestRunRequest = new Mock(); _mockTestPlatform.Setup(mt => mt.CreateTestRunRequest(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny>())).Callback( - (IRequestData requestData, TestRunCriteria runCriteria, TestPlatformOptions options) => actualTestRunCriteria = runCriteria).Returns(mockTestRunRequest.Object); + (IRequestData requestData, TestRunCriteria runCriteria, TestPlatformOptions options, Dictionary sourceToSourceDetailMap) => actualTestRunCriteria = runCriteria).Returns(mockTestRunRequest.Object); _testRequestManager.RunTests(payload, new Mock().Object, new Mock().Object, _protocolConfig); @@ -1738,7 +1755,7 @@ public void RunTestsShouldAddConsoleLoggerInRunSettingsIfDesignModeSetFalseInRun TestRunCriteria actualTestRunCriteria = null; var mockTestRunRequest = new Mock(); _mockTestPlatform.Setup(mt => mt.CreateTestRunRequest(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny>())).Callback( - (IRequestData requestData, TestRunCriteria runCriteria, TestPlatformOptions options) => actualTestRunCriteria = runCriteria).Returns(mockTestRunRequest.Object); + (IRequestData requestData, TestRunCriteria runCriteria, TestPlatformOptions options, Dictionary sourceToSourceDetailMap) => actualTestRunCriteria = runCriteria).Returns(mockTestRunRequest.Object); _testRequestManager.RunTests(payload, new Mock().Object, new Mock().Object, _protocolConfig); var loggerSettingsList = XmlRunSettingsUtilities.GetLoggerRunSettings(actualTestRunCriteria.TestRunSettings).LoggerSettingsList; @@ -1779,7 +1796,7 @@ public void DiscoverTestsShouldAddConsoleLoggerInRunSettingsIfDesignModeSetFalse _mockTestPlatform .Setup(mt => mt.CreateDiscoveryRequest(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny>())) .Callback( - (IRequestData requestData, DiscoveryCriteria discoveryCriteria, TestPlatformOptions options) => actualDiscoveryCriteria = discoveryCriteria).Returns(mockDiscoveryRequest.Object); + (IRequestData requestData, DiscoveryCriteria discoveryCriteria, TestPlatformOptions options, Dictionary sourceToSourceDetailMap) => actualDiscoveryCriteria = discoveryCriteria).Returns(mockDiscoveryRequest.Object); _testRequestManager.DiscoverTests(payload, new Mock().Object, _protocolConfig); @@ -1812,7 +1829,7 @@ public void RunTestsShouldNotAddConsoleLoggerInRunSettingsInDesignMode() TestRunCriteria actualTestRunCriteria = null; var mockTestRunRequest = new Mock(); _mockTestPlatform.Setup(mt => mt.CreateTestRunRequest(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny>())).Callback( - (IRequestData requestData, TestRunCriteria runCriteria, TestPlatformOptions options) => actualTestRunCriteria = runCriteria).Returns(mockTestRunRequest.Object); + (IRequestData requestData, TestRunCriteria runCriteria, TestPlatformOptions options, Dictionary sourceToSourceDetailMap) => actualTestRunCriteria = runCriteria).Returns(mockTestRunRequest.Object); _testRequestManager.RunTests(payload, new Mock().Object, new Mock().Object, _protocolConfig); Assert.IsFalse(actualTestRunCriteria.TestRunSettings.Contains("LoggerRunSettings")); @@ -1837,7 +1854,7 @@ public void DiscoverTestsShouldAddConsoleLoggerInRunSettingsInNonDesignMode() _mockTestPlatform .Setup(mt => mt.CreateDiscoveryRequest(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny>())) .Callback( - (IRequestData requestData, DiscoveryCriteria discoveryCriteria, TestPlatformOptions options) => actualDiscoveryCriteria = discoveryCriteria).Returns(mockDiscoveryRequest.Object); + (IRequestData requestData, DiscoveryCriteria discoveryCriteria, TestPlatformOptions options, Dictionary sourceToSourceDetailMap) => actualDiscoveryCriteria = discoveryCriteria).Returns(mockDiscoveryRequest.Object); _testRequestManager.DiscoverTests(payload, new Mock().Object, _protocolConfig); @@ -1869,7 +1886,7 @@ public void DiscoverTestsShouldNotAddConsoleLoggerInRunSettingsInDesignMode() _mockTestPlatform .Setup(mt => mt.CreateDiscoveryRequest(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny>())) .Callback( - (IRequestData requestData, DiscoveryCriteria discoveryCriteria, TestPlatformOptions options) => actualDiscoveryCriteria = discoveryCriteria).Returns(mockDiscoveryRequest.Object); + (IRequestData requestData, DiscoveryCriteria discoveryCriteria, TestPlatformOptions options, Dictionary sourceToSourceDetailMap) => actualDiscoveryCriteria = discoveryCriteria).Returns(mockDiscoveryRequest.Object); _testRequestManager.DiscoverTests(payload, new Mock().Object, _protocolConfig); @@ -1910,7 +1927,7 @@ public void RunTestsShouldOverrideOnlyAssemblyNameIfConsoleLoggerAlreadyPresentI TestRunCriteria actualTestRunCriteria = null; var mockTestRunRequest = new Mock(); _mockTestPlatform.Setup(mt => mt.CreateTestRunRequest(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny>())).Callback( - (IRequestData requestData, TestRunCriteria runCriteria, TestPlatformOptions options) => actualTestRunCriteria = runCriteria).Returns(mockTestRunRequest.Object); + (IRequestData requestData, TestRunCriteria runCriteria, TestPlatformOptions options, Dictionary sourceToSourceDetailMap) => actualTestRunCriteria = runCriteria).Returns(mockTestRunRequest.Object); _testRequestManager.RunTests(payload, new Mock().Object, new Mock().Object, _protocolConfig); var loggerSettingsList = XmlRunSettingsUtilities.GetLoggerRunSettings(actualTestRunCriteria.TestRunSettings).LoggerSettingsList; @@ -1960,7 +1977,7 @@ public void DiscoverTestsShouldOverrideOnlyAssemblyNameIfConsoleLoggerAlreadyPre _mockTestPlatform .Setup(mt => mt.CreateDiscoveryRequest(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny>())) .Callback( - (IRequestData requestData, DiscoveryCriteria discoveryCriteria, TestPlatformOptions options) => actualDiscoveryCriteria = discoveryCriteria).Returns(mockDiscoveryRequest.Object); + (IRequestData requestData, DiscoveryCriteria discoveryCriteria, TestPlatformOptions options, Dictionary sourceToSourceDetailMap) => actualDiscoveryCriteria = discoveryCriteria).Returns(mockDiscoveryRequest.Object); _testRequestManager.DiscoverTests(payload, new Mock().Object, _protocolConfig); @@ -2011,7 +2028,7 @@ public void RunTestsShouldOverrideOnlyAssemblyNameIfConsoleLoggerAlreadyPresentI TestRunCriteria actualTestRunCriteria = null; var mockTestRunRequest = new Mock(); _mockTestPlatform.Setup(mt => mt.CreateTestRunRequest(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny>())).Callback( - (IRequestData requestData, TestRunCriteria runCriteria, TestPlatformOptions options) => actualTestRunCriteria = runCriteria).Returns(mockTestRunRequest.Object); + (IRequestData requestData, TestRunCriteria runCriteria, TestPlatformOptions options, Dictionary sourceToSourceDetailMap) => actualTestRunCriteria = runCriteria).Returns(mockTestRunRequest.Object); _testRequestManager.RunTests(payload, new Mock().Object, new Mock().Object, _protocolConfig); var loggerSettingsList = XmlRunSettingsUtilities.GetLoggerRunSettings(actualTestRunCriteria.TestRunSettings).LoggerSettingsList; @@ -2061,7 +2078,7 @@ public void DiscoverTestsShouldOverrideOnlyAssemblyNameIfConsoleLoggerAlreadyPre _mockTestPlatform .Setup(mt => mt.CreateDiscoveryRequest(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny>())) .Callback( - (IRequestData requestData, DiscoveryCriteria discoveryCriteria, TestPlatformOptions options) => actualDiscoveryCriteria = discoveryCriteria).Returns(mockDiscoveryRequest.Object); + (IRequestData requestData, DiscoveryCriteria discoveryCriteria, TestPlatformOptions options, Dictionary sourceToSourceDetailMap) => actualDiscoveryCriteria = discoveryCriteria).Returns(mockDiscoveryRequest.Object); _testRequestManager.DiscoverTests(payload, new Mock().Object, _protocolConfig); @@ -2192,7 +2209,7 @@ public void StartTestSessionShouldPassCorrectTelemetryOptedInOptionToTestPlatfor It.IsAny>())) .Returns(true) .Callback( - (IRequestData rd, StartTestSessionCriteria _, ITestSessionEventsHandler _) => Assert.IsTrue(rd.IsTelemetryOptedIn)); + (IRequestData rd, StartTestSessionCriteria _, ITestSessionEventsHandler _, Dictionary _) => Assert.IsTrue(rd.IsTelemetryOptedIn)); Environment.SetEnvironmentVariable("VSTEST_TELEMETRY_OPTEDIN", "1"); @@ -2239,7 +2256,7 @@ public void StartTestSessionShouldUpdateSettings() It.IsAny>())) .Returns(true) .Callback( - (IRequestData _, StartTestSessionCriteria criteria, ITestSessionEventsHandler _) => + (IRequestData _, StartTestSessionCriteria criteria, ITestSessionEventsHandler _, Dictionary _) => { Assert.IsTrue(criteria.RunSettings.Contains(Constants.DotNetFramework46)); Assert.IsTrue(criteria.RunSettings.Contains(nameof(Architecture.ARM))); @@ -2500,7 +2517,7 @@ private void RunTestsIfThrowsExceptionShouldThrowOut(Exception exception) TestRunCriteria observedCriteria = null; var mockRunRequest = new Mock(); _mockTestPlatform.Setup(mt => mt.CreateTestRunRequest(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny>())).Callback( - (IRequestData requestData, TestRunCriteria runCriteria, TestPlatformOptions options) => + (IRequestData requestData, TestRunCriteria runCriteria, TestPlatformOptions options, Dictionary sourceToSourceDetailMap) => { createRunRequestCalled++; observedCriteria = runCriteria; @@ -2525,7 +2542,7 @@ private void DiscoverTestsIfThrowsExceptionShouldThrowOut(Exception exception) DiscoveryCriteria observedCriteria = null; var mockDiscoveryRequest = new Mock(); _mockTestPlatform.Setup(mt => mt.CreateDiscoveryRequest(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny>())).Callback( - (IRequestData requestData, DiscoveryCriteria discoveryCriteria, TestPlatformOptions options) => observedCriteria = discoveryCriteria).Returns(mockDiscoveryRequest.Object); + (IRequestData requestData, DiscoveryCriteria discoveryCriteria, TestPlatformOptions options, Dictionary sourceToSourceDetailMap) => observedCriteria = discoveryCriteria).Returns(mockDiscoveryRequest.Object); mockDiscoveryRequest.Setup(mr => mr.DiscoverAsync()).Throws(exception); From d7c059c0784d18274ce8d9b41830f924858ca878 Mon Sep 17 00:00:00 2001 From: nohwnd Date: Wed, 2 Mar 2022 18:51:43 +0100 Subject: [PATCH 016/112] Build --- playground/MSTest1/UnitTest1.cs | 3 --- scripts/build/TestPlatform.targets | 2 +- .../Interfaces/Engine/ClientProtocol/ITestEngine.cs | 2 -- .../PublicAPI/PublicAPI.Shipped.txt | 6 ------ .../PublicAPI/PublicAPI.Unshipped.txt | 4 ++++ .../PublicAPI/PublicAPI.Shipped.txt | 4 ---- .../common/IO/PlatformStream.cs | 4 ++-- .../common/System/ProcessHelper.cs | 4 ++-- .../common/Tracing/RollingFileTraceListener.cs | 4 ++-- .../ConsoleParameters.cs | 7 ------- .../PublicAPI/PublicAPI.Shipped.txt | 4 +++- .../PublicAPI/net451/PublicAPI.Shipped.txt | 2 -- .../VsTestConsoleProcessManager.cs | 5 ++--- .../Client/Parallel/ParallelOperationManagerTests.cs | 4 ++-- .../IntegrationTestBase.cs | 6 +++++- test/vstest.ProgrammerTests/BasicRunAndDiscovery.cs | 2 ++ .../Processors/EnvironmentArgumentProcessorTests.cs | 1 - 17 files changed, 25 insertions(+), 39 deletions(-) diff --git a/playground/MSTest1/UnitTest1.cs b/playground/MSTest1/UnitTest1.cs index a3c4b47de8..8be51be9be 100644 --- a/playground/MSTest1/UnitTest1.cs +++ b/playground/MSTest1/UnitTest1.cs @@ -6,9 +6,6 @@ #nullable disable namespace MSTest1; - -using Microsoft.VisualStudio.TestTools.UnitTesting; - [TestClass] public class UnitTest1 { diff --git a/scripts/build/TestPlatform.targets b/scripts/build/TestPlatform.targets index df4eb0ec58..737c067d2e 100644 --- a/scripts/build/TestPlatform.targets +++ b/scripts/build/TestPlatform.targets @@ -3,7 +3,7 @@ $(MSBuildThisFileDirectory)..\..\ false - $(NoWarn);CA1416 + $(NoWarn);CA1416;RS0037 diff --git a/src/Microsoft.TestPlatform.Common/Interfaces/Engine/ClientProtocol/ITestEngine.cs b/src/Microsoft.TestPlatform.Common/Interfaces/Engine/ClientProtocol/ITestEngine.cs index 2596880db2..1505e7c059 100644 --- a/src/Microsoft.TestPlatform.Common/Interfaces/Engine/ClientProtocol/ITestEngine.cs +++ b/src/Microsoft.TestPlatform.Common/Interfaces/Engine/ClientProtocol/ITestEngine.cs @@ -5,8 +5,6 @@ using Microsoft.VisualStudio.TestPlatform.ObjectModel.Client; -using Microsoft.VisualStudio.TestPlatform.ObjectModel.Host; - #nullable disable namespace Microsoft.VisualStudio.TestPlatform.ObjectModel.Engine; diff --git a/src/Microsoft.TestPlatform.Common/PublicAPI/PublicAPI.Shipped.txt b/src/Microsoft.TestPlatform.Common/PublicAPI/PublicAPI.Shipped.txt index bbb15c327f..c5be285b45 100644 --- a/src/Microsoft.TestPlatform.Common/PublicAPI/PublicAPI.Shipped.txt +++ b/src/Microsoft.TestPlatform.Common/PublicAPI/PublicAPI.Shipped.txt @@ -175,8 +175,6 @@ Microsoft.VisualStudio.TestPlatform.ObjectModel.Engine.ClientProtocol.TestExecut Microsoft.VisualStudio.TestPlatform.ObjectModel.Engine.ClientProtocol.TestExecutionContext.TestExecutionContext(long frequencyOfRunStatsChangeEvent, System.TimeSpan runStatsChangeEventTimeout, bool inIsolation, bool keepAlive, bool isDataCollectionEnabled, bool areTestCaseLevelEventsRequired, bool hasTestRun, bool isDebug, string testCaseFilter, Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.FilterOptions filterOptions) -> void Microsoft.VisualStudio.TestPlatform.ObjectModel.Engine.ClientProtocol.TestExecutionContext.TestRunConfiguration.get -> Microsoft.VisualStudio.TestPlatform.ObjectModel.RunConfiguration Microsoft.VisualStudio.TestPlatform.ObjectModel.Engine.ClientProtocol.TestExecutionContext.TestRunConfiguration.set -> void -Microsoft.VisualStudio.TestPlatform.ObjectModel.Engine.IParallelOperationManager -Microsoft.VisualStudio.TestPlatform.ObjectModel.Engine.IParallelOperationManager.UpdateParallelLevel(int parallelLevel) -> void Microsoft.VisualStudio.TestPlatform.ObjectModel.Engine.IParallelProxyDiscoveryManager Microsoft.VisualStudio.TestPlatform.ObjectModel.Engine.IParallelProxyDiscoveryManager.HandlePartialDiscoveryComplete(Microsoft.VisualStudio.TestPlatform.ObjectModel.Engine.IProxyDiscoveryManager proxyDiscoveryManager, long totalTests, System.Collections.Generic.IEnumerable lastChunk, bool isAborted) -> bool Microsoft.VisualStudio.TestPlatform.ObjectModel.Engine.IParallelProxyExecutionManager @@ -201,11 +199,8 @@ Microsoft.VisualStudio.TestPlatform.ObjectModel.Engine.ITestCaseEventsHandler.Se Microsoft.VisualStudio.TestPlatform.ObjectModel.Engine.ITestCaseEventsHandler.SendTestCaseStart(Microsoft.VisualStudio.TestPlatform.ObjectModel.TestCase testCase) -> void Microsoft.VisualStudio.TestPlatform.ObjectModel.Engine.ITestCaseEventsHandler.SendTestResult(Microsoft.VisualStudio.TestPlatform.ObjectModel.TestResult result) -> void Microsoft.VisualStudio.TestPlatform.ObjectModel.Engine.ITestEngine -Microsoft.VisualStudio.TestPlatform.ObjectModel.Engine.ITestEngine.GetDiscoveryManager(Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.IRequestData requestData, Microsoft.VisualStudio.TestPlatform.ObjectModel.Host.ITestRuntimeProvider testHostManager, Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.DiscoveryCriteria discoveryCriteria) -> Microsoft.VisualStudio.TestPlatform.ObjectModel.Engine.IProxyDiscoveryManager -Microsoft.VisualStudio.TestPlatform.ObjectModel.Engine.ITestEngine.GetExecutionManager(Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.IRequestData requestData, Microsoft.VisualStudio.TestPlatform.ObjectModel.Host.ITestRuntimeProvider testHostManager, Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.TestRunCriteria testRunCriteria) -> Microsoft.VisualStudio.TestPlatform.ObjectModel.Engine.IProxyExecutionManager Microsoft.VisualStudio.TestPlatform.ObjectModel.Engine.ITestEngine.GetExtensionManager() -> Microsoft.VisualStudio.TestPlatform.ObjectModel.Engine.ITestExtensionManager Microsoft.VisualStudio.TestPlatform.ObjectModel.Engine.ITestEngine.GetLoggerManager(Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.IRequestData requestData) -> Microsoft.VisualStudio.TestPlatform.ObjectModel.Engine.ITestLoggerManager -Microsoft.VisualStudio.TestPlatform.ObjectModel.Engine.ITestEngine.GetTestSessionManager(Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.IRequestData requestData, Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.StartTestSessionCriteria testSessionCriteria) -> Microsoft.VisualStudio.TestPlatform.ObjectModel.Engine.IProxyTestSessionManager Microsoft.VisualStudio.TestPlatform.ObjectModel.Engine.ITestExtensionManager Microsoft.VisualStudio.TestPlatform.ObjectModel.Engine.ITestExtensionManager.ClearExtensions() -> void Microsoft.VisualStudio.TestPlatform.ObjectModel.Engine.ITestExtensionManager.UseAdditionalExtensions(System.Collections.Generic.IEnumerable pathToAdditionalExtensions, bool skipExtensionFilters) -> void @@ -295,4 +290,3 @@ static Microsoft.VisualStudio.TestPlatform.Common.Utilities.RunSettingsUtilities virtual Microsoft.VisualStudio.TestPlatform.Common.ExtensionFramework.TestPluginCache.GetFilteredExtensions(System.Collections.Generic.List extensions, string endsWithPattern) -> System.Collections.Generic.IEnumerable virtual Microsoft.VisualStudio.TestPlatform.Common.ExtensionFramework.Utilities.TestPluginInformation.IdentifierData.get -> string virtual Microsoft.VisualStudio.TestPlatform.Common.ExtensionFramework.Utilities.TestPluginInformation.Metadata.get -> System.Collections.Generic.ICollection -virtual Microsoft.VisualStudio.TestPlatform.Common.Hosting.TestRuntimeProviderManager.GetTestHostManagerByRunConfiguration(string runConfiguration) -> Microsoft.VisualStudio.TestPlatform.ObjectModel.Host.ITestRuntimeProvider diff --git a/src/Microsoft.TestPlatform.Common/PublicAPI/PublicAPI.Unshipped.txt b/src/Microsoft.TestPlatform.Common/PublicAPI/PublicAPI.Unshipped.txt index 173c7c1cd4..9181f45d0e 100644 --- a/src/Microsoft.TestPlatform.Common/PublicAPI/PublicAPI.Unshipped.txt +++ b/src/Microsoft.TestPlatform.Common/PublicAPI/PublicAPI.Unshipped.txt @@ -13,3 +13,7 @@ Microsoft.VisualStudio.TestPlatform.ObjectModel.Engine.DiscoveryStatus Microsoft.VisualStudio.TestPlatform.ObjectModel.Engine.DiscoveryStatus.NotDiscovered = 0 -> Microsoft.VisualStudio.TestPlatform.ObjectModel.Engine.DiscoveryStatus Microsoft.VisualStudio.TestPlatform.ObjectModel.Engine.DiscoveryStatus.PartiallyDiscovered = 1 -> Microsoft.VisualStudio.TestPlatform.ObjectModel.Engine.DiscoveryStatus Microsoft.VisualStudio.TestPlatform.ObjectModel.Engine.DiscoveryStatus.FullyDiscovered = 2 -> Microsoft.VisualStudio.TestPlatform.ObjectModel.Engine.DiscoveryStatus +Microsoft.VisualStudio.TestPlatform.ObjectModel.Engine.ITestEngine.GetExecutionManager(Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.IRequestData requestData, Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.TestRunCriteria testRunCriteria, System.Collections.Generic.Dictionary sourceToSourceDetailMap) -> Microsoft.VisualStudio.TestPlatform.ObjectModel.Engine.IProxyExecutionManager +virtual Microsoft.VisualStudio.TestPlatform.Common.Hosting.TestRuntimeProviderManager.GetTestHostManagerByRunConfiguration(string runConfiguration, System.Collections.Generic.List _) -> Microsoft.VisualStudio.TestPlatform.ObjectModel.Host.ITestRuntimeProvider +Microsoft.VisualStudio.TestPlatform.ObjectModel.Engine.ITestEngine.GetDiscoveryManager(Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.IRequestData requestData, Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.DiscoveryCriteria discoveryCriteria, System.Collections.Generic.Dictionary sourceToSourceDetailMap) -> Microsoft.VisualStudio.TestPlatform.ObjectModel.Engine.IProxyDiscoveryManager +Microsoft.VisualStudio.TestPlatform.ObjectModel.Engine.ITestEngine.GetTestSessionManager(Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.IRequestData requestData, Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.StartTestSessionCriteria testSessionCriteria, System.Collections.Generic.Dictionary sourceToSourceDetailMap) -> Microsoft.VisualStudio.TestPlatform.ObjectModel.Engine.IProxyTestSessionManager diff --git a/src/Microsoft.TestPlatform.CrossPlatEngine/PublicAPI/PublicAPI.Shipped.txt b/src/Microsoft.TestPlatform.CrossPlatEngine/PublicAPI/PublicAPI.Shipped.txt index 7433a8f41b..7abace2fc9 100644 --- a/src/Microsoft.TestPlatform.CrossPlatEngine/PublicAPI/PublicAPI.Shipped.txt +++ b/src/Microsoft.TestPlatform.CrossPlatEngine/PublicAPI/PublicAPI.Shipped.txt @@ -86,13 +86,9 @@ Microsoft.VisualStudio.TestPlatform.CrossPlatEngine.Execution.ExecutionManager.I Microsoft.VisualStudio.TestPlatform.CrossPlatEngine.Execution.ExecutionManager.StartTestRun(System.Collections.Generic.Dictionary> adapterSourceMap, string package, string runSettings, Microsoft.VisualStudio.TestPlatform.ObjectModel.Engine.ClientProtocol.TestExecutionContext testExecutionContext, Microsoft.VisualStudio.TestPlatform.ObjectModel.Engine.ITestCaseEventsHandler testCaseEventsHandler, Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.ITestRunEventsHandler runEventsHandler) -> void Microsoft.VisualStudio.TestPlatform.CrossPlatEngine.Execution.ExecutionManager.StartTestRun(System.Collections.Generic.IEnumerable tests, string package, string runSettings, Microsoft.VisualStudio.TestPlatform.ObjectModel.Engine.ClientProtocol.TestExecutionContext testExecutionContext, Microsoft.VisualStudio.TestPlatform.ObjectModel.Engine.ITestCaseEventsHandler testCaseEventsHandler, Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.ITestRunEventsHandler runEventsHandler) -> void Microsoft.VisualStudio.TestPlatform.CrossPlatEngine.ProxyTestSessionManager -Microsoft.VisualStudio.TestPlatform.CrossPlatEngine.ProxyTestSessionManager.ProxyTestSessionManager(Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.StartTestSessionCriteria criteria, int testhostCount, System.Func proxyCreator) -> void Microsoft.VisualStudio.TestPlatform.CrossPlatEngine.TestEngine -Microsoft.VisualStudio.TestPlatform.CrossPlatEngine.TestEngine.GetDiscoveryManager(Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.IRequestData requestData, Microsoft.VisualStudio.TestPlatform.ObjectModel.Host.ITestRuntimeProvider testHostManager, Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.DiscoveryCriteria discoveryCriteria) -> Microsoft.VisualStudio.TestPlatform.ObjectModel.Engine.IProxyDiscoveryManager -Microsoft.VisualStudio.TestPlatform.CrossPlatEngine.TestEngine.GetExecutionManager(Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.IRequestData requestData, Microsoft.VisualStudio.TestPlatform.ObjectModel.Host.ITestRuntimeProvider testHostManager, Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.TestRunCriteria testRunCriteria) -> Microsoft.VisualStudio.TestPlatform.ObjectModel.Engine.IProxyExecutionManager Microsoft.VisualStudio.TestPlatform.CrossPlatEngine.TestEngine.GetExtensionManager() -> Microsoft.VisualStudio.TestPlatform.ObjectModel.Engine.ITestExtensionManager Microsoft.VisualStudio.TestPlatform.CrossPlatEngine.TestEngine.GetLoggerManager(Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.IRequestData requestData) -> Microsoft.VisualStudio.TestPlatform.ObjectModel.Engine.ITestLoggerManager -Microsoft.VisualStudio.TestPlatform.CrossPlatEngine.TestEngine.GetTestSessionManager(Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.IRequestData requestData, Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.StartTestSessionCriteria testSessionCriteria) -> Microsoft.VisualStudio.TestPlatform.ObjectModel.Engine.IProxyTestSessionManager Microsoft.VisualStudio.TestPlatform.CrossPlatEngine.TestEngine.TestEngine() -> void Microsoft.VisualStudio.TestPlatform.CrossPlatEngine.TestEngine.TestEngine(Microsoft.VisualStudio.TestPlatform.Common.Hosting.TestRuntimeProviderManager testHostProviderManager, Microsoft.VisualStudio.TestPlatform.PlatformAbstractions.Interfaces.IProcessHelper processHelper) -> void Microsoft.VisualStudio.TestPlatform.CrossPlatEngine.TestExtensionManager diff --git a/src/Microsoft.TestPlatform.PlatformAbstractions/common/IO/PlatformStream.cs b/src/Microsoft.TestPlatform.PlatformAbstractions/common/IO/PlatformStream.cs index 88af447185..eaaa320ffc 100644 --- a/src/Microsoft.TestPlatform.PlatformAbstractions/common/IO/PlatformStream.cs +++ b/src/Microsoft.TestPlatform.PlatformAbstractions/common/IO/PlatformStream.cs @@ -1,12 +1,12 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT license. See LICENSE file in the project root for full license information. +#if NETFRAMEWORK || NETCOREAPP || NETSTANDARD2_0 + using System.IO; using Microsoft.VisualStudio.TestPlatform.PlatformAbstractions.Interfaces; -#if NETFRAMEWORK || NETCOREAPP || NETSTANDARD2_0 - #nullable disable namespace Microsoft.VisualStudio.TestPlatform.PlatformAbstractions; diff --git a/src/Microsoft.TestPlatform.PlatformAbstractions/common/System/ProcessHelper.cs b/src/Microsoft.TestPlatform.PlatformAbstractions/common/System/ProcessHelper.cs index 92b00b6c59..61923a20d1 100644 --- a/src/Microsoft.TestPlatform.PlatformAbstractions/common/System/ProcessHelper.cs +++ b/src/Microsoft.TestPlatform.PlatformAbstractions/common/System/ProcessHelper.cs @@ -1,6 +1,8 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT license. See LICENSE file in the project root for full license information. +#if NETFRAMEWORK || NETCOREAPP || NETSTANDARD2_0 + using System; using System.Collections.Generic; using System.Diagnostics; @@ -11,8 +13,6 @@ using Microsoft.VisualStudio.TestPlatform.PlatformAbstractions.Interfaces; -#if NETFRAMEWORK || NETCOREAPP || NETSTANDARD2_0 - #nullable disable namespace Microsoft.VisualStudio.TestPlatform.PlatformAbstractions; diff --git a/src/Microsoft.TestPlatform.PlatformAbstractions/common/Tracing/RollingFileTraceListener.cs b/src/Microsoft.TestPlatform.PlatformAbstractions/common/Tracing/RollingFileTraceListener.cs index ff249331aa..a4dae6ddb5 100644 --- a/src/Microsoft.TestPlatform.PlatformAbstractions/common/Tracing/RollingFileTraceListener.cs +++ b/src/Microsoft.TestPlatform.PlatformAbstractions/common/Tracing/RollingFileTraceListener.cs @@ -1,13 +1,13 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT license. See LICENSE file in the project root for full license information. +#if NETFRAMEWORK || NETCOREAPP || NETSTANDARD2_0 + using System; using System.Diagnostics; using System.IO; using System.Text; -#if NETFRAMEWORK || NETCOREAPP || NETSTANDARD2_0 - #nullable disable namespace Microsoft.VisualStudio.TestPlatform.ObjectModel; diff --git a/src/Microsoft.TestPlatform.VsTestConsole.TranslationLayer/ConsoleParameters.cs b/src/Microsoft.TestPlatform.VsTestConsole.TranslationLayer/ConsoleParameters.cs index a1abda01df..7386f07d06 100644 --- a/src/Microsoft.TestPlatform.VsTestConsole.TranslationLayer/ConsoleParameters.cs +++ b/src/Microsoft.TestPlatform.VsTestConsole.TranslationLayer/ConsoleParameters.cs @@ -1,9 +1,7 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT license. See LICENSE file in the project root for full license information. -#if NETFRAMEWORK using System.Collections.Generic; -#endif using System.Diagnostics; using System.IO; @@ -41,16 +39,11 @@ public ConsoleParameters(IFileHelper fileHelper) _fileHelper = fileHelper; } -#if NETFRAMEWORK - /// - /// TODO: Remove the #if when project is targeted to netstandard2.0 /// Environment variables to be set for the process /// public Dictionary EnvironmentVariables { get; set; } -#endif - /// /// Trace level for logs. /// diff --git a/src/Microsoft.TestPlatform.VsTestConsole.TranslationLayer/PublicAPI/PublicAPI.Shipped.txt b/src/Microsoft.TestPlatform.VsTestConsole.TranslationLayer/PublicAPI/PublicAPI.Shipped.txt index cb8161d464..e9c4b6ee04 100644 --- a/src/Microsoft.TestPlatform.VsTestConsole.TranslationLayer/PublicAPI/PublicAPI.Shipped.txt +++ b/src/Microsoft.TestPlatform.VsTestConsole.TranslationLayer/PublicAPI/PublicAPI.Shipped.txt @@ -196,4 +196,6 @@ static Microsoft.VisualStudio.TestPlatform.VsTestConsole.TranslationLayer.Resour static Microsoft.VisualStudio.TestPlatform.VsTestConsole.TranslationLayer.Resources.Resources.InvalidFilePath.get -> string static Microsoft.VisualStudio.TestPlatform.VsTestConsole.TranslationLayer.Resources.Resources.ResourceManager.get -> System.Resources.ResourceManager static Microsoft.VisualStudio.TestPlatform.VsTestConsole.TranslationLayer.Resources.Resources.VsTestProcessExitedAbnormally.get -> string -virtual Microsoft.TestPlatform.VsTestConsole.TranslationLayer.TestSession.Dispose(bool disposing) -> void \ No newline at end of file +virtual Microsoft.TestPlatform.VsTestConsole.TranslationLayer.TestSession.Dispose(bool disposing) -> void +Microsoft.TestPlatform.VsTestConsole.TranslationLayer.ConsoleParameters.EnvironmentVariables.get -> System.Collections.Generic.Dictionary +Microsoft.TestPlatform.VsTestConsole.TranslationLayer.ConsoleParameters.EnvironmentVariables.set -> void diff --git a/src/Microsoft.TestPlatform.VsTestConsole.TranslationLayer/PublicAPI/net451/PublicAPI.Shipped.txt b/src/Microsoft.TestPlatform.VsTestConsole.TranslationLayer/PublicAPI/net451/PublicAPI.Shipped.txt index 2b1c581fe8..e69de29bb2 100644 --- a/src/Microsoft.TestPlatform.VsTestConsole.TranslationLayer/PublicAPI/net451/PublicAPI.Shipped.txt +++ b/src/Microsoft.TestPlatform.VsTestConsole.TranslationLayer/PublicAPI/net451/PublicAPI.Shipped.txt @@ -1,2 +0,0 @@ -Microsoft.TestPlatform.VsTestConsole.TranslationLayer.ConsoleParameters.EnvironmentVariables.get -> System.Collections.Generic.Dictionary -Microsoft.TestPlatform.VsTestConsole.TranslationLayer.ConsoleParameters.EnvironmentVariables.set -> void \ No newline at end of file diff --git a/src/Microsoft.TestPlatform.VsTestConsole.TranslationLayer/VsTestConsoleProcessManager.cs b/src/Microsoft.TestPlatform.VsTestConsole.TranslationLayer/VsTestConsoleProcessManager.cs index 48b804836e..b0d4719d2a 100644 --- a/src/Microsoft.TestPlatform.VsTestConsole.TranslationLayer/VsTestConsoleProcessManager.cs +++ b/src/Microsoft.TestPlatform.VsTestConsole.TranslationLayer/VsTestConsoleProcessManager.cs @@ -116,12 +116,11 @@ public void StartProcess(ConsoleParameters consoleParameters) UseShellExecute = false, CreateNoWindow = true, RedirectStandardOutput = true, - RedirectStandardError = true + RedirectStandardError = true, }; EqtTrace.Verbose("VsTestCommandLineWrapper: Process Start Info {0} {1}", info.FileName, info.Arguments); -#if NETFRAMEWORK if (consoleParameters.EnvironmentVariables != null) { info.EnvironmentVariables.Clear(); @@ -133,7 +132,7 @@ public void StartProcess(ConsoleParameters consoleParameters) } } } -#endif + try { _process = Process.Start(info); diff --git a/test/Microsoft.TestPlatform.CrossPlatEngine.UnitTests/Client/Parallel/ParallelOperationManagerTests.cs b/test/Microsoft.TestPlatform.CrossPlatEngine.UnitTests/Client/Parallel/ParallelOperationManagerTests.cs index d160f3b0af..9bc7b52115 100644 --- a/test/Microsoft.TestPlatform.CrossPlatEngine.UnitTests/Client/Parallel/ParallelOperationManagerTests.cs +++ b/test/Microsoft.TestPlatform.CrossPlatEngine.UnitTests/Client/Parallel/ParallelOperationManagerTests.cs @@ -1,5 +1,5 @@ -//// Copyright (c) Microsoft Corporation. All rights reserved. -//// Licensed under the MIT license. See LICENSE file in the project root for full license information. +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. //using System; //using System.Collections.Generic; diff --git a/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestBase.cs b/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestBase.cs index 9d714e1033..208bb73851 100644 --- a/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestBase.cs +++ b/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestBase.cs @@ -535,7 +535,11 @@ public IVsTestConsoleWrapper GetVsTestConsoleWrapper(out TempDirectory logFileDi throw new FileNotFoundException($"File '{dotnetPath}' was not found."); } - var vstestConsoleWrapper = new VsTestConsoleWrapper(consoleRunnerPath, dotnetPath, new ConsoleParameters() { LogFilePath = logFilePath }); + var vstestConsoleWrapper = new VsTestConsoleWrapper(consoleRunnerPath, dotnetPath, new ConsoleParameters() + { + LogFilePath = logFilePath, + EnvironmentVariables = new Dictionary { ["VSTEST_RUNNER_DEBUG_ATTACHVS"] = "1" } + }); vstestConsoleWrapper.StartSession(); return vstestConsoleWrapper; diff --git a/test/vstest.ProgrammerTests/BasicRunAndDiscovery.cs b/test/vstest.ProgrammerTests/BasicRunAndDiscovery.cs index 0a75a47e48..d992a66eea 100644 --- a/test/vstest.ProgrammerTests/BasicRunAndDiscovery.cs +++ b/test/vstest.ProgrammerTests/BasicRunAndDiscovery.cs @@ -9,7 +9,9 @@ using vstest.ProgrammerTests.Fakes; using Intent; +#pragma warning disable IDE1006 // Naming Styles namespace vstest.ProgrammerTests; +#pragma warning restore IDE1006 // Naming Styles // Tests are run by Intent library that is executed from our Program.Main. To debug press F5 in VS, and maybe mark just a single test with [Only]. // To just run, press Ctrl+F5 to run without debugging. It will use short timeout for abort in case something is wrong with your test. diff --git a/test/vstest.console.UnitTests/Processors/EnvironmentArgumentProcessorTests.cs b/test/vstest.console.UnitTests/Processors/EnvironmentArgumentProcessorTests.cs index 77048423cf..1656c003e2 100644 --- a/test/vstest.console.UnitTests/Processors/EnvironmentArgumentProcessorTests.cs +++ b/test/vstest.console.UnitTests/Processors/EnvironmentArgumentProcessorTests.cs @@ -1,7 +1,6 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT license. See LICENSE file in the project root for full license information. -using System; using System.Linq; using System.Xml.Linq; From 9e1f93aaed128d5a68b366cb3396d76c3194b121 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Jare=C5=A1?= Date: Thu, 3 Mar 2022 15:23:53 +0100 Subject: [PATCH 017/112] Apply suggestions from code review MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Amaury Levé --- playground/MSTest1/UnitTest1.cs | 1 + playground/TestPlatform.Playground/Program.cs | 1 - .../FeatureFlag/FeatureFlag.cs | 2 +- .../Friends.cs | 1 - .../Parallel/ParallelOperationManager.cs | 2 +- .../Parallel/ParallelProxyDiscoveryManager.cs | 15 ++++---- .../Client/TestRuntimeProviderInfo.cs | 1 - .../Utilities/SourceDetailHelper.cs | 4 +- .../BasicRunAndDiscovery.cs | 1 - .../Fakes/FakeDiscoveryEventsRegistrar.cs | 2 +- .../Fakes/FakeTestHostResponsesBuilder.cs | 6 +-- .../Fakes/FakeTestSessionEventsHandler.cs | 1 - test/vstest.ProgrammerTests/Fakes/Fixture.cs | 2 +- .../Fakes/TestMessage.cs | 1 - .../MultiTFMRunAndDiscovery.cs | 37 ++++++++++--------- 15 files changed, 36 insertions(+), 41 deletions(-) diff --git a/playground/MSTest1/UnitTest1.cs b/playground/MSTest1/UnitTest1.cs index 8be51be9be..aa92d18eef 100644 --- a/playground/MSTest1/UnitTest1.cs +++ b/playground/MSTest1/UnitTest1.cs @@ -6,6 +6,7 @@ #nullable disable namespace MSTest1; + [TestClass] public class UnitTest1 { diff --git a/playground/TestPlatform.Playground/Program.cs b/playground/TestPlatform.Playground/Program.cs index 8559ceaa90..ab0d43f731 100644 --- a/playground/TestPlatform.Playground/Program.cs +++ b/playground/TestPlatform.Playground/Program.cs @@ -61,7 +61,6 @@ static void Main(string[] args) var settingsFile = Path.GetTempFileName(); try { - File.WriteAllText(settingsFile, sourceSettings); Process.Start(console, string.Join(" ", sources) + " --settings:" + settingsFile).WaitForExit(); } diff --git a/src/Microsoft.TestPlatform.CoreUtilities/FeatureFlag/FeatureFlag.cs b/src/Microsoft.TestPlatform.CoreUtilities/FeatureFlag/FeatureFlag.cs index 3ef9992cea..c7e7fa99ac 100644 --- a/src/Microsoft.TestPlatform.CoreUtilities/FeatureFlag/FeatureFlag.cs +++ b/src/Microsoft.TestPlatform.CoreUtilities/FeatureFlag/FeatureFlag.cs @@ -36,7 +36,7 @@ internal static void Reset() internal static void SetFlag(string name, bool value) { if (!FeatureFlags.ContainsKey(name)) - throw new ArgumentException($"Feature flag {name} is a known feature flag."); + throw new ArgumentException($"Feature flag {name} is not a known feature flag."); FeatureFlags[name] = value; } diff --git a/src/Microsoft.TestPlatform.CoreUtilities/Friends.cs b/src/Microsoft.TestPlatform.CoreUtilities/Friends.cs index c1d79e7a57..4072f8a386 100644 --- a/src/Microsoft.TestPlatform.CoreUtilities/Friends.cs +++ b/src/Microsoft.TestPlatform.CoreUtilities/Friends.cs @@ -6,5 +6,4 @@ [assembly: InternalsVisibleTo("vstest.console, PublicKey=002400000480000094000000060200000024000052534131000400000100010007d1fa57c4aed9f0a32e84aa0faefd0de9e8fd6aec8f87fb03766c834c99921eb23be79ad9d5dcc1dd9ad236132102900b723cf980957fc4e177108fc607774f29e8320e92ea05ece4e821c0a5efe8f1645c4c0c93c1ab99285d622caa652c1dfad63d745d6f2de5f17e5eaf0fc4963d261c8a12436518206dc093344d5ad293")] [assembly: InternalsVisibleTo("vstest.console.arm64, PublicKey=002400000480000094000000060200000024000052534131000400000100010007d1fa57c4aed9f0a32e84aa0faefd0de9e8fd6aec8f87fb03766c834c99921eb23be79ad9d5dcc1dd9ad236132102900b723cf980957fc4e177108fc607774f29e8320e92ea05ece4e821c0a5efe8f1645c4c0c93c1ab99285d622caa652c1dfad63d745d6f2de5f17e5eaf0fc4963d261c8a12436518206dc093344d5ad293")] - [assembly: InternalsVisibleTo("vstest.ProgrammerTests, PublicKey=002400000480000094000000060200000024000052534131000400000100010007d1fa57c4aed9f0a32e84aa0faefd0de9e8fd6aec8f87fb03766c834c99921eb23be79ad9d5dcc1dd9ad236132102900b723cf980957fc4e177108fc607774f29e8320e92ea05ece4e821c0a5efe8f1645c4c0c93c1ab99285d622caa652c1dfad63d745d6f2de5f17e5eaf0fc4963d261c8a12436518206dc093344d5ad293")] diff --git a/src/Microsoft.TestPlatform.CrossPlatEngine/Client/Parallel/ParallelOperationManager.cs b/src/Microsoft.TestPlatform.CrossPlatEngine/Client/Parallel/ParallelOperationManager.cs index c37d9fb2aa..ea8d6b6e93 100644 --- a/src/Microsoft.TestPlatform.CrossPlatEngine/Client/Parallel/ParallelOperationManager.cs +++ b/src/Microsoft.TestPlatform.CrossPlatEngine/Client/Parallel/ParallelOperationManager.cs @@ -130,7 +130,7 @@ public bool RunNextWork(TManager completedManager!!) lock (_lock) { var completedSlot = _managerSlots.Where(s => ReferenceEquals(completedManager, s.Manager)).ToList(); - if (!completedSlot.Any()) + if (completedSlot.Count == 0) { throw new InvalidOperationException("The provided manager was not found in any slot."); } diff --git a/src/Microsoft.TestPlatform.CrossPlatEngine/Client/Parallel/ParallelProxyDiscoveryManager.cs b/src/Microsoft.TestPlatform.CrossPlatEngine/Client/Parallel/ParallelProxyDiscoveryManager.cs index fc63454529..8340a65d80 100644 --- a/src/Microsoft.TestPlatform.CrossPlatEngine/Client/Parallel/ParallelProxyDiscoveryManager.cs +++ b/src/Microsoft.TestPlatform.CrossPlatEngine/Client/Parallel/ParallelProxyDiscoveryManager.cs @@ -90,14 +90,12 @@ public void DiscoverTests(DiscoveryCriteria discoveryCriteria!!, ITestDiscoveryE } private ITestDiscoveryEventsHandler2 GetParallelEventHandler(ITestDiscoveryEventsHandler2 eventHandler, IProxyDiscoveryManager concurrentManager) - { - return new ParallelDiscoveryEventsHandler( - _requestData, - concurrentManager, - eventHandler, - this, - _currentDiscoveryDataAggregator); - } + => new ParallelDiscoveryEventsHandler( + _requestData, + concurrentManager, + eventHandler, + this, + _currentDiscoveryDataAggregator); /// public void Abort() @@ -222,6 +220,7 @@ private void DiscoverTestsOnConcurrentManager(IProxyDiscoveryManager proxyDiscov // Total tests must be zero here since parallel discovery events handler adds the count // Keep `lastChunk` as null since we don't want a message back to the IDE (discovery didn't even begin) // Set `isAborted` as true since we want this instance of discovery manager to be replaced + // TODO: the comment above mentions 0 tests but sends -1. Make sense of this. var discoveryCompleteEventsArgs = new DiscoveryCompleteEventArgs(-1, true); handler.HandleDiscoveryComplete(discoveryCompleteEventsArgs, null); }, diff --git a/src/Microsoft.TestPlatform.CrossPlatEngine/Client/TestRuntimeProviderInfo.cs b/src/Microsoft.TestPlatform.CrossPlatEngine/Client/TestRuntimeProviderInfo.cs index 5a1a409a1f..1369e89f55 100644 --- a/src/Microsoft.TestPlatform.CrossPlatEngine/Client/TestRuntimeProviderInfo.cs +++ b/src/Microsoft.TestPlatform.CrossPlatEngine/Client/TestRuntimeProviderInfo.cs @@ -1,7 +1,6 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT license. See LICENSE file in the project root for full license information. - using System; using System.Collections.Generic; diff --git a/src/Microsoft.TestPlatform.CrossPlatEngine/Utilities/SourceDetailHelper.cs b/src/Microsoft.TestPlatform.CrossPlatEngine/Utilities/SourceDetailHelper.cs index ea4813b6bd..09ad22d5b2 100644 --- a/src/Microsoft.TestPlatform.CrossPlatEngine/Utilities/SourceDetailHelper.cs +++ b/src/Microsoft.TestPlatform.CrossPlatEngine/Utilities/SourceDetailHelper.cs @@ -1,7 +1,6 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT license. See LICENSE file in the project root for full license information. - using System.IO; using System.Xml; @@ -10,11 +9,12 @@ using Microsoft.VisualStudio.TestPlatform.Utilities; namespace Microsoft.VisualStudio.TestPlatform.CrossPlatEngine.Utilities; + internal static class SourceDetailHelper { internal static string UpdateRunSettingsFromSourceDetail(string runSettings, SourceDetail sourceDetail) { - var stream = new StringReader(runSettings); + using var stream = new StringReader(runSettings); var reader = XmlReader.Create(stream, XmlRunSettingsUtilities.ReaderSettings); var document = new XmlDocument(); document.Load(reader); diff --git a/test/vstest.ProgrammerTests/BasicRunAndDiscovery.cs b/test/vstest.ProgrammerTests/BasicRunAndDiscovery.cs index d992a66eea..37e71ad330 100644 --- a/test/vstest.ProgrammerTests/BasicRunAndDiscovery.cs +++ b/test/vstest.ProgrammerTests/BasicRunAndDiscovery.cs @@ -1,7 +1,6 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT license. See LICENSE file in the project root for full license information. - using FluentAssertions; using Microsoft.VisualStudio.TestPlatform.ObjectModel; using Microsoft.VisualStudio.TestPlatform.ObjectModel.Client; diff --git a/test/vstest.ProgrammerTests/Fakes/FakeDiscoveryEventsRegistrar.cs b/test/vstest.ProgrammerTests/Fakes/FakeDiscoveryEventsRegistrar.cs index fdd1cb0ac2..2b22bf9b40 100644 --- a/test/vstest.ProgrammerTests/Fakes/FakeDiscoveryEventsRegistrar.cs +++ b/test/vstest.ProgrammerTests/Fakes/FakeDiscoveryEventsRegistrar.cs @@ -1,12 +1,12 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT license. See LICENSE file in the project root for full license information. - using Microsoft.VisualStudio.TestPlatform.Common.Interfaces; using Microsoft.VisualStudio.TestPlatform.ObjectModel.Client; using Microsoft.VisualStudio.TestPlatform.ObjectModel.Logging; namespace vstest.ProgrammerTests.Fakes; + internal class FakeTestDiscoveryEventsRegistrar : ITestDiscoveryEventsRegistrar { private readonly FakeErrorAggregator _fakeErrorAggregator; diff --git a/test/vstest.ProgrammerTests/Fakes/FakeTestHostResponsesBuilder.cs b/test/vstest.ProgrammerTests/Fakes/FakeTestHostResponsesBuilder.cs index eabaaeb1c5..2142f4077e 100644 --- a/test/vstest.ProgrammerTests/Fakes/FakeTestHostResponsesBuilder.cs +++ b/test/vstest.ProgrammerTests/Fakes/FakeTestHostResponsesBuilder.cs @@ -151,8 +151,9 @@ internal FakeTestHostResponsesBuilder StartDiscovery(List> test // the last batch will be sent as test run complete event // see TestRequestSender.OnDiscoveryMessageReceived to see how the vstest.console receives the data - List changeMessages = testResultBatches.Take(testResultBatches.Count - 1).Select(batch => - new FakeMessage>(MessageType.TestCasesFound, batch.Select(testResult => testResult.TestCase).ToList())).ToList(); + List changeMessages = testResultBatches.Take(testResultBatches.Count - 1) + .Select(batch => new FakeMessage>(MessageType.TestCasesFound, batch.Select(testResult => testResult.TestCase).ToList())) + .ToList(); // TODO: if we send this incorrectly the handler just continues, check logs if we can understand it from there. We should at least write a warning. // because otherwise it hangs. @@ -172,7 +173,6 @@ internal FakeTestHostResponsesBuilder StartDiscovery(List> test messages = completedMessage.AsList(); } - AddPairWithMultipleFakeMessages(MessageType.StartDiscovery, messages); return this; diff --git a/test/vstest.ProgrammerTests/Fakes/FakeTestSessionEventsHandler.cs b/test/vstest.ProgrammerTests/Fakes/FakeTestSessionEventsHandler.cs index 514734e59c..2597cb68ae 100644 --- a/test/vstest.ProgrammerTests/Fakes/FakeTestSessionEventsHandler.cs +++ b/test/vstest.ProgrammerTests/Fakes/FakeTestSessionEventsHandler.cs @@ -1,7 +1,6 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT license. See LICENSE file in the project root for full license information. - using Microsoft.VisualStudio.TestPlatform.ObjectModel.Client; using Microsoft.VisualStudio.TestPlatform.ObjectModel.Logging; diff --git a/test/vstest.ProgrammerTests/Fakes/Fixture.cs b/test/vstest.ProgrammerTests/Fakes/Fixture.cs index 93f6423333..a63782d57c 100644 --- a/test/vstest.ProgrammerTests/Fakes/Fixture.cs +++ b/test/vstest.ProgrammerTests/Fakes/Fixture.cs @@ -59,7 +59,7 @@ public Fixture(FixtureOptions? fixtureOptions = null) } #endif -#pragma warning disable CS0618 // Type or member is obsolete +#pragma warning disable CS0618 // Type or member is obsolete (to prevent use outside of test context) FeatureFlag.Reset(); fixtureOptions?.FeatureFlags?.ToList().ForEach(flag => FeatureFlag.SetFlag(flag.Key, flag.Value)); #pragma warning restore CS0618 // Type or member is obsolete diff --git a/test/vstest.ProgrammerTests/Fakes/TestMessage.cs b/test/vstest.ProgrammerTests/Fakes/TestMessage.cs index 45308b34cc..9a9ca99dcb 100644 --- a/test/vstest.ProgrammerTests/Fakes/TestMessage.cs +++ b/test/vstest.ProgrammerTests/Fakes/TestMessage.cs @@ -1,7 +1,6 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT license. See LICENSE file in the project root for full license information. - using Microsoft.VisualStudio.TestPlatform.ObjectModel.Logging; namespace vstest.ProgrammerTests.Fakes; diff --git a/test/vstest.ProgrammerTests/MultiTFMRunAndDiscovery.cs b/test/vstest.ProgrammerTests/MultiTFMRunAndDiscovery.cs index b7d71556f8..582ae70d62 100644 --- a/test/vstest.ProgrammerTests/MultiTFMRunAndDiscovery.cs +++ b/test/vstest.ProgrammerTests/MultiTFMRunAndDiscovery.cs @@ -12,19 +12,19 @@ using Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.Payloads; namespace vstest.ProgrammerTests; + public class MultiTFM { public class MultiTFMDiscovery { - [Test(@" - Given two test assemblies that have the same architecture - but have different target frameworks. + Given two test assemblies that have the same architecture + but have different target frameworks. - When we run test discovery. + When we run test discovery. - Then two testhosts should be started that target the same framework as each assembly. - ")] + Then two testhosts should be started that target the same framework as each assembly. + ")] public async Task A() { // -- arrange @@ -125,14 +125,14 @@ public async Task A() } [Test(@" - Given two test assemblies that have the same architecture - but have different target frameworks. + Given two test assemblies that have the same architecture + but have different target frameworks. - When we run test discovery - and provide runsettings that define the desired target framework. + When we run test discovery + and provide runsettings that define the desired target framework. - Then two testhosts should be started that target the framework chosen by runsettings. - ")] + Then two testhosts should be started that target the framework chosen by runsettings. + ")] public async Task B() { // -- arrange @@ -451,14 +451,14 @@ public class MultiTFMTestSessions { [Test(@" - Given two test assemblies that have the same architecture - but have different target frameworks. + Given two test assemblies that have the same architecture + but have different target frameworks. - When we execute tests - and provide runsettings that define the desired target framework. + When we execute tests + and provide runsettings that define the desired target framework. - Then two testhosts should be started that target the framework chosen by runsettings. - ")] + Then two testhosts should be started that target the framework chosen by runsettings. + ")] public async Task E() { // -- arrange @@ -575,6 +575,7 @@ public async Task E() public class MultiTFMRunAndDiscoveryCompatibilityMode { + // TODO: NOMERGE: Add backward compats flag functionality and test it. [Exclude] public async Task GivenMultipleMsTestAssembliesThatHaveTheSameArchitecture_AndHaveDifferentTargetFrameworks_AndMULTI_TFM_RUNFeatureFlagIsDisabled_WhenTestsAreRun_ThenTwoTesthostsAreStartedBothForTheSameTFM() { From 1d35dc5c55beefed356581225313f42dcb642db6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Jare=C5=A1?= Date: Thu, 3 Mar 2022 15:53:34 +0100 Subject: [PATCH 018/112] Update src/Microsoft.TestPlatform.CrossPlatEngine/Utilities/SourceDetailHelper.cs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Amaury Levé --- .../Utilities/SourceDetailHelper.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Microsoft.TestPlatform.CrossPlatEngine/Utilities/SourceDetailHelper.cs b/src/Microsoft.TestPlatform.CrossPlatEngine/Utilities/SourceDetailHelper.cs index 09ad22d5b2..b08c94d482 100644 --- a/src/Microsoft.TestPlatform.CrossPlatEngine/Utilities/SourceDetailHelper.cs +++ b/src/Microsoft.TestPlatform.CrossPlatEngine/Utilities/SourceDetailHelper.cs @@ -15,7 +15,7 @@ internal static class SourceDetailHelper internal static string UpdateRunSettingsFromSourceDetail(string runSettings, SourceDetail sourceDetail) { using var stream = new StringReader(runSettings); - var reader = XmlReader.Create(stream, XmlRunSettingsUtilities.ReaderSettings); + using var reader = XmlReader.Create(stream, XmlRunSettingsUtilities.ReaderSettings); var document = new XmlDocument(); document.Load(reader); var navigator = document.CreateNavigator(); From dfb2e811a510f8a83221f07b64effeeaee1a470d Mon Sep 17 00:00:00 2001 From: nohwnd Date: Fri, 4 Mar 2022 11:47:14 +0100 Subject: [PATCH 019/112] Fix acceptance tests. --- scripts/build.ps1 | 18 +++ .../AppDomainTests.cs | 13 +- .../ArgumentProcessorTests.cs | 4 +- .../BlameDataCollectorTests.cs | 56 ++++--- .../CUITTest.cs | 11 +- .../CodeCoverageAcceptanceTestBase.cs | 4 +- .../CodeCoverageTests.cs | 17 +-- .../DataCollectionTests.cs | 39 +++-- .../DebugAssertTests.cs | 3 +- .../DeprecateExtensionsPathWarningTests.cs | 15 +- .../DifferentTestFrameworkSimpleTests.cs | 144 +++++++++--------- .../DisableAppdomainTests.cs | 17 +-- .../DiscoveryTests.cs | 8 +- .../DotnetArchitectureSwitchTests.Windows.cs | 8 +- .../DotnetArchitectureSwitchTests.cs | 4 +- .../EventLogCollectorTests.cs | 20 ++- .../ExecutionTests.cs | 47 +++--- .../ExecutionThreadApartmentStateTests.cs | 12 +- .../Extension/RunnnerInfo.cs | 31 ++++ .../FilePatternParserTests.cs | 24 +-- .../FrameworkTests.cs | 12 +- .../LoggerTests.cs | 41 +++-- ...rosoft.TestPlatform.AcceptanceTests.csproj | 6 + .../MultitargetingTestHostTests.cs | 3 +- .../PlatformTests.cs | 7 +- .../PostProcessingTests.cs | 9 +- .../ProcessesInteractionTests.cs | 3 - .../ResultsDirectoryTests.cs | 7 +- .../RunsettingsTests.cs | 105 +++++++------ .../SelfContainedAppTests.cs | 3 +- .../TelemetryTests.cs | 30 ++-- .../TestCaseFilterTests.cs | 34 ++--- .../TestPlatformNugetPackageTests.cs | 2 +- .../CodeCoverageTests.cs | 6 +- .../TranslationLayerTests/RunTests.cs | 3 + ...RunTestsWithDifferentConfigurationTests.cs | 3 +- .../DotnetHostArchitectureVerifierTests.cs | 2 +- .../IntegrationTestBase.cs | 58 ++++--- ...icrosoft.TestPlatform.TestUtilities.csproj | 2 +- .../RandomId.cs | 46 ++++++ .../TempDirectory.cs | 34 +++-- 41 files changed, 487 insertions(+), 424 deletions(-) create mode 100644 test/Microsoft.TestPlatform.TestUtilities/RandomId.cs diff --git a/scripts/build.ps1 b/scripts/build.ps1 index 341d640b26..8e1e80d6ae 100644 --- a/scripts/build.ps1 +++ b/scripts/build.ps1 @@ -123,6 +123,24 @@ if ($env:PATH -notlike "*$attachVsPath") { $env:PATH = "$attachVsPath;$env:PATH" } +# VsixUtil gets regularly eaten by antivirus or something. Remove the package dir if it gets broken +# so nuget restores it correctly. +$vsSdkBuildToolsVersion = ([xml](Get-Content $env:TP_ROOT_DIR\scripts\build\TestPlatform.Dependencies.props)).Project.PropertyGroup.VSSdkBuildToolsVersion +$vsixUtilDir = "$env:TP_ROOT_DIR\packages\microsoft.vssdk.buildtools" +if ((Test-Path $vsixUtilDir) -and -not (Test-Path "$vsixUtilDir\$vsSdkBuildToolsVersion\tools\vssdk\bin\VsixUtil.exe")) +{ + Remove-Item -Recurse -Force $vsixUtilDir +} + +# Procdump gets regularly eaten by antivirus or something. Remove the package dir if it gets broken +# so nuget restores it correctly. +$procdumpDir = "$env:TP_ROOT_DIR\packages\procdump" +if ((Test-Path $procdumpDir) -and 2 -ne @(Get-Item "$procdumpDir\0.0.1\bin").Length) +{ + Remove-Item -Recurse -Force $procdumpDir +} + + function Invoke-Build { $timer = Start-Timer diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/AppDomainTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/AppDomainTests.cs index 442b24b0d4..776ad1956e 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/AppDomainTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/AppDomainTests.cs @@ -29,22 +29,21 @@ public void RunTestExecutionWithDisableAppDomain(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - using var tempDir = new TempDirectory(); - var testAppDomainDetailFileName = Path.Combine(tempDir.Path, "appdomain_test.txt"); - var dataCollectorAppDomainDetailFileName = Path.Combine(tempDir.Path, "appdomain_datacollector.txt"); + var testAppDomainDetailFileName = Path.Combine(TempDirectory.Path, "appdomain_test.txt"); + var dataCollectorAppDomainDetailFileName = Path.Combine(TempDirectory.Path, "appdomain_datacollector.txt"); // Delete test output files if already exist File.Delete(testAppDomainDetailFileName); File.Delete(dataCollectorAppDomainDetailFileName); - var runsettingsFilePath = GetInProcDataCollectionRunsettingsFile(true, tempDir); + var runsettingsFilePath = GetInProcDataCollectionRunsettingsFile(true, TempDirectory); var arguments = PrepareArguments( GetSampleTestAssembly(), GetTestAdapterPath(), runsettingsFilePath, FrameworkArgValue, runnerInfo.InIsolationValue, - tempDir.Path); + TempDirectory.Path); // Sets the environment variables used by the test project and test data collector. var env = new Dictionary @@ -74,9 +73,9 @@ private static bool IsFilesContentEqual(string filePath1, string filePath2) return string.Equals(content1, content2, StringComparison.Ordinal); } - private string GetInProcDataCollectionRunsettingsFile(bool disableAppDomain, TempDirectory tempDirectory) + private string GetInProcDataCollectionRunsettingsFile(bool disableAppDomain, TempDirectory TempDirectory) { - var runSettings = Path.Combine(tempDirectory.Path, "test_" + Guid.NewGuid() + ".runsettings"); + var runSettings = Path.Combine(TempDirectory.Path, "test_" + Guid.NewGuid() + ".runsettings"); var inprocasm = _testEnvironment.GetTestAsset("SimpleDataCollector.dll"); #if !NET451 var assemblyName = AssemblyLoadContext.GetAssemblyName(inprocasm); diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/ArgumentProcessorTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/ArgumentProcessorTests.cs index 37c8f7b88a..44a4a39dbd 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/ArgumentProcessorTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/ArgumentProcessorTests.cs @@ -1,7 +1,6 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT license. See LICENSE file in the project root for full license information. -using Microsoft.TestPlatform.TestUtilities; using Microsoft.VisualStudio.TestTools.UnitTesting; #nullable disable @@ -41,8 +40,7 @@ public void PassingInvalidArgumentsToVsTestConsoleShouldNotPrintHelpMessage(Runn { SetTestEnvironment(_testEnvironment, runnerInfo); - using var tempDir = new TempDirectory(); - var arguments = PrepareArguments(GetSampleTestAssembly(), GetTestAdapterPath(), string.Empty, FrameworkArgValue, resultsDirectory: tempDir.Path); + var arguments = PrepareArguments(GetSampleTestAssembly(), GetTestAdapterPath(), string.Empty, FrameworkArgValue, resultsDirectory: TempDirectory.Path); arguments = string.Concat(arguments, " /badArgument"); InvokeVsTest(arguments); diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/BlameDataCollectorTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/BlameDataCollectorTests.cs index 66813e4d15..fc00b22d7a 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/BlameDataCollectorTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/BlameDataCollectorTests.cs @@ -6,6 +6,7 @@ using System.IO; using System.Linq; using System.Text; +using System.Text.RegularExpressions; using System.Xml; using Microsoft.TestPlatform.TestUtilities; @@ -23,6 +24,19 @@ public class BlameDataCollectorTests : AcceptanceTestBase { public const string NETCOREANDFX = "net452;net472;netcoreapp3.1"; public const string NET50 = "net5.0"; + private readonly string _procDumpPath; + + public BlameDataCollectorTests() + { + _procDumpPath = Path.Combine(_testEnvironment.PackageDirectory, @"procdump\0.0.1\bin"); + var procDumpExePath = Path.Combine(_procDumpPath, "procdump.exe"); + if (!File.Exists(procDumpExePath)) + { + throw new InvalidOperationException($"Procdump path {procDumpExePath} does not exist. " + + $"It is possible that antivirus deleted it from your nuget cache. " + + $"Delete the whole procdump folder in your nuget cache, and run build, or restore"); + } + } [TestMethod] // netcoreapp2.1 dump is not supported on Linux @@ -31,15 +45,14 @@ public class BlameDataCollectorTests : AcceptanceTestBase [NetCoreTargetFrameworkDataSource] public void BlameDataCollectorShouldGiveCorrectTestCaseName(RunnerInfo runnerInfo) { - using var tempDir = new TempDirectory(); SetTestEnvironment(_testEnvironment, runnerInfo); var assemblyPaths = GetAssetFullPath("BlameUnitTestProject.dll"); var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue); arguments = string.Concat(arguments, $" /Blame"); - arguments = string.Concat(arguments, $" /ResultsDirectory:{tempDir.Path}"); + arguments = string.Concat(arguments, $" /ResultsDirectory:{TempDirectory.Path}"); InvokeVsTest(arguments); - VaildateOutput(tempDir, "BlameUnitTestProject.UnitTest1.TestMethod2"); + VaildateOutput(TempDirectory, "BlameUnitTestProject.UnitTest1.TestMethod2"); } [TestMethod] @@ -49,23 +62,22 @@ public void BlameDataCollectorShouldGiveCorrectTestCaseName(RunnerInfo runnerInf [NetCoreTargetFrameworkDataSource] public void BlameDataCollectorShouldOutputDumpFile(RunnerInfo runnerInfo) { - using var tempDir = new TempDirectory(); SetTestEnvironment(_testEnvironment, runnerInfo); var assemblyPaths = BuildMultipleAssemblyPath("SimpleTestProject3.dll").Trim('\"'); var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, string.Empty, runnerInfo.InIsolationValue); arguments = string.Concat(arguments, $" /Blame:CollectDump"); - arguments = string.Concat(arguments, $" /ResultsDirectory:{tempDir.Path}"); + arguments = string.Concat(arguments, $" /ResultsDirectory:{TempDirectory.Path}"); arguments = string.Concat(arguments, " /testcasefilter:ExitWithStackoverFlow"); var env = new Dictionary { - ["PROCDUMP_PATH"] = Path.Combine(_testEnvironment.PackageDirectory, @"procdump\0.0.1\bin"), + ["PROCDUMP_PATH"] = _procDumpPath, }; InvokeVsTest(arguments, env); - VaildateOutput(tempDir, "SampleUnitTestProject3.UnitTest1.ExitWithStackoverFlow", validateDumpFile: true); + VaildateOutput(TempDirectory, "SampleUnitTestProject3.UnitTest1.ExitWithStackoverFlow", validateDumpFile: true); } [TestMethod] @@ -75,18 +87,17 @@ public void BlameDataCollectorShouldOutputDumpFile(RunnerInfo runnerInfo) [NetCoreTargetFrameworkDataSource] public void BlameDataCollectorShouldNotOutputDumpFileWhenNoCrashOccurs(RunnerInfo runnerInfo) { - using var tempDir = new TempDirectory(); SetTestEnvironment(_testEnvironment, runnerInfo); var assemblyPaths = BuildMultipleAssemblyPath("SimpleTestProject.dll").Trim('\"'); var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, string.Empty, runnerInfo.InIsolationValue); arguments = string.Concat(arguments, $" /Blame:CollectDump"); - arguments = string.Concat(arguments, $" /ResultsDirectory:{tempDir.Path}"); + arguments = string.Concat(arguments, $" /ResultsDirectory:{TempDirectory.Path}"); arguments = string.Concat(arguments, " /testcasefilter:PassingTest"); var env = new Dictionary { - ["PROCDUMP_PATH"] = Path.Combine(_testEnvironment.PackageDirectory, @"procdump\0.0.1\bin"), + ["PROCDUMP_PATH"] = _procDumpPath }; InvokeVsTest(arguments, env); @@ -101,23 +112,22 @@ public void BlameDataCollectorShouldNotOutputDumpFileWhenNoCrashOccurs(RunnerInf [NetCoreTargetFrameworkDataSource] public void BlameDataCollectorShouldOutputDumpFileWhenNoCrashOccursButCollectAlwaysIsEnabled(RunnerInfo runnerInfo) { - using var tempDir = new TempDirectory(); SetTestEnvironment(_testEnvironment, runnerInfo); var assemblyPaths = BuildMultipleAssemblyPath("SimpleTestProject.dll").Trim('\"'); var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, string.Empty, runnerInfo.InIsolationValue); arguments = string.Concat(arguments, $" /Blame:CollectDump;CollectAlways=True"); - arguments = string.Concat(arguments, $" /ResultsDirectory:{tempDir.Path}"); + arguments = string.Concat(arguments, $" /ResultsDirectory:{TempDirectory.Path}"); arguments = string.Concat(arguments, " /testcasefilter:PassingTest"); var env = new Dictionary { - ["PROCDUMP_PATH"] = Path.Combine(_testEnvironment.PackageDirectory, @"procdump\0.0.1\bin"), + ["PROCDUMP_PATH"] = _procDumpPath }; InvokeVsTest(arguments, env); - Assert.IsTrue(StdOut.Contains(".dmp"), "it should collect dump, even if nothing crashed"); + StringAssert.Matches(StdOut, new Regex("\\.dmp"), "it should collect dump, even if nothing crashed"); } [TestMethod] @@ -133,7 +143,7 @@ public void HangDumpOnTimeout(RunnerInfo runnerInfo) var env = new Dictionary { - ["PROCDUMP_PATH"] = Path.Combine(_testEnvironment.PackageDirectory, @"procdump\0.0.1\bin"), + ["PROCDUMP_PATH"] = _procDumpPath }; InvokeVsTest(arguments, env); @@ -156,7 +166,7 @@ public void CrashDumpWhenThereIsNoTimeout(RunnerInfo runnerInfo) var env = new Dictionary { - ["PROCDUMP_PATH"] = Path.Combine(_testEnvironment.PackageDirectory, @"procdump\0.0.1\bin"), + ["PROCDUMP_PATH"] = _procDumpPath }; InvokeVsTest(arguments, env); @@ -179,7 +189,7 @@ public void CrashDumpOnExit(RunnerInfo runnerInfo) var env = new Dictionary { - ["PROCDUMP_PATH"] = Path.Combine(_testEnvironment.PackageDirectory, @"procdump\0.0.1\bin"), + ["PROCDUMP_PATH"] = _procDumpPath }; InvokeVsTest(arguments, env); @@ -200,7 +210,7 @@ public void CrashDumpOnStackOverflow(RunnerInfo runnerInfo) var env = new Dictionary { - ["PROCDUMP_PATH"] = Path.Combine(_testEnvironment.PackageDirectory, @"procdump\0.0.1\bin"), + ["PROCDUMP_PATH"] = _procDumpPath }; InvokeVsTest(arguments, env); @@ -313,14 +323,14 @@ private void ValidateDump(int expectedDumpCount = 1) } } - private void VaildateOutput(TempDirectory tempDir, string testName, bool validateDumpFile = false) + private void VaildateOutput(TempDirectory TempDirectory, string testName, bool validateDumpFile = false) { bool isSequenceAttachmentReceived = false; bool isDumpAttachmentReceived = false; bool isValid = false; StdErrorContains(testName); StdOutputContains("Sequence_"); - var resultFiles = Directory.GetFiles(tempDir.Path, "*", SearchOption.AllDirectories); + var resultFiles = Directory.GetFiles(TempDirectory.Path, "*", SearchOption.AllDirectories); foreach (var file in resultFiles) { @@ -335,9 +345,9 @@ private void VaildateOutput(TempDirectory tempDir, string testName, bool validat } } - Assert.IsTrue(isSequenceAttachmentReceived); - Assert.IsTrue(!validateDumpFile || isDumpAttachmentReceived); - Assert.IsTrue(isValid); + Assert.IsTrue(isSequenceAttachmentReceived, "Sequence attachment was not received."); + Assert.IsTrue(!validateDumpFile || isDumpAttachmentReceived, "Dump attachment was not received."); + Assert.IsTrue(isValid, "Sequence attachment is not valid."); } private bool IsValidXml(string xmlFilePath) diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/CUITTest.cs b/test/Microsoft.TestPlatform.AcceptanceTests/CUITTest.cs index 66559416ba..efc524ffde 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/CUITTest.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/CUITTest.cs @@ -18,20 +18,19 @@ public class CuitTest : AcceptanceTestBase public void CuitRunAllTests(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - CuitRunAll(runnerInfo.RunnerFramework); + CuitRunAll(runnerInfo); } - private void CuitRunAll(string runnerFramework) + private void CuitRunAll(RunnerInfo runnerInfo) { - if (runnerFramework.StartsWith("netcoreapp")) + if (runnerInfo.IsNetRunner) { - Assert.Inconclusive("CUIT tests are not supported with .Netcore runner."); + Assert.Inconclusive("CUIT tests are not supported with .NET Core runner."); return; } var assemblyAbsolutePath = _testEnvironment.GetTestAsset("CUITTestProject.dll", "net451"); - using var tempDir = new TempDirectory(); - var arguments = PrepareArguments(assemblyAbsolutePath, string.Empty, string.Empty, FrameworkArgValue, resultsDirectory: tempDir.Path); + var arguments = PrepareArguments(assemblyAbsolutePath, string.Empty, string.Empty, FrameworkArgValue, resultsDirectory: TempDirectory.Path); InvokeVsTest(arguments); ValidateSummaryStatus(1, 0, 0); diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/CodeCoverageAcceptanceTestBase.cs b/test/Microsoft.TestPlatform.AcceptanceTests/CodeCoverageAcceptanceTestBase.cs index 6aaf72b2b7..c7d3aaa1d7 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/CodeCoverageAcceptanceTestBase.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/CodeCoverageAcceptanceTestBase.cs @@ -59,7 +59,7 @@ protected XmlNode GetNode(XmlNode node, string type, string name) return node.SelectSingleNode($"//{type}[@name='{name}']"); } - protected XmlDocument GetXmlCoverage(string coverageResult, TempDirectory tempDirectory) + protected XmlDocument GetXmlCoverage(string coverageResult, TempDirectory TempDirectory) { var coverage = new XmlDocument(); @@ -70,7 +70,7 @@ protected XmlDocument GetXmlCoverage(string coverageResult, TempDirectory tempDi } var codeCoverageExe = GetCodeCoverageExePath(); - var output = Path.Combine(tempDirectory.Path, Guid.NewGuid().ToString() + ".xml"); + var output = Path.Combine(TempDirectory.Path, Guid.NewGuid().ToString() + ".xml"); var watch = new Stopwatch(); diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/CodeCoverageTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/CodeCoverageTests.cs index fafc3f4708..1dc4c3f618 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/CodeCoverageTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/CodeCoverageTests.cs @@ -208,10 +208,9 @@ public void CollectCodeCoverageSpecifyOutputFormatCoberturaOverrideRunSettingsCo private void CollectCodeCoverage(RunnerInfo runnerInfo, TestParameters testParameters) { - using var tempDir = new TempDirectory(); SetTestEnvironment(_testEnvironment, runnerInfo); - var arguments = CreateArguments(tempDir, runnerInfo, testParameters, out var trxFilePath); + var arguments = CreateArguments(TempDirectory, runnerInfo, testParameters, out var trxFilePath); InvokeVsTest(arguments); @@ -220,8 +219,8 @@ private void CollectCodeCoverage(RunnerInfo runnerInfo, TestParameters testParam testParameters.ExpectedSkippedTests, testParameters.ExpectedFailedTests); - var actualCoverageFile = GetCoverageFileNameFromTrx(trxFilePath, tempDir.Path); - Console.WriteLine($@"Coverage file: {actualCoverageFile} Results directory: {tempDir.Path} trxfile: {trxFilePath}"); + var actualCoverageFile = GetCoverageFileNameFromTrx(trxFilePath, TempDirectory.Path); + Console.WriteLine($@"Coverage file: {actualCoverageFile} Results directory: {TempDirectory.Path} trxfile: {trxFilePath}"); Assert.IsTrue(File.Exists(actualCoverageFile), "Coverage file not found: {0}", actualCoverageFile); if (testParameters.RunSettingsType == TestParameters.SettingsType.XmlOutput) @@ -237,7 +236,7 @@ private void CollectCodeCoverage(RunnerInfo runnerInfo, TestParameters testParam Assert.IsTrue(actualCoverageFile.EndsWith(".coverage", StringComparison.InvariantCultureIgnoreCase)); } - var coverageDocument = GetXmlCoverage(actualCoverageFile, tempDir); + var coverageDocument = GetXmlCoverage(actualCoverageFile, TempDirectory); if (testParameters.CheckSkipped) { AssertSkippedMethod(coverageDocument); @@ -247,7 +246,7 @@ private void CollectCodeCoverage(RunnerInfo runnerInfo, TestParameters testParam } private string CreateArguments( - TempDirectory tempDir, + TempDirectory TempDirectory, RunnerInfo runnerInfo, TestParameters testParameters, out string trxFilePath) @@ -257,14 +256,14 @@ private string CreateArguments( string traceDataCollectorDir = Path.Combine(IntegrationTestEnvironment.TestPlatformRootDirectory, "artifacts", IntegrationTestEnvironment.BuildConfiguration, "Microsoft.CodeCoverage"); - string diagFileName = Path.Combine(tempDir.Path, "diaglog.txt"); + string diagFileName = Path.Combine(TempDirectory.Path, "diaglog.txt"); var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, - FrameworkArgValue, runnerInfo.InIsolationValue, tempDir.Path); + FrameworkArgValue, runnerInfo.InIsolationValue, TempDirectory.Path); arguments = string.Concat(arguments, $" /Diag:{diagFileName}", $" /TestAdapterPath:{traceDataCollectorDir}"); arguments = string.Concat(arguments, $" /Platform:{testParameters.TargetPlatform}"); - trxFilePath = Path.Combine(tempDir.Path, Guid.NewGuid() + ".trx"); + trxFilePath = Path.Combine(TempDirectory.Path, Guid.NewGuid() + ".trx"); arguments = string.Concat(arguments, " /logger:trx;logfilename=" + trxFilePath); var defaultRunSettingsPath = Path.Combine( diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/DataCollectionTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/DataCollectionTests.cs index f400864e0f..b7907cbdd6 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/DataCollectionTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/DataCollectionTests.cs @@ -27,29 +27,28 @@ public class DataCollectionTests : AcceptanceTestBase public void ExecuteTestsWithDataCollection(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - using var tempDir = new TempDirectory(); var assemblyPaths = BuildMultipleAssemblyPath("SimpleTestProject2.dll").Trim('\"'); - string runSettings = GetRunsettingsFilePath(tempDir.Path); - string diagFileName = Path.Combine(tempDir.Path, "diaglog.txt"); + string runSettings = GetRunsettingsFilePath(TempDirectory.Path); + string diagFileName = Path.Combine(TempDirectory.Path, "diaglog.txt"); var extensionsPath = Path.Combine( _testEnvironment.TestAssetsPath, Path.GetFileNameWithoutExtension("OutOfProcDataCollector"), "bin", IntegrationTestEnvironment.BuildConfiguration, _testEnvironment.RunnerFramework); - var arguments = PrepareArguments(assemblyPaths, null, runSettings, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: tempDir.Path); + var arguments = PrepareArguments(assemblyPaths, null, runSettings, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: TempDirectory.Path); arguments = string.Concat(arguments, $" /Diag:{diagFileName}", $" /TestAdapterPath:{extensionsPath}"); var env = new Dictionary { - ["TEST_ASSET_SAMPLE_COLLECTOR_PATH"] = tempDir.Path, + ["TEST_ASSET_SAMPLE_COLLECTOR_PATH"] = TempDirectory.Path, }; InvokeVsTest(arguments, env); ValidateSummaryStatus(1, 1, 1); - VaildateDataCollectorOutput(tempDir.Path); + VaildateDataCollectorOutput(TempDirectory.Path); } [TestMethod] @@ -59,9 +58,8 @@ public void ExecuteTestsWithDataCollectionUsingCollectArgument(RunnerInfo runner { SetTestEnvironment(_testEnvironment, runnerInfo); - using var tempDir = new TempDirectory(); var assemblyPaths = BuildMultipleAssemblyPath("SimpleTestProject2.dll").Trim('\"'); - string diagFileName = Path.Combine(tempDir.Path, "diaglog.txt"); + string diagFileName = Path.Combine(TempDirectory.Path, "diaglog.txt"); var extensionsPath = Path.Combine( _testEnvironment.TestAssetsPath, Path.GetFileNameWithoutExtension("OutOfProcDataCollector"), @@ -69,18 +67,18 @@ public void ExecuteTestsWithDataCollectionUsingCollectArgument(RunnerInfo runner IntegrationTestEnvironment.BuildConfiguration, _testEnvironment.RunnerFramework); - var arguments = PrepareArguments(assemblyPaths, null, null, FrameworkArgValue, runnerInfo.InIsolationValue, tempDir.Path); + var arguments = PrepareArguments(assemblyPaths, null, null, FrameworkArgValue, runnerInfo.InIsolationValue, TempDirectory.Path); arguments = string.Concat(arguments, $" /Diag:{diagFileName}", $" /Collect:SampleDataCollector", $" /TestAdapterPath:{extensionsPath}"); var env = new Dictionary { - ["TEST_ASSET_SAMPLE_COLLECTOR_PATH"] = tempDir.Path, + ["TEST_ASSET_SAMPLE_COLLECTOR_PATH"] = TempDirectory.Path, }; InvokeVsTest(arguments, env); ValidateSummaryStatus(1, 1, 1); - VaildateDataCollectorOutput(tempDir.Path); + VaildateDataCollectorOutput(TempDirectory.Path); } [TestMethod] @@ -89,8 +87,7 @@ public void DataCollectorAssemblyLoadingShouldNotThrowErrorForNetCore(RunnerInfo { SetTestEnvironment(_testEnvironment, runnerInfo); - using var tempDir = new TempDirectory(); - var arguments = PrepareArguments(GetAssetFullPath("AppDomainGetAssembliesTestProject.dll", "netcoreapp2.1"), string.Empty, string.Empty, FrameworkArgValue, resultsDirectory: tempDir.Path); + var arguments = PrepareArguments(GetAssetFullPath("AppDomainGetAssembliesTestProject.dll", "netcoreapp2.1"), string.Empty, string.Empty, FrameworkArgValue, resultsDirectory: TempDirectory.Path); InvokeVsTest(arguments); ValidateSummaryStatus(1, 0, 0); @@ -103,8 +100,7 @@ public void DataCollectorAssemblyLoadingShouldNotThrowErrorForFullFramework(Runn { SetTestEnvironment(_testEnvironment, runnerInfo); - using var tempDir = new TempDirectory(); - var arguments = PrepareArguments(GetAssetFullPath("AppDomainGetAssembliesTestProject.dll"), string.Empty, string.Empty, FrameworkArgValue, resultsDirectory: tempDir.Path); + var arguments = PrepareArguments(GetAssetFullPath("AppDomainGetAssembliesTestProject.dll"), string.Empty, string.Empty, FrameworkArgValue, resultsDirectory: TempDirectory.Path); InvokeVsTest(arguments); ValidateSummaryStatus(1, 0, 0); @@ -117,18 +113,17 @@ public void DataCollectorAttachmentProcessor(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - using var tempDir = new TempDirectory(); var assemblyPath = BuildMultipleAssemblyPath("SimpleTestProject.dll").Trim('\"'); var secondAssemblyPath = BuildMultipleAssemblyPath("SimpleTestProject2.dll").Trim('\"'); - string runSettings = GetRunsettingsFilePath(tempDir.Path); - string diagFileName = Path.Combine(tempDir.Path, "diaglog.txt"); + string runSettings = GetRunsettingsFilePath(TempDirectory.Path); + string diagFileName = Path.Combine(TempDirectory.Path, "diaglog.txt"); var extensionsPath = Path.Combine( _testEnvironment.TestAssetsPath, Path.GetFileNameWithoutExtension("AttachmentProcessorDataCollector"), "bin", IntegrationTestEnvironment.BuildConfiguration, "netstandard2.0"); - var arguments = PrepareArguments(new string[] { assemblyPath, secondAssemblyPath }, null, runSettings, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: tempDir.Path); + var arguments = PrepareArguments(new string[] { assemblyPath, secondAssemblyPath }, null, runSettings, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: TempDirectory.Path); arguments = string.Concat(arguments, $" /Diag:{diagFileName}", $" /TestAdapterPath:{extensionsPath}"); XElement runSettingsXml = XElement.Load(runSettings); @@ -148,14 +143,14 @@ public void DataCollectorAttachmentProcessor(RunnerInfo runnerInfo) var env = new Dictionary { - ["SampleDataCollectorTempPath"] = tempDir.Path, + ["SampleDataCollectorTempPath"] = TempDirectory.Path, }; InvokeVsTest(arguments, env); ValidateSummaryStatus(2, 2, 2); - string mergedFile = Directory.GetFiles(tempDir.Path, "MergedFile.txt", SearchOption.AllDirectories).Single(); + string mergedFile = Directory.GetFiles(TempDirectory.Path, "MergedFile.txt", SearchOption.AllDirectories).Single(); var fileContent = new List(); using (var streamReader = new StreamReader(mergedFile)) { @@ -169,7 +164,7 @@ public void DataCollectorAttachmentProcessor(RunnerInfo runnerInfo) Assert.AreEqual(2, fileContent.Distinct().Count()); - var dataCollectorsLogs = Directory.GetFiles(tempDir.Path, "*.datacollector.*", SearchOption.TopDirectoryOnly); + var dataCollectorsLogs = Directory.GetFiles(TempDirectory.Path, "*.datacollector.*", SearchOption.TopDirectoryOnly); Assert.AreEqual(2, dataCollectorsLogs.Distinct().Count()); foreach (var dataCollectorLogFile in dataCollectorsLogs) { diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/DebugAssertTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/DebugAssertTests.cs index ed293c7e61..fba250c5a2 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/DebugAssertTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/DebugAssertTests.cs @@ -20,9 +20,8 @@ public void RunningTestWithAFailingDebugAssertDoesNotCrashTheHostingProcess(Runn // is to not crash the process when we are running in debug, and debugger is attached SetTestEnvironment(_testEnvironment, runnerInfo); - using var tempDir = new TempDirectory(); var assemblyPath = BuildMultipleAssemblyPath("CrashingOnDebugAssertTestProject.dll").Trim('\"'); - var arguments = PrepareArguments(assemblyPath, null, null, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: tempDir.Path); + var arguments = PrepareArguments(assemblyPath, null, null, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: TempDirectory.Path); InvokeVsTest(arguments); // this will have failed tests when our trace listener works and crash the testhost process when it does not diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/DeprecateExtensionsPathWarningTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/DeprecateExtensionsPathWarningTests.cs index acb36c8e7a..4c10549775 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/DeprecateExtensionsPathWarningTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/DeprecateExtensionsPathWarningTests.cs @@ -20,18 +20,6 @@ public class DeprecateExtensionsPathWarningTests : AcceptanceTestBase private IList _adapterDependencies; private IList _copiedFiles; - private string BuildConfiguration - { - get - { -#if DEBUG - return "Debug"; -#else - return "Release"; -#endif - } - } - [TestCleanup] public void Cleanup() { @@ -73,8 +61,7 @@ public void CopyAdapterToExtensions() [TestMethod] public void VerifyDeprecatedWarningIsThrownWhenAdaptersPickedFromExtensionDirectory() { - using var tempDir = new TempDirectory(); - var arguments = PrepareArguments(GetSampleTestAssembly(), null, null, FrameworkArgValue, resultsDirectory: tempDir.Path); + var arguments = PrepareArguments(GetSampleTestAssembly(), null, null, FrameworkArgValue, resultsDirectory: TempDirectory.Path); InvokeVsTest(arguments); StdOutputContains("Adapter lookup is being changed, please follow"); diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/DifferentTestFrameworkSimpleTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/DifferentTestFrameworkSimpleTests.cs index 76b2cabe06..fc3209099a 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/DifferentTestFrameworkSimpleTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/DifferentTestFrameworkSimpleTests.cs @@ -20,9 +20,8 @@ public class DifferentTestFrameworkSimpleTests : AcceptanceTestBase public void ChutzpahRunAllTestExecution(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - using var tempDir = new TempDirectory(); var testJSFileAbsolutePath = Path.Combine(_testEnvironment.TestAssetsPath, "test.js"); - var arguments = PrepareArguments(testJSFileAbsolutePath, GetTestAdapterPath(UnitTestFramework.Chutzpah), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: tempDir.Path); + var arguments = PrepareArguments(testJSFileAbsolutePath, GetTestAdapterPath(UnitTestFramework.Chutzpah), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: TempDirectory.Path); InvokeVsTest(arguments); ValidateSummaryStatus(1, 1, 0); @@ -35,7 +34,7 @@ public void ChutzpahRunAllTestExecution(RunnerInfo runnerInfo) public void CPPRunAllTestExecutionNetFramework(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - CppRunAllTests(runnerInfo.RunnerFramework, "x86"); + CppRunAllTests("x86"); } @@ -47,7 +46,7 @@ public void CPPRunAllTestExecutionNetFramework(RunnerInfo runnerInfo) public void CPPRunAllTestExecutionPlatformx64NetFramework(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - CppRunAllTests(runnerInfo.RunnerFramework, "x64"); + CppRunAllTests("x64"); } [TestMethod] @@ -58,7 +57,7 @@ public void CPPRunAllTestExecutionPlatformx64NetFramework(RunnerInfo runnerInfo) public void CPPRunAllTestExecutionPlatformx64Net(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - CppRunAllTests(runnerInfo.RunnerFramework, "x64"); + CppRunAllTests("x64"); } [TestMethod] @@ -66,9 +65,18 @@ public void CPPRunAllTestExecutionPlatformx64Net(RunnerInfo runnerInfo) [NetFullTargetFrameworkDataSource] public void WebTestRunAllTestsWithRunSettings(RunnerInfo runnerInfo) { - using var tempDir = new TempDirectory(); + // This test works on server but not locally, and it fails with + // Exception: Method not found: 'Int64 Microsoft.VisualStudio.TestTools.WebTesting.WebTestRequestStatistics.get_MillisecondsSincePageComplete()'. + // The dll is + + //if (!IsCI) + //{ + // Assert.Inconclusive("Web load tests need special workloads and setup locally"); + //} + + SetTestEnvironment(_testEnvironment, runnerInfo); - var runSettingsFilePath = Path.Combine(tempDir.Path, Guid.NewGuid() + ".runsettings"); + var runSettingsFilePath = Path.Combine(TempDirectory.Path, Guid.NewGuid() + ".runsettings"); //test the iterationCount setting for WebTestRunConfiguration in run settings var runSettingsXml = $@" @@ -81,9 +89,37 @@ public void WebTestRunAllTestsWithRunSettings(RunnerInfo runnerInfo) CreateRunSettingsFile(runSettingsFilePath, runSettingsXml); - //minWebTestResultFileSizeInKB is set to 150 here as the web test has a iteration count set to 5 //therefore, the test will run for 5 iterations resulting in web test result file size of at least 150 KB - WebTestRunAllTests(runnerInfo.RunnerFramework, runSettingsFilePath, 150); + var minWebTestResultFileSizeInKB = 150; + if (runnerInfo.IsNetRunner) + { + Assert.Inconclusive("WebTests tests not supported with .Netcore runner."); + return; + } + + string assemblyRelativePath = + @"microsoft.testplatform.qtools.assets\2.0.0\contentFiles\any\any\WebTestAssets\WebTest1.webtest"; + + var assemblyAbsolutePath = Path.Combine(_testEnvironment.PackageDirectory, assemblyRelativePath); + using var resultsDirectory = TempDirectory; + var arguments = PrepareArguments( + assemblyAbsolutePath, + string.Empty, + runSettingsFilePath, FrameworkArgValue, string.Empty, resultsDirectory.Path); + + InvokeVsTest(arguments); + ValidateSummaryStatus(1, 0, 0); + + if (minWebTestResultFileSizeInKB > 0) + { + var dirInfo = new DirectoryInfo(resultsDirectory.Path); + var webtestResultFile = "WebTest1.webtestResult"; + var files = dirInfo.GetFiles(webtestResultFile, SearchOption.AllDirectories); + Assert.IsTrue(files.Length > 0, $"File {webtestResultFile} not found under results directory {resultsDirectory}"); + + var fileSizeInKB = files[0].Length / 1024; + Assert.IsTrue(fileSizeInKB > minWebTestResultFileSizeInKB, $"Size of the file {webtestResultFile} is {fileSizeInKB} KB. It is not greater than {minWebTestResultFileSizeInKB} KB indicating iterationCount in run settings not honored."); + } } [TestMethod] @@ -92,7 +128,21 @@ public void WebTestRunAllTestsWithRunSettings(RunnerInfo runnerInfo) public void CodedWebTestRunAllTests(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - CodedWebTestRunAllTests(runnerInfo.RunnerFramework); + if (runnerInfo.IsNetRunner) + { + Assert.Inconclusive("WebTests tests not supported with .Netcore runner."); + return; + } + + string assemblyRelativePath = @"microsoft.testplatform.qtools.assets\2.0.0\contentFiles\any\any\WebTestAssets\BingWebTest.dll"; + var assemblyAbsolutePath = Path.Combine(_testEnvironment.PackageDirectory, assemblyRelativePath); + var arguments = PrepareArguments( + assemblyAbsolutePath, + string.Empty, + string.Empty, FrameworkArgValue, resultsDirectory: TempDirectory.Path); + + InvokeVsTest(arguments); + ValidateSummaryStatus(1, 0, 0); } [TestMethod] @@ -102,12 +152,11 @@ public void NUnitRunAllTestExecution(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - using var tempDir = new TempDirectory(); var arguments = PrepareArguments( - GetAssetFullPath("NUTestProject.dll"), - GetTestAdapterPath(UnitTestFramework.NUnit), - string.Empty, FrameworkArgValue, - runnerInfo.InIsolationValue, tempDir.Path); +GetAssetFullPath("NUTestProject.dll"), +GetTestAdapterPath(UnitTestFramework.NUnit), +string.Empty, FrameworkArgValue, +runnerInfo.InIsolationValue, TempDirectory.Path); InvokeVsTest(arguments); ValidateSummaryStatus(1, 1, 0); } @@ -118,7 +167,6 @@ public void NUnitRunAllTestExecution(RunnerInfo runnerInfo) public void XUnitRunAllTestExecution(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - using var tempDir = new TempDirectory(); string testAssemblyPath; // Xunit >= 2.2 won't support net451, Minimum target framework it supports is net452. @@ -135,77 +183,21 @@ public void XUnitRunAllTestExecution(RunnerInfo runnerInfo) testAssemblyPath, GetTestAdapterPath(UnitTestFramework.XUnit), string.Empty, FrameworkArgValue, - runnerInfo.InIsolationValue, tempDir.Path); + runnerInfo.InIsolationValue, TempDirectory.Path); InvokeVsTest(arguments); ValidateSummaryStatus(1, 1, 0); } - private void CppRunAllTests(string runnerFramework, string platform) + private void CppRunAllTests(string platform) { - using var tempDir = new TempDirectory(); - string assemblyRelativePathFormat = - @"microsoft.testplatform.testasset.nativecpp\2.0.0\contentFiles\any\any\{0}\Microsoft.TestPlatform.TestAsset.NativeCPP.dll"; + string assemblyRelativePathFormat = @"microsoft.testplatform.testasset.nativecpp\2.0.0\contentFiles\any\any\{0}\Microsoft.TestPlatform.TestAsset.NativeCPP.dll"; var assemblyRelativePath = platform.Equals("x64", StringComparison.OrdinalIgnoreCase) ? string.Format(assemblyRelativePathFormat, platform) : string.Format(assemblyRelativePathFormat, ""); var assemblyAbsolutePath = Path.Combine(_testEnvironment.PackageDirectory, assemblyRelativePath); - var arguments = PrepareArguments(assemblyAbsolutePath, string.Empty, string.Empty, FrameworkArgValue, _testEnvironment.InIsolationValue, resultsDirectory: tempDir.Path); + var arguments = PrepareArguments(assemblyAbsolutePath, string.Empty, string.Empty, FrameworkArgValue, _testEnvironment.InIsolationValue, resultsDirectory: TempDirectory.Path); InvokeVsTest(arguments); ValidateSummaryStatus(1, 1, 0); } - - private void WebTestRunAllTests(string runnerFramework, string runSettingsFilePath = null, int minWebTestResultFileSizeInKB = 0) - { - if (runnerFramework.StartsWith("netcoreapp")) - { - Assert.Inconclusive("WebTests tests not supported with .Netcore runner."); - return; - } - - string assemblyRelativePath = - @"microsoft.testplatform.qtools.assets\2.0.0\contentFiles\any\any\WebTestAssets\WebTest1.webtest"; - - var assemblyAbsolutePath = Path.Combine(_testEnvironment.PackageDirectory, assemblyRelativePath); - using var resultsDirectory = new TempDirectory(); - var arguments = PrepareArguments( - assemblyAbsolutePath, - string.Empty, - runSettingsFilePath, FrameworkArgValue, string.Empty, resultsDirectory.Path); - - InvokeVsTest(arguments); - ValidateSummaryStatus(1, 0, 0); - - if (minWebTestResultFileSizeInKB > 0) - { - var dirInfo = new DirectoryInfo(resultsDirectory.Path); - var webtestResultFile = "WebTest1.webtestResult"; - var files = dirInfo.GetFiles(webtestResultFile, SearchOption.AllDirectories); - Assert.IsTrue(files.Length > 0, $"File {webtestResultFile} not found under results directory {resultsDirectory}"); - - var fileSizeInKB = files[0].Length / 1024; - Assert.IsTrue(fileSizeInKB > minWebTestResultFileSizeInKB, $"Size of the file {webtestResultFile} is {fileSizeInKB} KB. It is not greater than {minWebTestResultFileSizeInKB} KB indicating iterationCount in run settings not honored."); - } - } - - private void CodedWebTestRunAllTests(string runnerFramework) - { - if (runnerFramework.StartsWith("netcoreapp")) - { - Assert.Inconclusive("WebTests tests not supported with .Netcore runner."); - return; - } - - using var tempDir = new TempDirectory(); - string assemblyRelativePath = - @"microsoft.testplatform.qtools.assets\2.0.0\contentFiles\any\any\WebTestAssets\BingWebTest.dll"; - var assemblyAbsolutePath = Path.Combine(_testEnvironment.PackageDirectory, assemblyRelativePath); - var arguments = PrepareArguments( - assemblyAbsolutePath, - string.Empty, - string.Empty, FrameworkArgValue, resultsDirectory: tempDir.Path); - - InvokeVsTest(arguments); - ValidateSummaryStatus(1, 0, 0); - } } diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/DisableAppdomainTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/DisableAppdomainTests.cs index d78c6fe7e8..78add6fe64 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/DisableAppdomainTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/DisableAppdomainTests.cs @@ -26,7 +26,7 @@ public void DisableAppdomainTest(RunnerInfo runnerInfo) var diableAppdomainTest1 = _testEnvironment.GetTestAsset("DisableAppdomainTest1.dll", "net451"); var diableAppdomainTest2 = _testEnvironment.GetTestAsset("DisableAppdomainTest2.dll", "net451"); - RunTests(runnerInfo.RunnerFramework, string.Format("{0}\" \"{1}", diableAppdomainTest1, diableAppdomainTest2), 2); + RunTests(runnerInfo, string.Format("{0}\" \"{1}", diableAppdomainTest1, diableAppdomainTest2), 2); } [TestMethod] @@ -38,12 +38,12 @@ public void NewtonSoftDependencyWithDisableAppdomainTest(RunnerInfo runnerInfo) var newtonSoftDependnecyTest = _testEnvironment.GetTestAsset("NewtonSoftDependency.dll", "net451"); - RunTests(runnerInfo.RunnerFramework, newtonSoftDependnecyTest, 1); + RunTests(runnerInfo, newtonSoftDependnecyTest, 1); } - private void RunTests(string runnerFramework, string testAssembly, int passedTestCount) + private void RunTests(RunnerInfo runnerInfo, string testAssembly, int passedTestCount) { - if (runnerFramework.StartsWith("netcoreapp")) + if (runnerInfo.IsNetRunner) { Assert.Inconclusive("This test is not meant for .netcore."); return; @@ -54,20 +54,19 @@ private void RunTests(string runnerFramework, string testAssembly, int passedTes { "DisableAppDomain", "true" } }; - using var tempDir = new TempDirectory(); var arguments = PrepareArguments( testAssembly, string.Empty, - GetRunsettingsFilePath(tempDir, runConfigurationDictionary), - FrameworkArgValue, resultsDirectory: tempDir.Path); + GetRunsettingsFilePath(TempDirectory, runConfigurationDictionary), + FrameworkArgValue, resultsDirectory: TempDirectory.Path); InvokeVsTest(arguments); ValidateSummaryStatus(passedTestCount, 0, 0); } - private string GetRunsettingsFilePath(TempDirectory tempDir, Dictionary runConfigurationDictionary) + private string GetRunsettingsFilePath(TempDirectory TempDirectory, Dictionary runConfigurationDictionary) { - var runsettingsPath = Path.Combine(tempDir.Path, "test_" + Guid.NewGuid() + ".runsettings"); + var runsettingsPath = Path.Combine(TempDirectory.Path, "test_" + Guid.NewGuid() + ".runsettings"); CreateRunSettingsFile(runsettingsPath, runConfigurationDictionary); return runsettingsPath; } diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/DiscoveryTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/DiscoveryTests.cs index 8bc5843ed2..2f6087aa17 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/DiscoveryTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/DiscoveryTests.cs @@ -59,14 +59,13 @@ public void MultipleSourcesDiscoverAllTests(RunnerInfo runnerInfo) [NetFullTargetFrameworkDataSource(inIsolation: true, inProcess: true)] public void DiscoverFullyQualifiedTests(RunnerInfo runnerInfo) { - using var tempDir = new TempDirectory(); - var dummyFilePath = Path.Combine(tempDir.Path, $"{Guid.NewGuid()}.txt"); + var dummyFilePath = Path.Combine(TempDirectory.Path, $"{Guid.NewGuid()}.txt"); SetTestEnvironment(_testEnvironment, runnerInfo); var listOfTests = new[] { "SampleUnitTestProject.UnitTest1.PassingTest", "SampleUnitTestProject.UnitTest1.FailingTest", "SampleUnitTestProject.UnitTest1.SkippingTest" }; - var arguments = PrepareArguments(GetSampleTestAssembly(), GetTestAdapterPath(), string.Empty, FrameworkArgValue, _testEnvironment.InIsolationValue, resultsDirectory: tempDir.Path); + var arguments = PrepareArguments(GetSampleTestAssembly(), GetTestAdapterPath(), string.Empty, FrameworkArgValue, _testEnvironment.InIsolationValue, resultsDirectory: TempDirectory.Path); arguments = string.Concat(arguments, " /ListFullyQualifiedTests", " /ListTestsTargetPath:\"" + dummyFilePath + "\""); InvokeVsTest(arguments); @@ -80,10 +79,9 @@ public void DiscoverFullyQualifiedTests(RunnerInfo runnerInfo) public void DiscoverTestsShouldShowProperWarningIfNoTestsOnTestCaseFilter(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - using var tempDir = new TempDirectory(); var assetFullPath = GetAssetFullPath("SimpleTestProject2.dll"); - var arguments = PrepareArguments(assetFullPath, GetTestAdapterPath(), string.Empty, FrameworkArgValue, _testEnvironment.InIsolationValue, resultsDirectory: tempDir.Path); + var arguments = PrepareArguments(assetFullPath, GetTestAdapterPath(), string.Empty, FrameworkArgValue, _testEnvironment.InIsolationValue, resultsDirectory: TempDirectory.Path); arguments = string.Concat(arguments, " /listtests"); arguments = string.Concat(arguments, " /testcasefilter:NonExistTestCaseName"); arguments = string.Concat(arguments, " /logger:\"console;prefix=true\""); diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/DotnetArchitectureSwitchTests.Windows.cs b/test/Microsoft.TestPlatform.AcceptanceTests/DotnetArchitectureSwitchTests.Windows.cs index 0102a4e692..cbb764e423 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/DotnetArchitectureSwitchTests.Windows.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/DotnetArchitectureSwitchTests.Windows.cs @@ -1,17 +1,15 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT license. See LICENSE file in the project root for full license information. +#if !NET451 using System.Collections.Generic; using System.IO; using System.Linq; -using Microsoft.TestPlatform.TestUtilities; using Microsoft.VisualStudio.TestTools.UnitTesting; using Newtonsoft.Json.Linq; -#if !NET451 - #nullable disable namespace Microsoft.TestPlatform.AcceptanceTests; @@ -25,12 +23,12 @@ public class DotnetArchitectureSwitchTestsWindowsOnly : AcceptanceTestBase [DataRow("X86", "X64")] public void Use_EnvironmentVariables(string architectureFrom, string architectureTo) { - using var workSpace = new TempDirectory(); + using var workSpace = TempDirectory; string dotnetPath = GetDownloadedDotnetMuxerFromTools(architectureFrom); string dotnetPathTo = GetDownloadedDotnetMuxerFromTools(architectureTo); var vstestConsolePath = GetDotnetRunnerPath(); var dotnetRunnerPath = workSpace.CreateDirectory("dotnetrunner"); - workSpace.CopyAll(new DirectoryInfo(Path.GetDirectoryName(vstestConsolePath)), dotnetRunnerPath); + workSpace.CopyDirectory(new DirectoryInfo(Path.GetDirectoryName(vstestConsolePath)), dotnetRunnerPath); // Patch the runner string sdkVersion = GetLatestSdkVersion(dotnetPath); diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/DotnetArchitectureSwitchTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/DotnetArchitectureSwitchTests.cs index 2e3e384ff6..e2d11506cf 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/DotnetArchitectureSwitchTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/DotnetArchitectureSwitchTests.cs @@ -1,6 +1,8 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT license. See LICENSE file in the project root for full license information. +#if !NET451 + using System; using System.Collections.Generic; using System.IO; @@ -10,8 +12,6 @@ using Microsoft.TestPlatform.TestUtilities; using Microsoft.VisualStudio.TestTools.UnitTesting; -#if !NET451 - #nullable disable namespace Microsoft.TestPlatform.AcceptanceTests; diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/EventLogCollectorTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/EventLogCollectorTests.cs index 4b140fd35e..b7e80ebf5e 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/EventLogCollectorTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/EventLogCollectorTests.cs @@ -26,16 +26,15 @@ public class EventLogCollectorTests : AcceptanceTestBase public void EventLogDataCollectorShoudCreateLogFileHavingEvents(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - using var tempDir = new TempDirectory(); var assemblyPaths = _testEnvironment.GetTestAsset("EventLogUnitTestProject.dll"); - string runSettings = GetRunsettingsFilePath(tempDir); - var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), runSettings, FrameworkArgValue, resultsDirectory: tempDir.Path); + string runSettings = GetRunsettingsFilePath(TempDirectory); + var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), runSettings, FrameworkArgValue, resultsDirectory: TempDirectory.Path); InvokeVsTest(arguments); ValidateSummaryStatus(3, 0, 0); - VaildateDataCollectorOutput(tempDir); + VaildateDataCollectorOutput(TempDirectory); StdOutputDoesNotContains("An exception occurred while collecting final entries from the event log"); StdErrorDoesNotContains("event log has encountered an exception, some events might get lost"); StdOutputDoesNotContains("event log may have been cleared during collection; some events may not have been collected"); @@ -49,10 +48,9 @@ public void EventLogDataCollectorShoudCreateLogFileWithoutEventsIfEventsAreNotLo { SetTestEnvironment(_testEnvironment, runnerInfo); var assemblyPaths = _testEnvironment.GetTestAsset("SimpleTestProject.dll"); - using var tempDir = new TempDirectory(); - string runSettings = GetRunsettingsFilePath(tempDir); - var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), runSettings, FrameworkArgValue, resultsDirectory: tempDir.Path); + string runSettings = GetRunsettingsFilePath(TempDirectory); + var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), runSettings, FrameworkArgValue, resultsDirectory: TempDirectory.Path); InvokeVsTest(arguments); @@ -63,9 +61,9 @@ public void EventLogDataCollectorShoudCreateLogFileWithoutEventsIfEventsAreNotLo StdErrorDoesNotContains("Unable to read event log"); } - private string GetRunsettingsFilePath(TempDirectory tempDirectory) + private string GetRunsettingsFilePath(TempDirectory TempDirectory) { - var runsettingsPath = Path.Combine(tempDirectory.Path, "test_" + Guid.NewGuid() + ".runsettings"); + var runsettingsPath = Path.Combine(TempDirectory.Path, "test_" + Guid.NewGuid() + ".runsettings"); string runSettingsXml = @" @@ -87,10 +85,10 @@ private string GetRunsettingsFilePath(TempDirectory tempDirectory) return runsettingsPath; } - private void VaildateDataCollectorOutput(TempDirectory tempDir) + private void VaildateDataCollectorOutput(TempDirectory TempDirectory) { // Verify attachments - var di = new DirectoryInfo(tempDir.Path); + var di = new DirectoryInfo(TempDirectory.Path); var resultFiles = di.EnumerateFiles("Event Log.xml", SearchOption.AllDirectories) .OrderBy(d => d.CreationTime) .Select(d => d.FullName) diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/ExecutionTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/ExecutionTests.cs index accc13eada..b88bf2e02d 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/ExecutionTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/ExecutionTests.cs @@ -59,13 +59,12 @@ public void RunMultipleTestAssembliesWithoutTestAdapterPath(RunnerInfo runnerInf public void RunMultipleTestAssembliesInParallel(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - using var tempDir = new TempDirectory(); var assemblyPaths = BuildMultipleAssemblyPath("SimpleTestProject.dll", "SimpleTestProject2.dll").Trim('\"'); - var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: tempDir.Path); + var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: TempDirectory.Path); arguments = string.Concat(arguments, " /Parallel"); arguments = string.Concat(arguments, " /Platform:x86"); - arguments += GetDiagArg(tempDir.Path); + arguments += GetDiagArg(TempDirectory.Path); // for the desktop we will run testhost.x86 in two copies, but for core // we will run a combination of testhost.x86 and dotnet, where the dotnet will be @@ -78,7 +77,7 @@ public void RunMultipleTestAssembliesInParallel(RunnerInfo runnerInfo) InvokeVsTest(arguments); - AssertExpectedNumberOfHostProcesses(expectedNumOfProcessCreated, tempDir.Path, testHostProcessNames); + AssertExpectedNumberOfHostProcesses(expectedNumOfProcessCreated, TempDirectory.Path, testHostProcessNames); ValidateSummaryStatus(2, 2, 2); ExitCodeEquals(1); // failing tests } @@ -89,11 +88,10 @@ public void RunMultipleTestAssembliesInParallel(RunnerInfo runnerInfo) public void TestSessionTimeOutTests(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - using var tempDir = new TempDirectory(); var assemblyPaths = BuildMultipleAssemblyPath("SimpleTestProject3.dll").Trim('\"'); - var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: tempDir.Path); + var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: TempDirectory.Path); arguments = string.Concat(arguments, " /TestCaseFilter:TestSessionTimeoutTest"); // set TestSessionTimeOut = 7 sec @@ -111,10 +109,9 @@ public void TestSessionTimeOutTests(RunnerInfo runnerInfo) public void TestPlatformShouldBeCompatibleWithOldTestHost(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - using var tempDir = new TempDirectory(); var assemblyPaths = BuildMultipleAssemblyPath("SampleProjectWithOldTestHost.dll").Trim('\"'); - var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: tempDir.Path); + var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: TempDirectory.Path); InvokeVsTest(arguments); @@ -128,10 +125,9 @@ public void TestPlatformShouldBeCompatibleWithOldTestHost(RunnerInfo runnerInfo) public void WorkingDirectoryIsSourceDirectory(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - using var tempDir = new TempDirectory(); var assemblyPaths = BuildMultipleAssemblyPath("SimpleTestProject3.dll").Trim('\"'); - var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: tempDir.Path); + var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: TempDirectory.Path); arguments = string.Concat(arguments, " /tests:WorkingDirectoryTest"); InvokeVsTest(arguments); @@ -146,7 +142,6 @@ public void WorkingDirectoryIsSourceDirectory(RunnerInfo runnerInfo) public void StackOverflowExceptionShouldBeLoggedToConsoleAndDiagLogFile(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - using var tempDir = new TempDirectory(); if (IntegrationTestEnvironment.BuildConfiguration.Equals("release", StringComparison.OrdinalIgnoreCase)) { @@ -155,11 +150,11 @@ public void StackOverflowExceptionShouldBeLoggedToConsoleAndDiagLogFile(RunnerIn return; } - var diagLogFilePath = Path.Combine(tempDir.Path, $"std_error_log_{Guid.NewGuid()}.txt"); + var diagLogFilePath = Path.Combine(TempDirectory.Path, $"std_error_log_{Guid.NewGuid()}.txt"); File.Delete(diagLogFilePath); var assemblyPaths = BuildMultipleAssemblyPath("SimpleTestProject3.dll").Trim('\"'); - var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: tempDir.Path); + var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: TempDirectory.Path); arguments = string.Concat(arguments, " /testcasefilter:ExitWithStackoverFlow"); arguments = string.Concat(arguments, $" /diag:{diagLogFilePath}"); @@ -183,14 +178,13 @@ public void StackOverflowExceptionShouldBeLoggedToConsoleAndDiagLogFile(RunnerIn public void UnhandleExceptionExceptionShouldBeLoggedToDiagLogFile(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - using var tempDir = new TempDirectory(); - var diagLogFilePath = Path.Combine(tempDir.Path, $"std_error_log_{Guid.NewGuid()}.txt"); + var diagLogFilePath = Path.Combine(TempDirectory.Path, $"std_error_log_{Guid.NewGuid()}.txt"); File.Delete(diagLogFilePath); var assemblyPaths = BuildMultipleAssemblyPath("SimpleTestProject3.dll").Trim('\"'); - var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: tempDir.Path); + var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: TempDirectory.Path); arguments = string.Concat(arguments, " /testcasefilter:ExitwithUnhandleException"); arguments = string.Concat(arguments, $" /diag:{diagLogFilePath}"); @@ -207,12 +201,11 @@ public void UnhandleExceptionExceptionShouldBeLoggedToDiagLogFile(RunnerInfo run public void IncompatibleSourcesWarningShouldBeDisplayedInTheConsole(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - using var tempDir = new TempDirectory(); var expectedWarningContains = @"Following DLL(s) do not match current settings, which are .NETFramework,Version=v4.5.1 framework and X86 platform. SimpleTestProject3.dll is built for Framework .NETFramework,Version=v4.5.1 and Platform X64"; var assemblyPaths = BuildMultipleAssemblyPath("SimpleTestProject3.dll", "SimpleTestProjectx86.dll").Trim('\"'); - var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: tempDir.Path); + var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: TempDirectory.Path); arguments = string.Concat(arguments, " /testcasefilter:PassingTestx86"); InvokeVsTest(arguments); @@ -230,12 +223,11 @@ public void IncompatibleSourcesWarningShouldBeDisplayedInTheConsole(RunnerInfo r public void NoIncompatibleSourcesWarningShouldBeDisplayedInTheConsole(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - using var tempDir = new TempDirectory(); var expectedWarningContains = @"Following DLL(s) do not match current settings, which are .NETFramework,Version=v4.5.1 framework and X86 platform. SimpleTestProjectx86 is built for Framework .NETFramework,Version=v4.5.1 and Platform X86"; var assemblyPaths = BuildMultipleAssemblyPath("SimpleTestProjectx86.dll"); - var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: tempDir.Path); + var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: TempDirectory.Path); InvokeVsTest(arguments); @@ -251,12 +243,11 @@ public void NoIncompatibleSourcesWarningShouldBeDisplayedInTheConsole(RunnerInfo public void IncompatibleSourcesWarningShouldBeDisplayedInTheConsoleOnlyWhenRunningIn32BitOS(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - using var tempDir = new TempDirectory(); var expectedWarningContains = @"Following DLL(s) do not match current settings, which are .NETFramework,Version=v4.5.1 framework and X86 platform. SimpleTestProject2.dll is built for Framework .NETFramework,Version=v4.5.1 and Platform X64"; var assemblyPaths = BuildMultipleAssemblyPath("SimpleTestProject2.dll"); - var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: tempDir.Path); + var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: TempDirectory.Path); InvokeVsTest(arguments); @@ -281,11 +272,10 @@ public void IncompatibleSourcesWarningShouldBeDisplayedInTheConsoleOnlyWhenRunni public void ExitCodeShouldReturnOneWhenTreatNoTestsAsErrorParameterSetToTrueAndNoTestMatchesFilter(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - using var tempDir = new TempDirectory(); var assemblyPaths = BuildMultipleAssemblyPath("SimpleTestProject2.dll").Trim('\"'); - var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: tempDir.Path); + var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: TempDirectory.Path); // Setting /TestCaseFilter to the test name, which does not exists in the assembly, so we will have 0 tests executed arguments = string.Concat(arguments, " /TestCaseFilter:TestNameThatMatchesNoTestInTheAssembly"); @@ -302,10 +292,9 @@ public void ExitCodeShouldReturnOneWhenTreatNoTestsAsErrorParameterSetToTrueAndN public void ExitCodeShouldReturnZeroWhenTreatNoTestsAsErrorParameterSetToFalseAndNoTestMatchesFilter(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - using var tempDir = new TempDirectory(); var assemblyPaths = BuildMultipleAssemblyPath("SimpleTestProject2.dll").Trim('\"'); - var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: tempDir.Path); + var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: TempDirectory.Path); // Setting /TestCaseFilter to the test name, which does not exists in the assembly, so we will have 0 tests executed arguments = string.Concat(arguments, " /TestCaseFilter:TestNameThatMatchesNoTestInTheAssembly"); @@ -322,11 +311,10 @@ public void ExitCodeShouldReturnZeroWhenTreatNoTestsAsErrorParameterSetToFalseAn public void ExitCodeShouldNotDependOnTreatNoTestsAsErrorTrueValueWhenThereAreAnyTestsToRun(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - using var tempDir = new TempDirectory(); var assemblyPaths = BuildMultipleAssemblyPath("SimpleTestProject2.dll").Trim('\"'); - var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: tempDir.Path); + var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: TempDirectory.Path); arguments = string.Concat(arguments, " -- RunConfiguration.TreatNoTestsAsError=true"); InvokeVsTest(arguments); @@ -341,10 +329,9 @@ public void ExitCodeShouldNotDependOnTreatNoTestsAsErrorTrueValueWhenThereAreAny public void ExitCodeShouldNotDependOnFailTreatNoTestsAsErrorFalseValueWhenThereAreAnyTestsToRun(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - using var tempDir = new TempDirectory(); var assemblyPaths = BuildMultipleAssemblyPath("SimpleTestProject2.dll").Trim('\"'); - var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: tempDir.Path); + var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: TempDirectory.Path); arguments = string.Concat(arguments, " -- RunConfiguration.TreatNoTestsAsError=false"); InvokeVsTest(arguments); diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/ExecutionThreadApartmentStateTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/ExecutionThreadApartmentStateTests.cs index d1611403f2..763e0d5016 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/ExecutionThreadApartmentStateTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/ExecutionThreadApartmentStateTests.cs @@ -17,10 +17,9 @@ public class ExecutionThreadApartmentStateTests : AcceptanceTestBase public void UITestShouldPassIfApartmentStateIsSTA(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - using var tempDir = new TempDirectory(); var assemblyPaths = BuildMultipleAssemblyPath("SimpleTestProject3.dll").Trim('\"'); - var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: tempDir.Path); + var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: TempDirectory.Path); arguments = string.Concat(arguments, " /testcasefilter:UITestMethod"); InvokeVsTest(arguments); ValidateSummaryStatus(1, 0, 0); @@ -31,11 +30,10 @@ public void UITestShouldPassIfApartmentStateIsSTA(RunnerInfo runnerInfo) public void WarningShouldBeShownWhenValueIsSTAForNetCore(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - using var tempDir = new TempDirectory(); var assemblyPaths = BuildMultipleAssemblyPath("SimpleTestProject2.dll").Trim('\"'); - var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: tempDir.Path); + var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: TempDirectory.Path); arguments = string.Concat(arguments, " /testcasefilter:PassingTest2 -- RunConfiguration.ExecutionThreadApartmentState=STA"); InvokeVsTest(arguments); StdOutputContains("ExecutionThreadApartmentState option not supported for framework:"); @@ -47,11 +45,10 @@ public void WarningShouldBeShownWhenValueIsSTAForNetCore(RunnerInfo runnerInfo) public void UITestShouldFailWhenDefaultApartmentStateIsMTA(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - using var tempDir = new TempDirectory(); var assemblyPaths = BuildMultipleAssemblyPath("SimpleTestProject3.dll").Trim('\"'); - var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: tempDir.Path); + var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: TempDirectory.Path); arguments = string.Concat(arguments, " /testcasefilter:UITestMethod -- RunConfiguration.ExecutionThreadApartmentState=MTA"); InvokeVsTest(arguments); ValidateSummaryStatus(0, 1, 0); @@ -63,11 +60,10 @@ public void UITestShouldFailWhenDefaultApartmentStateIsMTA(RunnerInfo runnerInfo public void CancelTestExectionShouldWorkWhenApartmentStateIsSTA(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - using var tempDir = new TempDirectory(); var assemblyPaths = BuildMultipleAssemblyPath("SimpleTestProject3.dll").Trim('\"'); - var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: tempDir.Path); + var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: TempDirectory.Path); arguments = string.Concat(arguments, " /tests:UITestWithSleep1,UITestMethod -- RunConfiguration.ExecutionThreadApartmentState=STA RunConfiguration.TestSessionTimeout=2000"); InvokeVsTest(arguments); StdOutputContains("Canceling test run: test run timeout of"); diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/RunnnerInfo.cs b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/RunnnerInfo.cs index e948649a28..bf7c89fb1d 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/RunnnerInfo.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/RunnnerInfo.cs @@ -16,6 +16,13 @@ public RunnerInfo(string runnerType, string targetFramework, string inIsolation) RunnerFramework = runnerType; TargetFramework = targetFramework; InIsolationValue = inIsolation; + // The value is netcoreapp2.1. + IsNetRunner = RunnerFramework.StartsWith("netcoreapp"); + // The value is net451. + IsNetFrameworkRunner = !IsNetRunner; + IsNetTarget = TargetFramework.StartsWith("netcoreapp"); + IsNetFrameworkTarget = !IsNetTarget; + } /// /// Gets the target framework. @@ -48,4 +55,28 @@ public override string ToString() { return string.Join(",", new[] { "RunnerFramework = " + RunnerFramework, " TargetFramework = " + TargetFramework, string.IsNullOrEmpty(InIsolationValue) ? " InProcess" : " InIsolation" }); } + + /// + /// Is running via .NET "Core" vstest.console? + /// + /// + public bool IsNetRunner { get; } + + /// + /// Is running via .NET Framework vstest.console? + /// + /// + public bool IsNetFrameworkRunner { get; } + + /// + /// Is running via .NET "Core" testhost? + /// + /// + public bool IsNetTarget { get; } + + /// + /// Is running via .NET Framework testhost? + /// + /// + public bool IsNetFrameworkTarget { get; } } diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/FilePatternParserTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/FilePatternParserTests.cs index 0fc7a719d9..4ab4715e72 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/FilePatternParserTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/FilePatternParserTests.cs @@ -19,7 +19,6 @@ public class FilePatternParserTests : AcceptanceTestBase public void WildCardPatternShouldCorrectlyWorkOnFiles(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - using var tempDir = new TempDirectory(); var testAssembly = GetSampleTestAssembly(); testAssembly = testAssembly.Replace("SimpleTestProject.dll", "*TestProj*.dll"); @@ -28,7 +27,7 @@ public void WildCardPatternShouldCorrectlyWorkOnFiles(RunnerInfo runnerInfo) testAssembly, GetTestAdapterPath(), string.Empty, FrameworkArgValue, - runnerInfo.InIsolationValue, resultsDirectory: tempDir.Path); + runnerInfo.InIsolationValue, resultsDirectory: TempDirectory.Path); InvokeVsTest(arguments); ValidateSummaryStatus(1, 1, 1); @@ -40,18 +39,21 @@ public void WildCardPatternShouldCorrectlyWorkOnFiles(RunnerInfo runnerInfo) public void WildCardPatternShouldCorrectlyWorkOnArbitraryDepthDirectories(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - using var tempDir = new TempDirectory(); var testAssembly = GetSampleTestAssembly(); - var oldAssemblyPath = Path.Combine("Debug", _testEnvironment.TargetFramework, "SimpleTestProject.dll"); - var newAssemblyPath = Path.Combine("**", _testEnvironment.TargetFramework, "*TestProj*.dll"); - testAssembly = testAssembly.Replace(oldAssemblyPath, newAssemblyPath); + + // Add one more directory to the temp path, so we can substitute it with ** + // and copy then whole directory there. + TempDirectory.CopyDirectory(Path.GetDirectoryName(testAssembly), Path.Combine(TempDirectory.Path, "dir1")); + + // The path will end up looking like \**\"*TestProj*.dll". + var wildcardedPath = Path.Combine(TempDirectory.Path, "**", "*TestProj*.dll"); var arguments = PrepareArguments( - testAssembly, + wildcardedPath, GetTestAdapterPath(), string.Empty, string.Empty, - runnerInfo.InIsolationValue, resultsDirectory: tempDir.Path); + runnerInfo.InIsolationValue, resultsDirectory: TempDirectory.Path); InvokeVsTest(arguments); ValidateSummaryStatus(1, 1, 1); @@ -63,7 +65,6 @@ public void WildCardPatternShouldCorrectlyWorkOnArbitraryDepthDirectories(Runner public void WildCardPatternShouldCorrectlyWorkForRelativeAssemblyPath(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - using var tempDir = new TempDirectory(); var testAssembly = GetSampleTestAssembly(); testAssembly = testAssembly.Replace("SimpleTestProject.dll", "*TestProj*.dll"); @@ -78,7 +79,7 @@ public void WildCardPatternShouldCorrectlyWorkForRelativeAssemblyPath(RunnerInfo testAssembly, GetTestAdapterPath(), string.Empty, string.Empty, - runnerInfo.InIsolationValue, resultsDirectory: tempDir.Path); + runnerInfo.InIsolationValue, resultsDirectory: TempDirectory.Path); InvokeVsTest(arguments); ValidateSummaryStatus(1, 1, 1); @@ -90,7 +91,6 @@ public void WildCardPatternShouldCorrectlyWorkForRelativeAssemblyPath(RunnerInfo public void WildCardPatternShouldCorrectlyWorkOnMultipleFiles(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - using var tempDir = new TempDirectory(); var testAssembly = BuildMultipleAssemblyPath("SimpleTestProject.dll", "SimpleTestProject2.dll").Trim('\"'); testAssembly = testAssembly.Replace("SimpleTestProject.dll", "*TestProj*.dll"); @@ -100,7 +100,7 @@ public void WildCardPatternShouldCorrectlyWorkOnMultipleFiles(RunnerInfo runnerI testAssembly, GetTestAdapterPath(), string.Empty, FrameworkArgValue, - runnerInfo.InIsolationValue, resultsDirectory: tempDir.Path); + runnerInfo.InIsolationValue, resultsDirectory: TempDirectory.Path); InvokeVsTest(arguments); ValidateSummaryStatus(2, 2, 2); diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/FrameworkTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/FrameworkTests.cs index 61cf91196b..394db87a80 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/FrameworkTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/FrameworkTests.cs @@ -18,9 +18,8 @@ public class FrameworkTests : AcceptanceTestBase public void FrameworkArgumentShouldWork(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - using var tempDir = new TempDirectory(); - var arguments = PrepareArguments(GetSampleTestAssembly(), string.Empty, string.Empty, string.Empty, resultsDirectory: tempDir.Path); + var arguments = PrepareArguments(GetSampleTestAssembly(), string.Empty, string.Empty, string.Empty, resultsDirectory: TempDirectory.Path); arguments = string.Concat(arguments, " ", $"/Framework:{FrameworkArgValue}"); InvokeVsTest(arguments); @@ -33,9 +32,8 @@ public void FrameworkArgumentShouldWork(RunnerInfo runnerInfo) public void FrameworkShortNameArgumentShouldWork(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - using var tempDir = new TempDirectory(); - var arguments = PrepareArguments(GetSampleTestAssembly(), string.Empty, string.Empty, string.Empty, resultsDirectory: tempDir.Path); + var arguments = PrepareArguments(GetSampleTestAssembly(), string.Empty, string.Empty, string.Empty, resultsDirectory: TempDirectory.Path); arguments = string.Concat(arguments, " ", $"/Framework:{_testEnvironment.TargetFramework}"); InvokeVsTest(arguments); @@ -50,9 +48,8 @@ public void FrameworkShortNameArgumentShouldWork(RunnerInfo runnerInfo) public void OnWrongFrameworkPassedTestRunShouldNotRun(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - using var tempDir = new TempDirectory(); - var arguments = PrepareArguments(GetSampleTestAssembly(), string.Empty, string.Empty, string.Empty, resultsDirectory: tempDir.Path); + var arguments = PrepareArguments(GetSampleTestAssembly(), string.Empty, string.Empty, string.Empty, resultsDirectory: TempDirectory.Path); if (runnerInfo.TargetFramework.Contains("netcore")) { arguments = string.Concat(arguments, " ", "/Framework:Framework45"); @@ -82,9 +79,8 @@ public void OnWrongFrameworkPassedTestRunShouldNotRun(RunnerInfo runnerInfo) public void RunSpecificTestsShouldWorkWithFrameworkInCompatibleWarning(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - using var tempDir = new TempDirectory(); - var arguments = PrepareArguments(GetSampleTestAssembly(), string.Empty, string.Empty, string.Empty, resultsDirectory: tempDir.Path); + var arguments = PrepareArguments(GetSampleTestAssembly(), string.Empty, string.Empty, string.Empty, resultsDirectory: TempDirectory.Path); arguments = string.Concat(arguments, " ", "/tests:PassingTest"); arguments = string.Concat(arguments, " ", "/Framework:Framework40"); diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/LoggerTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/LoggerTests.cs index 0f277b407b..784ad8ca3d 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/LoggerTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/LoggerTests.cs @@ -22,19 +22,18 @@ public class LoggerTests : AcceptanceTestBase public void TrxLoggerWithFriendlyNameShouldProperlyOverwriteFile(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - using var tempDir = new TempDirectory(); - var arguments = PrepareArguments(GetSampleTestAssembly(), GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, tempDir.Path); + var arguments = PrepareArguments(GetSampleTestAssembly(), GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, TempDirectory.Path); var trxFileName = "TestResults.trx"; arguments = string.Concat(arguments, $" /logger:\"trx;LogFileName={trxFileName}\""); InvokeVsTest(arguments); - arguments = PrepareArguments(GetSampleTestAssembly(), GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, tempDir.Path); + arguments = PrepareArguments(GetSampleTestAssembly(), GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, TempDirectory.Path); arguments = string.Concat(arguments, $" /logger:\"trx;LogFileName={trxFileName}\""); arguments = string.Concat(arguments, " /testcasefilter:Name~Pass"); InvokeVsTest(arguments); - var trxFilePath = Path.Combine(tempDir.Path, trxFileName); + var trxFilePath = Path.Combine(TempDirectory.Path, trxFileName); Assert.IsTrue(IsValidXml(trxFilePath), "Invalid content in Trx log file"); } @@ -44,19 +43,18 @@ public void TrxLoggerWithFriendlyNameShouldProperlyOverwriteFile(RunnerInfo runn public void HtmlLoggerWithFriendlyNameShouldProperlyOverwriteFile(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - using var tempDir = new TempDirectory(); - var arguments = PrepareArguments(GetSampleTestAssembly(), GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, tempDir.Path); + var arguments = PrepareArguments(GetSampleTestAssembly(), GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, TempDirectory.Path); var htmlFileName = "TestResults.html"; arguments = string.Concat(arguments, $" /logger:\"html;LogFileName={htmlFileName}\""); InvokeVsTest(arguments); - arguments = PrepareArguments(GetSampleTestAssembly(), GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, tempDir.Path); + arguments = PrepareArguments(GetSampleTestAssembly(), GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, TempDirectory.Path); arguments = string.Concat(arguments, $" /logger:\"html;LogFileName={htmlFileName}\""); arguments = string.Concat(arguments, " /testcasefilter:Name~Pass"); InvokeVsTest(arguments); - var htmlLogFilePath = Path.Combine(tempDir.Path, htmlFileName); + var htmlLogFilePath = Path.Combine(TempDirectory.Path, htmlFileName); IsFileAndContentEqual(htmlLogFilePath); } @@ -65,19 +63,18 @@ public void HtmlLoggerWithFriendlyNameShouldProperlyOverwriteFile(RunnerInfo run public void TrxLoggerWithExecutorUriShouldProperlyOverwriteFile(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - using var tempDir = new TempDirectory(); - var arguments = PrepareArguments(GetSampleTestAssembly(), GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, tempDir.Path); + var arguments = PrepareArguments(GetSampleTestAssembly(), GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, TempDirectory.Path); var trxFileName = "TestResults.trx"; arguments = string.Concat(arguments, $" /logger:\"logger://Microsoft/TestPlatform/TrxLogger/v1;LogFileName={trxFileName}\""); InvokeVsTest(arguments); - arguments = PrepareArguments(GetSampleTestAssembly(), GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, tempDir.Path); + arguments = PrepareArguments(GetSampleTestAssembly(), GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, TempDirectory.Path); arguments = string.Concat(arguments, $" /logger:\"logger://Microsoft/TestPlatform/TrxLogger/v1;LogFileName={trxFileName}\""); arguments = string.Concat(arguments, " /testcasefilter:Name~Pass"); InvokeVsTest(arguments); - var trxLogFilePath = Path.Combine(tempDir.Path, trxFileName); + var trxLogFilePath = Path.Combine(TempDirectory.Path, trxFileName); Assert.IsTrue(IsValidXml(trxLogFilePath), "Invalid content in Trx log file"); } @@ -87,19 +84,18 @@ public void TrxLoggerWithExecutorUriShouldProperlyOverwriteFile(RunnerInfo runne public void TrxLoggerWithLogFilePrefixShouldGenerateMultipleTrx(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - using var tempDir = new TempDirectory(); var trxFileNamePattern = "TestResults"; - var arguments = PrepareArguments(GetSampleTestAssembly(), GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, tempDir.Path); + var arguments = PrepareArguments(GetSampleTestAssembly(), GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, TempDirectory.Path); arguments = string.Concat(arguments, $" /logger:\"logger://Microsoft/TestPlatform/TrxLogger/v1;LogFilePrefix={trxFileNamePattern}\""); InvokeVsTest(arguments); - arguments = PrepareArguments(GetSampleTestAssembly(), GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, tempDir.Path); + arguments = PrepareArguments(GetSampleTestAssembly(), GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, TempDirectory.Path); arguments = string.Concat(arguments, $" /logger:\"logger://Microsoft/TestPlatform/TrxLogger/v1;LogFilePrefix={trxFileNamePattern}\""); arguments = string.Concat(arguments, " /testcasefilter:Name~Pass"); InvokeVsTest(arguments); - var trxFilePaths = Directory.EnumerateFiles(tempDir.Path, trxFileNamePattern + "_net*.trx"); + var trxFilePaths = Directory.EnumerateFiles(TempDirectory.Path, trxFileNamePattern + "_net*.trx"); Assert.IsTrue(trxFilePaths.Count() > 1); } @@ -108,19 +104,18 @@ public void TrxLoggerWithLogFilePrefixShouldGenerateMultipleTrx(RunnerInfo runne public void HtmlLoggerWithExecutorUriShouldProperlyOverwriteFile(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - using var tempDir = new TempDirectory(); - var arguments = PrepareArguments(GetSampleTestAssembly(), GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, tempDir.Path); + var arguments = PrepareArguments(GetSampleTestAssembly(), GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, TempDirectory.Path); var htmlFileName = "TestResults.html"; arguments = string.Concat(arguments, $" /logger:\"logger://Microsoft/TestPlatform/htmlLogger/v1;LogFileName{htmlFileName}\""); InvokeVsTest(arguments); - arguments = PrepareArguments(GetSampleTestAssembly(), GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, tempDir.Path); + arguments = PrepareArguments(GetSampleTestAssembly(), GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, TempDirectory.Path); arguments = string.Concat(arguments, $" /logger:\"logger://Microsoft/TestPlatform/htmlLogger/v1;LogFileName={htmlFileName}\""); arguments = string.Concat(arguments, " /testcasefilter:Name~Pass"); InvokeVsTest(arguments); - var htmlLogFilePath = Path.Combine(tempDir.Path, htmlFileName); + var htmlLogFilePath = Path.Combine(TempDirectory.Path, htmlFileName); IsFileAndContentEqual(htmlLogFilePath); } @@ -130,10 +125,9 @@ public void HtmlLoggerWithExecutorUriShouldProperlyOverwriteFile(RunnerInfo runn public void TrxLoggerResultSummaryOutcomeValueShouldBeFailedIfNoTestsExecutedAndTreatNoTestsAsErrorIsTrue(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - using var tempDir = new TempDirectory(); var arguments = PrepareArguments(GetSampleTestAssembly(), GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue); - var trxFilePath = Path.Combine(tempDir.Path, "TrxLogger.trx"); + var trxFilePath = Path.Combine(TempDirectory.Path, "TrxLogger.trx"); arguments = string.Concat(arguments, $" /logger:\"trx;LogFileName={trxFilePath}\""); @@ -154,10 +148,9 @@ public void TrxLoggerResultSummaryOutcomeValueShouldBeFailedIfNoTestsExecutedAnd public void TrxLoggerResultSummaryOutcomeValueShouldNotChangeIfNoTestsExecutedAndTreatNoTestsAsErrorIsFalse(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - using var tempDir = new TempDirectory(); var arguments = PrepareArguments(GetSampleTestAssembly(), GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue); - var trxFilePath = Path.Combine(tempDir.Path, "TrxLogger.trx"); + var trxFilePath = Path.Combine(TempDirectory.Path, "TrxLogger.trx"); arguments = string.Concat(arguments, $" /logger:\"trx;LogFileName={trxFilePath}\""); diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/Microsoft.TestPlatform.AcceptanceTests.csproj b/test/Microsoft.TestPlatform.AcceptanceTests/Microsoft.TestPlatform.AcceptanceTests.csproj index 31d535775e..890ed63c39 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/Microsoft.TestPlatform.AcceptanceTests.csproj +++ b/test/Microsoft.TestPlatform.AcceptanceTests/Microsoft.TestPlatform.AcceptanceTests.csproj @@ -12,6 +12,12 @@ netcoreapp3.1 Microsoft.TestPlatform.AcceptanceTests + + False + + + False + diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/MultitargetingTestHostTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/MultitargetingTestHostTests.cs index f420f861fe..7b1dff7eb9 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/MultitargetingTestHostTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/MultitargetingTestHostTests.cs @@ -22,10 +22,9 @@ public class MultitargetingTestHostTests : AcceptanceTestBase public void TestRunInATesthostThatTargetsTheirChosenNETFramework(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - using var tempDir = new TempDirectory(); var assemblyPath = BuildMultipleAssemblyPath("MultitargetedNetFrameworkProject.dll").Trim('\"'); - var arguments = PrepareArguments(assemblyPath, null, null, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: tempDir.Path); + var arguments = PrepareArguments(assemblyPath, null, null, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: TempDirectory.Path); // Tell the test project which target framework we are expecting it to run as. // It has this value conditionally compiled, so it can compare it. diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/PlatformTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/PlatformTests.cs index 4dc7b4552b..b149fa2994 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/PlatformTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/PlatformTests.cs @@ -43,18 +43,17 @@ public void RunTestExecutionWithPlatformx86(RunnerInfo runnerInfo) private void RunTestExecutionWithPlatform(string platformArg, string testhostProcessName, int expectedNumOfProcessCreated) { - using var tempDir = new TempDirectory(); var arguments = PrepareArguments( GetSampleTestAssembly(), GetTestAdapterPath(), string.Empty, FrameworkArgValue, - _testEnvironment.InIsolationValue, resultsDirectory: tempDir.Path); + _testEnvironment.InIsolationValue, resultsDirectory: TempDirectory.Path); - arguments = string.Concat(arguments, platformArg, GetDiagArg(tempDir.Path)); + arguments = string.Concat(arguments, platformArg, GetDiagArg(TempDirectory.Path)); InvokeVsTest(arguments); - AssertExpectedNumberOfHostProcesses(expectedNumOfProcessCreated, tempDir.Path, new[] { testhostProcessName }, arguments, GetConsoleRunnerPath()); + AssertExpectedNumberOfHostProcesses(expectedNumOfProcessCreated, TempDirectory.Path, new[] { testhostProcessName }, arguments, GetConsoleRunnerPath()); ValidateSummaryStatus(1, 1, 1); } } diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/PostProcessingTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/PostProcessingTests.cs index 2204859563..b059a0bb0e 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/PostProcessingTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/PostProcessingTests.cs @@ -22,7 +22,6 @@ public class PostProcessingTests : AcceptanceTestBase [TestMethod] public void DotnetSDKSimulation_PostProcessing() { - using var tempDir = new TempDirectory(); var extensionsPath = Path.Combine( _testEnvironment.TestAssetsPath, @@ -32,7 +31,7 @@ public void DotnetSDKSimulation_PostProcessing() "netstandard2.0"); _testEnvironment.RunnerFramework = CoreRunnerFramework; - string runSettings = GetRunsettingsFilePath(tempDir.Path); + string runSettings = GetRunsettingsFilePath(TempDirectory.Path); string correlationSessionId = Guid.NewGuid().ToString(); // Set datacollector parameters @@ -46,7 +45,7 @@ public void DotnetSDKSimulation_PostProcessing() // Build and run tests like msbuild Parallel.For(0, 5, i => { - string projectFolder = Path.Combine(tempDir.Path, i.ToString()); + string projectFolder = Path.Combine(TempDirectory.Path, i.ToString()); ExecuteApplication(GetConsoleRunnerPath(), $"new mstest -o {projectFolder}", out string stdOut, out string stdError, out int exitCode); Assert.AreEqual(exitCode, 0); ExecuteApplication(GetConsoleRunnerPath(), $"build {projectFolder} -c release", out stdOut, out stdError, out exitCode); @@ -54,13 +53,13 @@ public void DotnetSDKSimulation_PostProcessing() string testContainer = Directory.GetFiles(Path.Combine(projectFolder, "bin"), $"{i}.dll", SearchOption.AllDirectories).Single(); - ExecuteVsTestConsole($"{testContainer} --Collect:\"SampleDataCollector\" --TestAdapterPath:\"{extensionsPath}\" --ResultsDirectory:\"{Path.GetDirectoryName(testContainer)}\" --Settings:\"{runSettings}\" --ArtifactsProcessingMode-Collect --TestSessionCorrelationId:\"{correlationSessionId}\" --Diag:\"{tempDir.Path + '/'}\"", out stdOut, out stdError, out exitCode, + ExecuteVsTestConsole($"{testContainer} --Collect:\"SampleDataCollector\" --TestAdapterPath:\"{extensionsPath}\" --ResultsDirectory:\"{Path.GetDirectoryName(testContainer)}\" --Settings:\"{runSettings}\" --ArtifactsProcessingMode-Collect --TestSessionCorrelationId:\"{correlationSessionId}\" --Diag:\"{TempDirectory.Path + '/'}\"", out stdOut, out stdError, out exitCode, new Dictionary() { { "VSTEST_FEATURE_ARTIFACTS_POSTPROCESSING", "1" } }); Assert.AreEqual(exitCode, 0); }); // Post process artifacts - ExecuteVsTestConsole($"--ArtifactsProcessingMode-PostProcess --TestSessionCorrelationId:\"{correlationSessionId}\" --Diag:\"{tempDir.Path + '/'}\"", out string stdOut, out string stdError, out int exitCode, + ExecuteVsTestConsole($"--ArtifactsProcessingMode-PostProcess --TestSessionCorrelationId:\"{correlationSessionId}\" --Diag:\"{TempDirectory.Path + '/'}\"", out string stdOut, out string stdError, out int exitCode, new Dictionary() { { "VSTEST_FEATURE_ARTIFACTS_POSTPROCESSING", "1" } }); Assert.AreEqual(exitCode, 0); diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/ProcessesInteractionTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/ProcessesInteractionTests.cs index 35b7170d8c..b4fe07e22c 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/ProcessesInteractionTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/ProcessesInteractionTests.cs @@ -3,7 +3,6 @@ using System.IO; -using Microsoft.TestPlatform.TestUtilities; using Microsoft.VisualStudio.TestTools.UnitTesting; namespace Microsoft.TestPlatform.AcceptanceTests; @@ -25,10 +24,8 @@ public void WhenTestHostProcessExitsBecauseTheTargetedRuntimeIsNoFoundThenTheMes const string testAssetProjectName = "SimpleTestProjectMessedUpTargetFramework"; var assemblyPath = GetAssetFullPath(testAssetProjectName + ".dll", Core21TargetFramework); UpdateRuntimeConfigJsonWithInvalidFramework(assemblyPath, testAssetProjectName); - using var tempDir = new TempDirectory(); // Act - var arguments = PrepareArguments(assemblyPath, GetTestAdapterPath(), "", FrameworkArgValue, tempDir.Path); InvokeVsTest(assemblyPath); // Assert diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/ResultsDirectoryTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/ResultsDirectoryTests.cs index b45a06862d..a7b02265a7 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/ResultsDirectoryTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/ResultsDirectoryTests.cs @@ -22,13 +22,12 @@ public void TrxFileShouldBeCreatedInResultsDirectory(RunnerInfo runnerInfo) var arguments = PrepareArguments(GetSampleTestAssembly(), GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue); var trxFileName = "TestResults.trx"; - using var tempDir = new TempDirectory(); - var trxFilePath = Path.Combine(tempDir.Path, trxFileName); + var trxFilePath = Path.Combine(TempDirectory.Path, trxFileName); arguments = string.Concat(arguments, $" /logger:\"trx;LogFileName={trxFileName}\""); - arguments = string.Concat(arguments, $" /ResultsDirectory:{tempDir.Path}"); + arguments = string.Concat(arguments, $" /ResultsDirectory:{TempDirectory.Path}"); // Delete if already exists - TempDirectory.TryRemoveDirectory(tempDir.Path); + TempDirectory.TryRemoveDirectory(TempDirectory.Path); InvokeVsTest(arguments); diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/RunsettingsTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/RunsettingsTests.cs index 01ab24c4ab..8ea5a48b5c 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/RunsettingsTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/RunsettingsTests.cs @@ -30,7 +30,10 @@ public void CommandLineRunSettingsShouldWinAmongAllOptions(RunnerInfo runnerInfo var targetPlatform = "x86"; var testhostProcessName = new[] { "testhost.x86" }; - var expectedNumOfProcessCreated = 1; + + // We pass 2 dlls in RunTestWithRunSettings, for .NET Framework they run in + // 1 hosts because that host is Shared. + var expectedNumOfProcessCreated = runnerInfo.IsNetFrameworkTarget ? 1 : 2; // passing parallel var runConfigurationDictionary = new Dictionary @@ -67,7 +70,10 @@ public void CLIRunsettingsShouldWinBetweenCLISwitchesAndCLIRunsettings(RunnerInf var targetPlatform = "x86"; var testhostProcessName = new[] { "testhost.x86" }; - var expectedNumOfProcessCreated = 1; + + // We pass 2 dlls in RunTestWithRunSettings, for .NET Framework they run in + // 1 hosts because that host is Shared. + var expectedNumOfProcessCreated = runnerInfo.IsNetFrameworkTarget ? 1 : 2; // Pass parallel var additionalArgs = "/Parallel"; @@ -100,7 +106,10 @@ public void CommandLineSwitchesShouldWinBetweenSettingsFileAndCommandLineSwitche SetTestEnvironment(_testEnvironment, runnerInfo); var testhostProcessName = new[] { "testhost.x86" }; - var expectedNumOfProcessCreated = 1; + + // We pass 2 dlls in RunTestWithRunSettings, for .NET Framework they run in + // 1 hosts because that host is Shared. + var expectedNumOfProcessCreated = runnerInfo.IsNetFrameworkTarget ? 1 : 2; // passing different platform var runConfigurationDictionary = new Dictionary @@ -126,7 +135,10 @@ public void RunSettingsWithoutParallelAndPlatformX86(RunnerInfo runnerInfo) var targetPlatform = "x86"; var testhostProcessNames = new[] { "testhost.x86" }; - var expectedNumOfProcessCreated = 1; + + // We pass 2 dlls in RunTestWithRunSettings, for .NET Framework they run in + // 1 hosts because that host is Shared. + var expectedNumOfProcessCreated = runnerInfo.IsNetFrameworkTarget ? 1 : 2; var runConfigurationDictionary = new Dictionary { @@ -147,7 +159,10 @@ public void RunSettingsParamsAsArguments(RunnerInfo runnerInfo) var targetPlatform = "x86"; var testhostProcessName = new[] { "testhost.x86" }; - var expectedNumOfProcessCreated = 1; + + // We pass 2 dlls in RunTestWithRunSettings, for .NET Framework they run in + // 1 hosts because that host is Shared. + var expectedNumOfProcessCreated = runnerInfo.IsNetFrameworkTarget ? 1 : 2; var runSettingsArgs = string.Join( " ", @@ -171,7 +186,11 @@ public void RunSettingsAndRunSettingsParamsAsArguments(RunnerInfo runnerInfo) var targetPlatform = "x86"; var testhostProcessName = new[] { "testhost.x86" }; - var expectedNumOfProcessCreated = 1; + + // We pass 2 dlls in RunTestWithRunSettings, for .NET Framework they run in + // 1 hosts because that host is Shared. + var expectedNumOfProcessCreated = runnerInfo.IsNetFrameworkTarget ? 1 : 2; + var runConfigurationDictionary = new Dictionary { { "MaxCpuCount", "2" }, @@ -220,18 +239,17 @@ public void RunSettingsWithParallelAndPlatformX64(RunnerInfo runnerInfo) public void RunSettingsWithInvalidValueShouldLogError(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - using var tempDir = new TempDirectory(); var runConfigurationDictionary = new Dictionary - { - { "TargetPlatform", "123" } - }; - var runsettingsFilePath = GetRunsettingsFilePath(runConfigurationDictionary, tempDir); + { + { "TargetPlatform", "123" } + }; + var runsettingsFilePath = GetRunsettingsFilePath(runConfigurationDictionary, TempDirectory); var arguments = PrepareArguments( GetSampleTestAssembly(), string.Empty, runsettingsFilePath, FrameworkArgValue, - runnerInfo.InIsolationValue, resultsDirectory: tempDir.Path); + runnerInfo.InIsolationValue, resultsDirectory: TempDirectory.Path); InvokeVsTest(arguments); StdErrorContains(@"Settings file provided does not conform to required format. An error occurred while loading the settings. Error: Invalid setting 'RunConfiguration'. Invalid value '123' specified for 'TargetPlatform'."); } @@ -242,18 +260,17 @@ public void RunSettingsWithInvalidValueShouldLogError(RunnerInfo runnerInfo) public void TestAdapterPathFromRunSettings(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - using var tempDir = new TempDirectory(); var runConfigurationDictionary = new Dictionary - { - { "TestAdaptersPaths", GetTestAdapterPath() } - }; - var runsettingsFilePath = GetRunsettingsFilePath(runConfigurationDictionary, tempDir); + { + { "TestAdaptersPaths", GetTestAdapterPath() } + }; + var runsettingsFilePath = GetRunsettingsFilePath(runConfigurationDictionary, TempDirectory); var arguments = PrepareArguments( GetSampleTestAssembly(), string.Empty, runsettingsFilePath, FrameworkArgValue, - runnerInfo.InIsolationValue, resultsDirectory: tempDir.Path); + runnerInfo.InIsolationValue, resultsDirectory: TempDirectory.Path); InvokeVsTest(arguments); ValidateSummaryStatus(1, 1, 1); } @@ -266,7 +283,6 @@ public void TestAdapterPathFromRunSettings(RunnerInfo runnerInfo) public void LegacySettingsWithPlatform(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - using var tempDir = new TempDirectory(); var testAssemblyPath = GetAssetFullPath("LegacySettingsUnitTestProject.dll"); _ = Path.GetDirectoryName(testAssemblyPath); @@ -281,13 +297,13 @@ public void LegacySettingsWithPlatform(RunnerInfo runnerInfo) "; - var runsettingsFilePath = GetRunsettingsFilePath(null, tempDir); + var runsettingsFilePath = GetRunsettingsFilePath(null, TempDirectory); File.WriteAllText(runsettingsFilePath, runsettingsXml); var arguments = PrepareArguments( testAssemblyPath, string.Empty, - runsettingsFilePath, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: tempDir.Path); + runsettingsFilePath, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: TempDirectory.Path); InvokeVsTest(arguments); ValidateSummaryStatus(0, 0, 0); } @@ -298,18 +314,17 @@ public void LegacySettingsWithPlatform(RunnerInfo runnerInfo) public void LegacySettingsWithScripts(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - using var tempDir = new TempDirectory(); var testAssemblyPath = GetAssetFullPath("LegacySettingsUnitTestProject.dll"); // Create the script files var guid = Guid.NewGuid(); var setupScriptName = "setupScript_" + guid + ".bat"; - var setupScriptPath = Path.Combine(tempDir.Path, setupScriptName); + var setupScriptPath = Path.Combine(TempDirectory.Path, setupScriptName); File.WriteAllText(setupScriptPath, @"echo > %temp%\ScriptTestingFile.txt"); var cleanupScriptName = "cleanupScript_" + guid + ".bat"; - var cleanupScriptPath = Path.Combine(tempDir.Path, cleanupScriptName); + var cleanupScriptPath = Path.Combine(TempDirectory.Path, cleanupScriptName); File.WriteAllText(cleanupScriptPath, @"del %temp%\ScriptTestingFile.txt"); var runsettingsFormat = @" @@ -323,19 +338,19 @@ public void LegacySettingsWithScripts(RunnerInfo runnerInfo) // Scripts have relative paths to temp directory where the runsettings is created. var runsettingsXml = string.Format(runsettingsFormat, setupScriptName, cleanupScriptName); - var runsettingsPath = GetRunsettingsFilePath(null, tempDir); + var runsettingsPath = GetRunsettingsFilePath(null, TempDirectory); File.WriteAllText(runsettingsPath, runsettingsXml); var arguments = PrepareArguments( testAssemblyPath, string.Empty, - runsettingsPath, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: tempDir.Path); + runsettingsPath, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: TempDirectory.Path); arguments = string.Concat(arguments, " /testcasefilter:Name=ScriptsTest"); InvokeVsTest(arguments); ValidateSummaryStatus(1, 0, 0); // Validate cleanup script ran - var scriptPath = Path.Combine(tempDir.Path, "ScriptTestingFile.txt"); + var scriptPath = Path.Combine(TempDirectory.Path, "ScriptTestingFile.txt"); Assert.IsFalse(File.Exists(scriptPath)); } @@ -345,7 +360,6 @@ public void LegacySettingsWithScripts(RunnerInfo runnerInfo) public void LegacySettingsWithDeploymentItem(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - using var tempDir = new TempDirectory(); var testAssemblyPath = GetAssetFullPath("LegacySettingsUnitTestProject.dll"); var testAssemblyDirectory = Path.GetDirectoryName(testAssemblyPath); @@ -364,13 +378,13 @@ public void LegacySettingsWithDeploymentItem(RunnerInfo runnerInfo) "; var runsettingsXml = string.Format(runsettingsFormat, deploymentItem); - var runsettingsPath = GetRunsettingsFilePath(null, tempDir); + var runsettingsPath = GetRunsettingsFilePath(null, TempDirectory); File.WriteAllText(runsettingsPath, runsettingsXml); var arguments = PrepareArguments( testAssemblyPath, string.Empty, - runsettingsPath, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: tempDir.Path); + runsettingsPath, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: TempDirectory.Path); arguments = string.Concat(arguments, " /testcasefilter:Name=DeploymentItemTest"); InvokeVsTest(arguments); ValidateSummaryStatus(1, 0, 0); @@ -382,7 +396,6 @@ public void LegacySettingsWithDeploymentItem(RunnerInfo runnerInfo) public void LegacySettingsTestTimeout(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - using var tempDir = new TempDirectory(); var testAssemblyPath = GetAssetFullPath("LegacySettingsUnitTestProject.dll"); var runsettingsXml = @" @@ -394,9 +407,9 @@ public void LegacySettingsTestTimeout(RunnerInfo runnerInfo) "; - var runsettingsPath = GetRunsettingsFilePath(null, tempDir); + var runsettingsPath = GetRunsettingsFilePath(null, TempDirectory); File.WriteAllText(runsettingsPath, runsettingsXml); - var arguments = PrepareArguments(testAssemblyPath, string.Empty, runsettingsPath, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: tempDir.Path); + var arguments = PrepareArguments(testAssemblyPath, string.Empty, runsettingsPath, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: TempDirectory.Path); arguments = string.Concat(arguments, " /testcasefilter:Name~TimeTest"); InvokeVsTest(arguments); @@ -410,7 +423,6 @@ public void LegacySettingsTestTimeout(RunnerInfo runnerInfo) public void LegacySettingsAssemblyResolution(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - using var tempDir = new TempDirectory(); var testAssemblyPath = GetAssetFullPath("LegacySettingsUnitTestProject.dll"); var runsettingsFormat = @" @@ -433,9 +445,9 @@ public void LegacySettingsAssemblyResolution(RunnerInfo runnerInfo) var testAssemblyDirectory = Path.Combine(_testEnvironment.TestAssetsPath, "LegacySettingsUnitTestProject", "DependencyAssembly"); var runsettingsXml = string.Format(runsettingsFormat, testAssemblyDirectory); - var runsettingsPath = GetRunsettingsFilePath(null, tempDir); + var runsettingsPath = GetRunsettingsFilePath(null, TempDirectory); File.WriteAllText(runsettingsPath, runsettingsXml); - var arguments = PrepareArguments(testAssemblyPath, string.Empty, runsettingsPath, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: tempDir.Path); + var arguments = PrepareArguments(testAssemblyPath, string.Empty, runsettingsPath, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: TempDirectory.Path); arguments = string.Concat(arguments, " /testcasefilter:Name=DependencyTest"); InvokeVsTest(arguments); @@ -453,7 +465,6 @@ public void LegacySettingsAssemblyResolution(RunnerInfo runnerInfo) public void EnvironmentVariablesSettingsShouldSetEnvironmentVariables(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - using var tempDir = new TempDirectory(); var testAssemblyPath = GetAssetFullPath("EnvironmentVariablesTestProject.dll"); @@ -465,13 +476,13 @@ public void EnvironmentVariablesSettingsShouldSetEnvironmentVariables(RunnerInfo "; - var runsettingsPath = GetRunsettingsFilePath(null, tempDir); + var runsettingsPath = GetRunsettingsFilePath(null, TempDirectory); File.WriteAllText(runsettingsPath, runsettingsXml); var arguments = PrepareArguments( testAssemblyPath, string.Empty, - runsettingsPath, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: tempDir.Path); + runsettingsPath, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: TempDirectory.Path); InvokeVsTest(arguments); ValidateSummaryStatus(1, 0, 0); } @@ -511,9 +522,9 @@ public void RunSettingsAreLoadedFromProject(RunnerInfo runnerInfo) #endregion - private string GetRunsettingsFilePath(Dictionary runConfigurationDictionary, TempDirectory tempDirectory) + private string GetRunsettingsFilePath(Dictionary runConfigurationDictionary, TempDirectory TempDirectory) { - var runsettingsPath = Path.Combine(tempDirectory.Path, "test_" + Guid.NewGuid() + ".runsettings"); + var runsettingsPath = Path.Combine(TempDirectory.Path, "test_" + Guid.NewGuid() + ".runsettings"); if (runConfigurationDictionary != null) { CreateRunSettingsFile(runsettingsPath, runConfigurationDictionary); @@ -525,20 +536,18 @@ private string GetRunsettingsFilePath(Dictionary runConfiguratio private void RunTestWithRunSettings(Dictionary runConfigurationDictionary, string runSettingsArgs, string additionalArgs, IEnumerable testhostProcessNames, int expectedNumOfProcessCreated) { - using var tempDir = new TempDirectory(); - var assemblyPaths = - BuildMultipleAssemblyPath("SimpleTestProject.dll", "SimpleTestProject2.dll").Trim('\"'); + var assemblyPaths = BuildMultipleAssemblyPath("SimpleTestProject.dll", "SimpleTestProject2.dll").Trim('\"'); var runsettingsPath = string.Empty; if (runConfigurationDictionary != null) { - runsettingsPath = GetRunsettingsFilePath(runConfigurationDictionary, tempDir); + runsettingsPath = GetRunsettingsFilePath(runConfigurationDictionary, TempDirectory); } - var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), runsettingsPath, FrameworkArgValue, _testEnvironment.InIsolationValue, resultsDirectory: tempDir.Path); - arguments += GetDiagArg(tempDir.Path); + var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), runsettingsPath, FrameworkArgValue, _testEnvironment.InIsolationValue, resultsDirectory: TempDirectory.Path); + arguments += GetDiagArg(TempDirectory.Path); if (!string.IsNullOrWhiteSpace(additionalArgs)) { @@ -553,7 +562,7 @@ private void RunTestWithRunSettings(Dictionary runConfigurationD InvokeVsTest(arguments); // assert - AssertExpectedNumberOfHostProcesses(expectedNumOfProcessCreated, tempDir.Path, testhostProcessNames, arguments, GetConsoleRunnerPath()); + AssertExpectedNumberOfHostProcesses(expectedNumOfProcessCreated, TempDirectory.Path, testhostProcessNames, arguments, GetConsoleRunnerPath()); ValidateSummaryStatus(2, 2, 2); //cleanup diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/SelfContainedAppTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/SelfContainedAppTests.cs index 39b94fcaac..c7dc6b7804 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/SelfContainedAppTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/SelfContainedAppTests.cs @@ -25,11 +25,10 @@ public void RunningApplicationThatIsBuiltAsSelfContainedWillNotFailToFindHostpol // that will fail if we run the testhost.exe from the .nuget location, but will work when we run it from the output folder // see https://github.com/dotnet/runtime/issues/3569#issuecomment-595820524 and below for description of how it works SetTestEnvironment(_testEnvironment, runnerInfo); - using var tempDir = new TempDirectory(); // the app is published to win10-x64 because of the runtime identifier in the project var assemblyPath = BuildMultipleAssemblyPath($@"win10-x64{Path.DirectorySeparatorChar}SelfContainedAppTestProject.dll").Trim('\"'); - var arguments = PrepareArguments(assemblyPath, null, null, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: tempDir.Path); + var arguments = PrepareArguments(assemblyPath, null, null, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: TempDirectory.Path); InvokeVsTest(arguments); ValidateSummaryStatus(passedTestsCount: 1, 0, 0); diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/TelemetryTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/TelemetryTests.cs index d5649d4a59..2083b84bde 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/TelemetryTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/TelemetryTests.cs @@ -28,7 +28,7 @@ public void RunTestsShouldPublishMetrics(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - RunTests(runnerInfo.RunnerFramework); + RunTests(runnerInfo); } [TestMethod] @@ -38,12 +38,12 @@ public void DiscoverTestsShouldPublishMetrics(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - DiscoverTests(runnerInfo.RunnerFramework); + DiscoverTests(runnerInfo); } - private void RunTests(string runnerFramework) + private void RunTests(RunnerInfo runnerInfo) { - if (runnerFramework.StartsWith("netcoreapp")) + if (runnerInfo.IsNetRunner) { Assert.Inconclusive("Telemetry API is not supported for .NetCore runner"); return; @@ -51,21 +51,20 @@ private void RunTests(string runnerFramework) var assemblyPaths = GetAssetFullPath("SimpleTestProject2.dll"); - using var tempDir = new TempDirectory(); var env = new Dictionary { - [LOG_TELEMETRY_PATH] = tempDir.Path, + [LOG_TELEMETRY_PATH] = TempDirectory.Path, [TELEMETRY_OPTEDIN] = "1", [LOG_TELEMETRY] = "1", }; InvokeVsTestForExecution(assemblyPaths, GetTestAdapterPath(), FrameworkArgValue, string.Empty, env); - ValidateOutput("Execution", tempDir); + ValidateOutput("Execution", TempDirectory); } - private void DiscoverTests(string runnerFramework) + private void DiscoverTests(RunnerInfo runnerInfo) { - if (runnerFramework.StartsWith("netcoreapp")) + if (runnerInfo.IsNetRunner) { Assert.Inconclusive("Telemetry API is not supported for .NetCore runner"); return; @@ -73,27 +72,26 @@ private void DiscoverTests(string runnerFramework) var assemblyPaths = GetAssetFullPath("SimpleTestProject2.dll"); - using var tempDir = new TempDirectory(); var env = new Dictionary { - [LOG_TELEMETRY_PATH] = tempDir.Path, + [LOG_TELEMETRY_PATH] = TempDirectory.Path, [TELEMETRY_OPTEDIN] = "1", [LOG_TELEMETRY] = "1", }; InvokeVsTestForDiscovery(assemblyPaths, GetTestAdapterPath(), string.Empty, FrameworkArgValue, env); - ValidateOutput("Discovery", tempDir); + ValidateOutput("Discovery", TempDirectory); } - private void ValidateOutput(string command, TempDirectory tempDir) + private void ValidateOutput(string command, TempDirectory TempDirectory) { - if (!Directory.Exists(tempDir.Path)) + if (!Directory.Exists(TempDirectory.Path)) { - Assert.Fail("Could not find the telemetry logs folder at {0}", tempDir.Path); + Assert.Fail("Could not find the telemetry logs folder at {0}", TempDirectory.Path); } bool isValid = false; - var directory = new DirectoryInfo(tempDir.Path); + var directory = new DirectoryInfo(TempDirectory.Path); var file = directory.GetFiles().OrderByDescending(f => f.CreationTime).First(); string[] lines = File.ReadAllLines(file.FullName); diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/TestCaseFilterTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/TestCaseFilterTests.cs index 08c7972125..9f90e13949 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/TestCaseFilterTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/TestCaseFilterTests.cs @@ -19,13 +19,12 @@ public class TestCaseFilterTests : AcceptanceTestBase public void RunSelectedTestsWithAndOperatorTrait(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - using var tempDir = new TempDirectory(); var arguments = PrepareArguments( GetSampleTestAssembly(), GetTestAdapterPath(), string.Empty, FrameworkArgValue, - runnerInfo.InIsolationValue, resultsDirectory: tempDir.Path); + runnerInfo.InIsolationValue, resultsDirectory: TempDirectory.Path); arguments = string.Concat(arguments, " /TestCaseFilter:\"(TestCategory=CategoryA&Priority=3)\""); InvokeVsTest(arguments); ValidateSummaryStatus(0, 1, 0); @@ -37,13 +36,12 @@ public void RunSelectedTestsWithAndOperatorTrait(RunnerInfo runnerInfo) public void RunSelectedTestsWithCategoryTraitInMixCase(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - using var tempDir = new TempDirectory(); var arguments = PrepareArguments( GetSampleTestAssembly(), GetTestAdapterPath(), string.Empty, FrameworkArgValue, - runnerInfo.InIsolationValue, resultsDirectory: tempDir.Path); + runnerInfo.InIsolationValue, resultsDirectory: TempDirectory.Path); arguments = string.Concat(arguments, " /TestCaseFilter:\"TestCategory=Categorya\""); InvokeVsTest(arguments); ValidateSummaryStatus(0, 1, 0); @@ -55,13 +53,12 @@ public void RunSelectedTestsWithCategoryTraitInMixCase(RunnerInfo runnerInfo) public void RunSelectedTestsWithClassNameTrait(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - using var tempDir = new TempDirectory(); var arguments = PrepareArguments( GetSampleTestAssembly(), GetTestAdapterPath(), string.Empty, FrameworkArgValue, - runnerInfo.InIsolationValue, resultsDirectory: tempDir.Path); + runnerInfo.InIsolationValue, resultsDirectory: TempDirectory.Path); arguments = string.Concat(arguments, " /TestCaseFilter:\"ClassName=SampleUnitTestProject.UnitTest1\""); InvokeVsTest(arguments); ValidateSummaryStatus(1, 1, 1); @@ -73,13 +70,12 @@ public void RunSelectedTestsWithClassNameTrait(RunnerInfo runnerInfo) public void RunSelectedTestsWithFullyQualifiedNameTrait(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - using var tempDir = new TempDirectory(); var arguments = PrepareArguments( GetSampleTestAssembly(), GetTestAdapterPath(), string.Empty, FrameworkArgValue, - runnerInfo.InIsolationValue, resultsDirectory: tempDir.Path); + runnerInfo.InIsolationValue, resultsDirectory: TempDirectory.Path); arguments = string.Concat( arguments, " /TestCaseFilter:\"FullyQualifiedName=SampleUnitTestProject.UnitTest1.FailingTest\""); @@ -93,13 +89,12 @@ public void RunSelectedTestsWithFullyQualifiedNameTrait(RunnerInfo runnerInfo) public void RunSelectedTestsWithNameTrait(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - using var tempDir = new TempDirectory(); var arguments = PrepareArguments( GetSampleTestAssembly(), GetTestAdapterPath(), string.Empty, FrameworkArgValue, - runnerInfo.InIsolationValue, resultsDirectory: tempDir.Path); + runnerInfo.InIsolationValue, resultsDirectory: TempDirectory.Path); arguments = string.Concat(arguments, " /TestCaseFilter:\"Name=PassingTest\""); InvokeVsTest(arguments); ValidateSummaryStatus(1, 0, 0); @@ -111,13 +106,12 @@ public void RunSelectedTestsWithNameTrait(RunnerInfo runnerInfo) public void RunSelectedTestsWithOrOperatorTrait(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - using var tempDir = new TempDirectory(); var arguments = PrepareArguments( GetSampleTestAssembly(), GetTestAdapterPath(), string.Empty, FrameworkArgValue, - runnerInfo.InIsolationValue, resultsDirectory: tempDir.Path); + runnerInfo.InIsolationValue, resultsDirectory: TempDirectory.Path); arguments = string.Concat(arguments, " /TestCaseFilter:\"(TestCategory=CategoryA|Priority=2)\""); InvokeVsTest(arguments); ValidateSummaryStatus(1, 1, 0); @@ -129,13 +123,12 @@ public void RunSelectedTestsWithOrOperatorTrait(RunnerInfo runnerInfo) public void RunSelectedTestsWithPriorityTrait(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - using var tempDir = new TempDirectory(); var arguments = PrepareArguments( GetSampleTestAssembly(), GetTestAdapterPath(), string.Empty, FrameworkArgValue, - runnerInfo.InIsolationValue, resultsDirectory: tempDir.Path); + runnerInfo.InIsolationValue, resultsDirectory: TempDirectory.Path); arguments = string.Concat(arguments, " /TestCaseFilter:\"Priority=2\""); InvokeVsTest(arguments); ValidateSummaryStatus(1, 0, 0); @@ -151,13 +144,12 @@ public void RunSelectedTestsWithPriorityTrait(RunnerInfo runnerInfo) public void TestCaseFilterShouldWorkIfOnlyPropertyValueGivenInExpression(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - using var tempDir = new TempDirectory(); var arguments = PrepareArguments( _testEnvironment.GetTestAsset("SimpleTestProject2.dll"), GetTestAdapterPath(), string.Empty, FrameworkArgValue, - runnerInfo.InIsolationValue, resultsDirectory: tempDir.Path); + runnerInfo.InIsolationValue, resultsDirectory: TempDirectory.Path); arguments = string.Concat(arguments, " /TestCaseFilter:UnitTest1"); InvokeVsTest(arguments); ValidateSummaryStatus(1, 1, 1); @@ -171,20 +163,19 @@ public void TestCaseFilterShouldWorkIfOnlyPropertyValueGivenInExpression(RunnerI [NetFullTargetFrameworkDataSource] public void DiscoverMstestV1TestsWithAndOperatorTrait(RunnerInfo runnerInfo) { - if (runnerInfo.RunnerFramework.StartsWith("netcoreapp")) + if (runnerInfo.IsNetRunner) { Assert.Inconclusive("Mstest v1 tests not supported with .Netcore runner."); return; } SetTestEnvironment(_testEnvironment, runnerInfo); - using var tempDir = new TempDirectory(); var arguments = PrepareArguments( _testEnvironment.GetTestAsset("MstestV1UnitTestProject.dll"), GetTestAdapterPath(), string.Empty, FrameworkArgValue, - runnerInfo.InIsolationValue, resultsDirectory: tempDir.Path); + runnerInfo.InIsolationValue, resultsDirectory: TempDirectory.Path); arguments = string.Concat(arguments, " /listtests /TestCaseFilter:\"(TestCategory!=CategoryA&Priority!=3)\""); InvokeVsTest(arguments); @@ -204,21 +195,20 @@ public void DiscoverMstestV1TestsWithAndOperatorTrait(RunnerInfo runnerInfo) [NetFullTargetFrameworkDataSource] public void DiscoverTmiTestsWithOnlyPropertyValue(RunnerInfo runnerInfo) { - if (runnerInfo.RunnerFramework.StartsWith("netcoreapp")) + if (runnerInfo.IsNetRunner) { Assert.Inconclusive("Tmi tests not supported with .Netcore runner."); return; } SetTestEnvironment(_testEnvironment, runnerInfo); - using var tempDir = new TempDirectory(); string testAssemblyPath = _testEnvironment.GetTestAsset("MstestV1UnitTestProject.dll"); var arguments = PrepareArguments( testAssemblyPath, GetTestAdapterPath(), string.Empty, FrameworkArgValue, - runnerInfo.InIsolationValue, resultsDirectory: tempDir.Path); + runnerInfo.InIsolationValue, resultsDirectory: TempDirectory.Path); string testSettingsPath = Path.Combine(Path.GetDirectoryName(testAssemblyPath), "MstestV1UnitTestProjectTestSettings.testsettings"); arguments = string.Concat(arguments, " /listtests /TestCaseFilter:PassingTest /settings:", testSettingsPath); diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/TestPlatformNugetPackageTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/TestPlatformNugetPackageTests.cs index caa2488f5d..6125803ad0 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/TestPlatformNugetPackageTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/TestPlatformNugetPackageTests.cs @@ -47,7 +47,7 @@ public static void ClassCleanup() [TestInitialize] public void SetUp() { - _resultsDirectory = new TempDirectory(); + _resultsDirectory = TempDirectory; } [TestCleanup] diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/CodeCoverageTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/CodeCoverageTests.cs index 92a25714d9..b14c602758 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/CodeCoverageTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/CodeCoverageTests.cs @@ -28,13 +28,13 @@ namespace Microsoft.TestPlatform.AcceptanceTests.TranslationLayerTests; public class CodeCoverageTests : CodeCoverageAcceptanceTestBase { private IVsTestConsoleWrapper _vstestConsoleWrapper; - private TempDirectory _tempDirectory; + private TempDirectory _TempDirectory; private RunEventHandler _runEventHandler; private TestRunAttachmentsProcessingEventHandler _testRunAttachmentsProcessingEventHandler; private void Setup() { - _vstestConsoleWrapper = GetVsTestConsoleWrapper(out _tempDirectory); + _vstestConsoleWrapper = GetVsTestConsoleWrapper(out _TempDirectory); _runEventHandler = new RunEventHandler(); _testRunAttachmentsProcessingEventHandler = new TestRunAttachmentsProcessingEventHandler(); } @@ -492,7 +492,7 @@ private void AssertCoverageResults(IList attachments) { if (attachments.Count == 1) { - var xmlCoverage = GetXmlCoverage(attachments.First().Attachments.First().Uri.LocalPath, _tempDirectory); + var xmlCoverage = GetXmlCoverage(attachments.First().Attachments.First().Uri.LocalPath, _TempDirectory); foreach (var project in GetProjects()) { diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/RunTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/RunTests.cs index c06123c493..f8b15f77c9 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/RunTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/RunTests.cs @@ -71,6 +71,9 @@ public void EndSessionShouldEnsureVstestConsoleProcessDies(RunnerInfo runnerInfo _vstestConsoleWrapper?.EndSession(); // Assert + // TODO: This still works reliably, but it is accidental. Correctly we should look at our "tree" of processes + // but there is no such thing on Windows. We can still replicate it quite well. There is code for it in blame + // hang collector. Assert.AreEqual(numOfProcesses, Process.GetProcessesByName("vstest.console").Length); _vstestConsoleWrapper = null; diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/RunTestsWithDifferentConfigurationTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/RunTestsWithDifferentConfigurationTests.cs index 00311527e8..0da7554f11 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/RunTestsWithDifferentConfigurationTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/RunTestsWithDifferentConfigurationTests.cs @@ -108,8 +108,7 @@ public void RunTestsWithTestSettings(RunnerInfo runnerInfo) ExecuteNotSupportedRunnerFrameworkTests(runnerInfo.RunnerFramework, Netcoreapp, Message); Setup(); - using var tempDir = new TempDirectory(); - var testsettingsFile = Path.Combine(tempDir.Path, "tempsettings.testsettings"); + var testsettingsFile = Path.Combine(TempDirectory.Path, "tempsettings.testsettings"); string testSettingsXml = @""; File.WriteAllText(testsettingsFile, testSettingsXml, Encoding.UTF8); diff --git a/test/Microsoft.TestPlatform.SmokeTests/DotnetHostArchitectureVerifierTests.cs b/test/Microsoft.TestPlatform.SmokeTests/DotnetHostArchitectureVerifierTests.cs index e64459980a..b701b1aa0d 100644 --- a/test/Microsoft.TestPlatform.SmokeTests/DotnetHostArchitectureVerifierTests.cs +++ b/test/Microsoft.TestPlatform.SmokeTests/DotnetHostArchitectureVerifierTests.cs @@ -28,7 +28,7 @@ public void VerifyHostArchitecture(string architecture) string dotnetPath = GetDownloadedDotnetMuxerFromTools(architecture); var vstestConsolePath = GetDotnetRunnerPath(); var dotnetRunnerPath = workSpace.CreateDirectory("dotnetrunner"); - workSpace.CopyAll(new DirectoryInfo(Path.GetDirectoryName(vstestConsolePath)), dotnetRunnerPath); + workSpace.CopyDirectory(new DirectoryInfo(Path.GetDirectoryName(vstestConsolePath)), dotnetRunnerPath); // Patch the runner string sdkVersion = GetLatestSdkVersion(dotnetPath); diff --git a/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestBase.cs b/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestBase.cs index 9d714e1033..9cb9a189e0 100644 --- a/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestBase.cs +++ b/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestBase.cs @@ -42,6 +42,8 @@ public class IntegrationTestBase protected readonly IntegrationTestEnvironment _testEnvironment; + public string BuildConfiguration { get; } + private readonly string _testAdapterRelativePath = @"mstest.testadapter\{0}\build\_common".Replace('\\', Path.DirectorySeparatorChar); private readonly string _nUnitTestAdapterRelativePath = @"nunit3testadapter\{0}\build".Replace('\\', Path.DirectorySeparatorChar); private readonly string _xUnitTestAdapterRelativePath = @"xunit.runner.visualstudio\{0}\build\_common".Replace('\\', Path.DirectorySeparatorChar); @@ -57,6 +59,8 @@ public enum UnitTestFramework public IntegrationTestBase() { _testEnvironment = new IntegrationTestEnvironment(); + BuildConfiguration = IntegrationTestEnvironment.BuildConfiguration; + TempDirectory = new TempDirectory(); } public string StdOut => _standardTestOutput; @@ -65,6 +69,20 @@ public IntegrationTestBase() public string StdErr => _standardTestError; public string StdErrWithWhiteSpace { get; private set; } = string.Empty; + public TempDirectory TempDirectory { get; } + + public TestContext TestContext { get; set; } + + [TestCleanup] + public void TempDirectoryCleanup() + { + // Delete the directory only when we passed, so we can look at results and logs of failed tests. + if (TestContext.CurrentTestOutcome == UnitTestOutcome.Passed) + { + TempDirectory.Dispose(); + } + } + /// /// Prepare arguments for vstest.console.exe. /// @@ -752,25 +770,7 @@ protected static string GetDiagArg(string rootDir) /// Counts the number of logs following the '*.host.*' pattern in the given folder. /// protected static int CountTestHostLogs(string diagLogsDir, IEnumerable testHostProcessNames) - => Directory.GetFiles(diagLogsDir, "*.host.*") - .Count(filePath => - { - var firstLine = File.ReadLines(filePath).FirstOrDefault(); - return testHostProcessNames.Any(processName => - { - var parts = processName.Split('.'); - if (parts.Length > 2) - { - throw new InvalidOperationException(""); - } - - var hostName = parts[0]; - var platformName = parts.Length > 1 ? @$"\.{parts[1]}" : string.Empty; - - var isMatch = Regex.IsMatch(firstLine, @$",\s{hostName}(?:\.net\d+)?{platformName}\.(?:exe|dll),"); - return isMatch; - }); - }); + => Directory.GetFiles(diagLogsDir, "*.host.*").Count(); protected static void AssertExpectedNumberOfHostProcesses(int expectedNumOfProcessCreated, string diagLogsDir, IEnumerable testHostProcessNames, string arguments = null, string runnerPath = null) { @@ -800,4 +800,24 @@ protected static string GetDownloadedDotnetMuxerFromTools(string architecture) } protected static string GetDotnetRunnerPath() => Path.Combine(IntegrationTestEnvironment.TestPlatformRootDirectory, "artifacts", IntegrationTestEnvironment.BuildConfiguration, "netcoreapp2.1", "vstest.console.dll"); + + public static void CopyDirectory(string sourceDirectory, string destinationDirectory) + { + if (!Directory.Exists(destinationDirectory)) + Directory.CreateDirectory(destinationDirectory); + string[] files = Directory.GetFiles(sourceDirectory); + foreach (string file in files) + { + string name = Path.GetFileName(file); + string dest = Path.Combine(destinationDirectory, name); + File.Copy(file, dest); + } + string[] folders = Directory.GetDirectories(sourceDirectory); + foreach (string folder in folders) + { + string name = Path.GetFileName(folder); + string dest = Path.Combine(destinationDirectory, name); + CopyDirectory(folder, dest); + } + } } diff --git a/test/Microsoft.TestPlatform.TestUtilities/Microsoft.TestPlatform.TestUtilities.csproj b/test/Microsoft.TestPlatform.TestUtilities/Microsoft.TestPlatform.TestUtilities.csproj index a91d6a82ca..adb97b9b5a 100644 --- a/test/Microsoft.TestPlatform.TestUtilities/Microsoft.TestPlatform.TestUtilities.csproj +++ b/test/Microsoft.TestPlatform.TestUtilities/Microsoft.TestPlatform.TestUtilities.csproj @@ -33,4 +33,4 @@ - + \ No newline at end of file diff --git a/test/Microsoft.TestPlatform.TestUtilities/RandomId.cs b/test/Microsoft.TestPlatform.TestUtilities/RandomId.cs new file mode 100644 index 0000000000..cad0f84330 --- /dev/null +++ b/test/Microsoft.TestPlatform.TestUtilities/RandomId.cs @@ -0,0 +1,46 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using System.Security.Cryptography; + +namespace Microsoft.TestPlatform.TestUtilities; + +/// +/// Slightly random id that is just good enough for creating disctinct directories for each test. +/// +public static class RandomId +{ + private static readonly RandomNumberGenerator Rng = RandomNumberGenerator.Create(); + private const string Pool = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz"; + + /// + /// 5 character long id from 0-9A-Za-z0, for example fUfko, A6uvM, sOMXa, RY1ei, KvdJZ. + /// + /// + public static string Next() + { + return Next(5); + } + + private static string Next(int length) + { + var poolLength = Pool.Length; + var id = new char[length]; + lock (Pool) + { + for (var idIndex = 0; idIndex < length; idIndex++) + { + var poolIndex = poolLength + 1; + while (poolIndex >= poolLength) + { + var bytes = new byte[1]; + Rng.GetNonZeroBytes(bytes); + poolIndex = bytes[0]; + } + id[idIndex] = Pool[poolIndex]; + } + } + + return new string(id); + } +} diff --git a/test/Microsoft.TestPlatform.TestUtilities/TempDirectory.cs b/test/Microsoft.TestPlatform.TestUtilities/TempDirectory.cs index f218d96f7a..3b45ecc3dc 100644 --- a/test/Microsoft.TestPlatform.TestUtilities/TempDirectory.cs +++ b/test/Microsoft.TestPlatform.TestUtilities/TempDirectory.cs @@ -3,6 +3,8 @@ using System; using System.IO; +using IO = System.IO; + #nullable disable @@ -26,26 +28,31 @@ public void Dispose() } public DirectoryInfo CreateDirectory(string dir) - => Directory.CreateDirectory(System.IO.Path.Combine(Path, dir)); + => Directory.CreateDirectory(IO.Path.Combine(Path, dir)); + + public void CopyDirectory(string sourceDirectory, string targetDirectory) + { + CopyDirectory(new DirectoryInfo(sourceDirectory), new DirectoryInfo(targetDirectory)); + } - public void CopyAll(DirectoryInfo source, DirectoryInfo target) + public void CopyDirectory(DirectoryInfo source, DirectoryInfo target) { Directory.CreateDirectory(target.FullName); // Copy each file into the new directory. foreach (FileInfo fi in source.GetFiles()) { - fi.CopyTo(System.IO.Path.Combine(target.FullName, fi.Name), true); + fi.CopyTo(IO.Path.Combine(target.FullName, fi.Name), true); } // Copy each subdirectory using recursion. foreach (DirectoryInfo diSourceSubDir in source.GetDirectories()) { - DirectoryInfo nextTargetSubDir = - target.CreateSubdirectory(diSourceSubDir.Name); - CopyAll(diSourceSubDir, nextTargetSubDir); + DirectoryInfo nextTargetSubDir = target.CreateSubdirectory(diSourceSubDir.Name); + CopyDirectory(diSourceSubDir, nextTargetSubDir); } } + /// /// Creates an unique temporary directory. /// @@ -54,19 +61,20 @@ public void CopyAll(DirectoryInfo source, DirectoryInfo target) /// internal static string CreateUniqueDirectory() { - // AGENT_TEMPDIRECTORY is AzureDevops variable, which is set to path - // that is cleaned up after every job. This is preferable to use over - // just the normal temp. var temp = GetTempPath(); - var directoryPath = System.IO.Path.Combine(temp, Guid.NewGuid().ToString("n")); + var directoryPath = IO.Path.Combine(temp, "vstest", RandomId.Next()); Directory.CreateDirectory(directoryPath); return directoryPath; } private static string GetTempPath() - => Environment.GetEnvironmentVariable("AGENT_TEMPDIRECTORY") - ?? System.IO.Path.GetTempPath(); + { + // AGENT_TEMPDIRECTORY is AzureDevops variable, which is set to path + // that is cleaned up after every job. This is preferable to use over + // just the normal TEMP, because that is not cleaned up for every run. + return Environment.GetEnvironmentVariable("AGENT_TEMPDIRECTORY") ?? IO.Path.GetTempPath(); + } public static void TryRemoveDirectory(string directory) { @@ -74,7 +82,7 @@ public static void TryRemoveDirectory(string directory) { try { - Directory.Delete(directory, true); + Directory.Delete(directory, recursive: true); } catch { } } From aff2f4a12872d624ae5562352ff732c5e5b09ae6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Jare=C5=A1?= Date: Fri, 4 Mar 2022 12:14:56 +0100 Subject: [PATCH 020/112] Apply suggestions from code review --- .../AppDomainTests.cs | 2 +- .../DifferentTestFrameworkSimpleTests.cs | 10 ---------- 2 files changed, 1 insertion(+), 11 deletions(-) diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/AppDomainTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/AppDomainTests.cs index 776ad1956e..235aa4c59f 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/AppDomainTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/AppDomainTests.cs @@ -29,7 +29,7 @@ public void RunTestExecutionWithDisableAppDomain(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - var testAppDomainDetailFileName = Path.Combine(TempDirectory.Path, "appdomain_test.txt"); + var testAppDomainDetailFileName = Path.Combine(TempDirectory.Path, "appdomain_test.txt"); var dataCollectorAppDomainDetailFileName = Path.Combine(TempDirectory.Path, "appdomain_datacollector.txt"); // Delete test output files if already exist diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/DifferentTestFrameworkSimpleTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/DifferentTestFrameworkSimpleTests.cs index fc3209099a..f360caeb84 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/DifferentTestFrameworkSimpleTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/DifferentTestFrameworkSimpleTests.cs @@ -65,16 +65,6 @@ public void CPPRunAllTestExecutionPlatformx64Net(RunnerInfo runnerInfo) [NetFullTargetFrameworkDataSource] public void WebTestRunAllTestsWithRunSettings(RunnerInfo runnerInfo) { - // This test works on server but not locally, and it fails with - // Exception: Method not found: 'Int64 Microsoft.VisualStudio.TestTools.WebTesting.WebTestRequestStatistics.get_MillisecondsSincePageComplete()'. - // The dll is - - //if (!IsCI) - //{ - // Assert.Inconclusive("Web load tests need special workloads and setup locally"); - //} - - SetTestEnvironment(_testEnvironment, runnerInfo); var runSettingsFilePath = Path.Combine(TempDirectory.Path, Guid.NewGuid() + ".runsettings"); From 83a9fc6bae48520da76fd9dc35291b9c322dc6e7 Mon Sep 17 00:00:00 2001 From: nohwnd Date: Fri, 4 Mar 2022 12:15:29 +0100 Subject: [PATCH 021/112] Remove more workspace --- .../DotnetArchitectureSwitchTests.Windows.cs | 3 +-- .../DotnetHostArchitectureVerifierTests.cs | 3 +-- .../IntegrationTestBase.cs | 8 ++------ 3 files changed, 4 insertions(+), 10 deletions(-) diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/DotnetArchitectureSwitchTests.Windows.cs b/test/Microsoft.TestPlatform.AcceptanceTests/DotnetArchitectureSwitchTests.Windows.cs index cbb764e423..bedcb5d62f 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/DotnetArchitectureSwitchTests.Windows.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/DotnetArchitectureSwitchTests.Windows.cs @@ -23,12 +23,11 @@ public class DotnetArchitectureSwitchTestsWindowsOnly : AcceptanceTestBase [DataRow("X86", "X64")] public void Use_EnvironmentVariables(string architectureFrom, string architectureTo) { - using var workSpace = TempDirectory; string dotnetPath = GetDownloadedDotnetMuxerFromTools(architectureFrom); string dotnetPathTo = GetDownloadedDotnetMuxerFromTools(architectureTo); var vstestConsolePath = GetDotnetRunnerPath(); var dotnetRunnerPath = workSpace.CreateDirectory("dotnetrunner"); - workSpace.CopyDirectory(new DirectoryInfo(Path.GetDirectoryName(vstestConsolePath)), dotnetRunnerPath); + TempDirectory.CopyDirectory(new DirectoryInfo(Path.GetDirectoryName(vstestConsolePath)), dotnetRunnerPath); // Patch the runner string sdkVersion = GetLatestSdkVersion(dotnetPath); diff --git a/test/Microsoft.TestPlatform.SmokeTests/DotnetHostArchitectureVerifierTests.cs b/test/Microsoft.TestPlatform.SmokeTests/DotnetHostArchitectureVerifierTests.cs index b701b1aa0d..8dfe7f2258 100644 --- a/test/Microsoft.TestPlatform.SmokeTests/DotnetHostArchitectureVerifierTests.cs +++ b/test/Microsoft.TestPlatform.SmokeTests/DotnetHostArchitectureVerifierTests.cs @@ -24,11 +24,10 @@ public class DotnetHostArchitectureVerifierTests : IntegrationTestBase [DataRow("X86")] public void VerifyHostArchitecture(string architecture) { - using var workSpace = new TempDirectory(); string dotnetPath = GetDownloadedDotnetMuxerFromTools(architecture); var vstestConsolePath = GetDotnetRunnerPath(); var dotnetRunnerPath = workSpace.CreateDirectory("dotnetrunner"); - workSpace.CopyDirectory(new DirectoryInfo(Path.GetDirectoryName(vstestConsolePath)), dotnetRunnerPath); + TempDirectory.CopyDirectory(new DirectoryInfo(Path.GetDirectoryName(vstestConsolePath)), dotnetRunnerPath); // Patch the runner string sdkVersion = GetLatestSdkVersion(dotnetPath); diff --git a/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestBase.cs b/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestBase.cs index 9cb9a189e0..2198ba74c2 100644 --- a/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestBase.cs +++ b/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestBase.cs @@ -201,9 +201,7 @@ public void InvokeVsTestForExecution(string testAssembly, string runSettings = "", Dictionary environmentVariables = null) { - using var workspace = new TempDirectory(); - - var arguments = PrepareArguments(testAssembly, testAdapterPath, runSettings, framework, _testEnvironment.InIsolationValue, resultsDirectory: workspace.Path); + var arguments = PrepareArguments(testAssembly, testAdapterPath, runSettings, framework, _testEnvironment.InIsolationValue, resultsDirectory: TempDirectory.Path); InvokeVsTest(arguments, environmentVariables); } @@ -215,9 +213,7 @@ public void InvokeVsTestForExecution(string testAssembly, /// Run settings for execution. public void InvokeVsTestForDiscovery(string testAssembly, string testAdapterPath, string runSettings = "", string targetFramework = "", Dictionary environmentVariables = null) { - using var workspace = new TempDirectory(); - - var arguments = PrepareArguments(testAssembly, testAdapterPath, runSettings, targetFramework, _testEnvironment.InIsolationValue, resultsDirectory: workspace.Path); + var arguments = PrepareArguments(testAssembly, testAdapterPath, runSettings, targetFramework, _testEnvironment.InIsolationValue, resultsDirectory: TempDirectory.Path); arguments = string.Concat(arguments, " /listtests"); InvokeVsTest(arguments, environmentVariables); } From 7536588369978d08c3be0dd5bec607bbf47789d1 Mon Sep 17 00:00:00 2001 From: nohwnd Date: Fri, 4 Mar 2022 12:20:18 +0100 Subject: [PATCH 022/112] Revert non-optmize on Release --- .../Microsoft.TestPlatform.AcceptanceTests.csproj | 6 ------ 1 file changed, 6 deletions(-) diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/Microsoft.TestPlatform.AcceptanceTests.csproj b/test/Microsoft.TestPlatform.AcceptanceTests/Microsoft.TestPlatform.AcceptanceTests.csproj index 890ed63c39..31d535775e 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/Microsoft.TestPlatform.AcceptanceTests.csproj +++ b/test/Microsoft.TestPlatform.AcceptanceTests/Microsoft.TestPlatform.AcceptanceTests.csproj @@ -12,12 +12,6 @@ netcoreapp3.1 Microsoft.TestPlatform.AcceptanceTests - - False - - - False - From a94b1bd7c6b1b554d69022ac4c107999ebf862a4 Mon Sep 17 00:00:00 2001 From: nohwnd Date: Fri, 4 Mar 2022 12:30:51 +0100 Subject: [PATCH 023/112] More fixes --- .../DotnetArchitectureSwitchTests.Windows.cs | 12 +++++------ .../IntegrationTestBase.cs | 20 ------------------- 2 files changed, 6 insertions(+), 26 deletions(-) diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/DotnetArchitectureSwitchTests.Windows.cs b/test/Microsoft.TestPlatform.AcceptanceTests/DotnetArchitectureSwitchTests.Windows.cs index bedcb5d62f..3067a8bcbd 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/DotnetArchitectureSwitchTests.Windows.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/DotnetArchitectureSwitchTests.Windows.cs @@ -26,7 +26,7 @@ public void Use_EnvironmentVariables(string architectureFrom, string architectur string dotnetPath = GetDownloadedDotnetMuxerFromTools(architectureFrom); string dotnetPathTo = GetDownloadedDotnetMuxerFromTools(architectureTo); var vstestConsolePath = GetDotnetRunnerPath(); - var dotnetRunnerPath = workSpace.CreateDirectory("dotnetrunner"); + var dotnetRunnerPath = TempDirectory.CreateDirectory("dotnetrunner"); TempDirectory.CopyDirectory(new DirectoryInfo(Path.GetDirectoryName(vstestConsolePath)), dotnetRunnerPath); // Patch the runner @@ -42,10 +42,10 @@ public void Use_EnvironmentVariables(string architectureFrom, string architectur [$"DOTNET_ROOT_{architectureTo}"] = Path.GetDirectoryName(dotnetPathTo), ["ExpectedArchitecture"] = architectureTo }; - ExecuteApplication(dotnetPath, "new mstest", out string stdOut, out string stdError, out int exitCode, environmentVariables, workSpace.Path); + ExecuteApplication(dotnetPath, "new mstest", out string stdOut, out string stdError, out int exitCode, environmentVariables, TempDirectory.Path); // Patch test file - File.WriteAllText(Path.Combine(workSpace.Path, "UnitTest1.cs"), + File.WriteAllText(Path.Combine(TempDirectory.Path, "UnitTest1.cs"), @" using Microsoft.VisualStudio.TestTools.UnitTesting; using System; @@ -64,7 +64,7 @@ public void TestMethod1() } }"); - ExecuteApplication(dotnetPath, $"test -p:VsTestConsolePath=\"{Path.Combine(dotnetRunnerPath.FullName, Path.GetFileName(vstestConsolePath))}\" --arch {architectureTo.ToLower()} --diag:log.txt", out stdOut, out stdError, out exitCode, environmentVariables, workSpace.Path); + ExecuteApplication(dotnetPath, $"test -p:VsTestConsolePath=\"{Path.Combine(dotnetRunnerPath.FullName, Path.GetFileName(vstestConsolePath))}\" --arch {architectureTo.ToLower()} --diag:log.txt", out stdOut, out stdError, out exitCode, environmentVariables, TempDirectory.Path); Assert.AreEqual(0, exitCode, stdOut); environmentVariables = new Dictionary @@ -73,7 +73,7 @@ public void TestMethod1() ["DOTNET_ROOT"] = Path.GetDirectoryName(dotnetPathTo), ["ExpectedArchitecture"] = architectureTo }; - ExecuteApplication(dotnetPath, $"test -p:VsTestConsolePath=\"{Path.Combine(dotnetRunnerPath.FullName, Path.GetFileName(vstestConsolePath))}\" --arch {architectureTo.ToLower()} --diag:log.txt", out stdOut, out stdError, out exitCode, environmentVariables, workSpace.Path); + ExecuteApplication(dotnetPath, $"test -p:VsTestConsolePath=\"{Path.Combine(dotnetRunnerPath.FullName, Path.GetFileName(vstestConsolePath))}\" --arch {architectureTo.ToLower()} --diag:log.txt", out stdOut, out stdError, out exitCode, environmentVariables, TempDirectory.Path); Assert.AreEqual(0, exitCode, stdOut); environmentVariables = new Dictionary @@ -83,7 +83,7 @@ public void TestMethod1() ["DOTNET_ROOT"] = "WE SHOULD PICK THE ABOVE ONE BEFORE FALLBACK TO DOTNET_ROOT", ["ExpectedArchitecture"] = architectureTo }; - ExecuteApplication(dotnetPath, $"test -p:VsTestConsolePath=\"{Path.Combine(dotnetRunnerPath.FullName, Path.GetFileName(vstestConsolePath))}\" --arch {architectureTo.ToLower()} --diag:log.txt", out stdOut, out stdError, out exitCode, environmentVariables, workSpace.Path); + ExecuteApplication(dotnetPath, $"test -p:VsTestConsolePath=\"{Path.Combine(dotnetRunnerPath.FullName, Path.GetFileName(vstestConsolePath))}\" --arch {architectureTo.ToLower()} --diag:log.txt", out stdOut, out stdError, out exitCode, environmentVariables, TempDirectory.Path); Assert.AreEqual(0, exitCode, stdOut); } diff --git a/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestBase.cs b/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestBase.cs index 2198ba74c2..91cc6c019a 100644 --- a/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestBase.cs +++ b/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestBase.cs @@ -796,24 +796,4 @@ protected static string GetDownloadedDotnetMuxerFromTools(string architecture) } protected static string GetDotnetRunnerPath() => Path.Combine(IntegrationTestEnvironment.TestPlatformRootDirectory, "artifacts", IntegrationTestEnvironment.BuildConfiguration, "netcoreapp2.1", "vstest.console.dll"); - - public static void CopyDirectory(string sourceDirectory, string destinationDirectory) - { - if (!Directory.Exists(destinationDirectory)) - Directory.CreateDirectory(destinationDirectory); - string[] files = Directory.GetFiles(sourceDirectory); - foreach (string file in files) - { - string name = Path.GetFileName(file); - string dest = Path.Combine(destinationDirectory, name); - File.Copy(file, dest); - } - string[] folders = Directory.GetDirectories(sourceDirectory); - foreach (string folder in folders) - { - string name = Path.GetFileName(folder); - string dest = Path.Combine(destinationDirectory, name); - CopyDirectory(folder, dest); - } - } } From ef668f074b87165bac84d6d2289944bae06987e1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Jare=C5=A1?= Date: Fri, 4 Mar 2022 12:31:18 +0100 Subject: [PATCH 024/112] Update test/Microsoft.TestPlatform.TestUtilities/IntegrationTestBase.cs --- .../Microsoft.TestPlatform.TestUtilities/IntegrationTestBase.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestBase.cs b/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestBase.cs index 91cc6c019a..8dd9c262f7 100644 --- a/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestBase.cs +++ b/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestBase.cs @@ -76,7 +76,7 @@ public IntegrationTestBase() [TestCleanup] public void TempDirectoryCleanup() { - // Delete the directory only when we passed, so we can look at results and logs of failed tests. + // Delete the directory only when the test succeeded, so we can look at results and logs of failed tests. if (TestContext.CurrentTestOutcome == UnitTestOutcome.Passed) { TempDirectory.Dispose(); From 8b04531092c1729b4ff52c8f60d52be706c59223 Mon Sep 17 00:00:00 2001 From: nohwnd Date: Fri, 4 Mar 2022 12:56:06 +0100 Subject: [PATCH 025/112] Fix unit test --- .../EventLogDataCollectorTests.cs | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/test/DataCollectors/Microsoft.TestPlatform.Extensions.EventLogCollector.UnitTests/EventLogDataCollectorTests.cs b/test/DataCollectors/Microsoft.TestPlatform.Extensions.EventLogCollector.UnitTests/EventLogDataCollectorTests.cs index 3096009d27..473c6b8af6 100644 --- a/test/DataCollectors/Microsoft.TestPlatform.Extensions.EventLogCollector.UnitTests/EventLogDataCollectorTests.cs +++ b/test/DataCollectors/Microsoft.TestPlatform.Extensions.EventLogCollector.UnitTests/EventLogDataCollectorTests.cs @@ -217,11 +217,10 @@ public void InitializeShouldSubscribeToDataCollectionEvents() { var testableDataCollectionEvents = new TestableDataCollectionEvents(); _eventLogDataCollector.Initialize(null, testableDataCollectionEvents, _mockDataCollectionSink, _mockDataCollectionLogger.Object, _dataCollectionEnvironmentContext); - Assert.AreEqual(1, testableDataCollectionEvents.GetTestHostLaunchedInvocationList().Length); - Assert.AreEqual(1, testableDataCollectionEvents.GetTestCaseStartInvocationList().Length); - Assert.AreEqual(1, testableDataCollectionEvents.GetTestCaseEndInvocationList().Length); - Assert.AreEqual(1, testableDataCollectionEvents.GetTestSessionEndInvocationList().Length); - Assert.AreEqual(1, testableDataCollectionEvents.GetTestSessionStartInvocationList().Length); + Assert.AreEqual(1, testableDataCollectionEvents.GetTestCaseStartInvocationList()?.Length, "GetTestCaseStartInvocationList"); + Assert.AreEqual(1, testableDataCollectionEvents.GetTestCaseEndInvocationList()?.Length, "GetTestCaseEndInvocationList"); + Assert.AreEqual(1, testableDataCollectionEvents.GetTestSessionEndInvocationList()?.Length, "GetTestSessionEndInvocationList"); + Assert.AreEqual(1, testableDataCollectionEvents.GetTestSessionStartInvocationList().Length, "GetTestSessionStartInvocationList"); } [TestMethod] From e57fe1d281ded602c16723156f287e55db39288c Mon Sep 17 00:00:00 2001 From: nohwnd Date: Fri, 4 Mar 2022 12:57:00 +0100 Subject: [PATCH 026/112] Missing null operator --- .../EventLogDataCollectorTests.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/DataCollectors/Microsoft.TestPlatform.Extensions.EventLogCollector.UnitTests/EventLogDataCollectorTests.cs b/test/DataCollectors/Microsoft.TestPlatform.Extensions.EventLogCollector.UnitTests/EventLogDataCollectorTests.cs index 473c6b8af6..aca802aa53 100644 --- a/test/DataCollectors/Microsoft.TestPlatform.Extensions.EventLogCollector.UnitTests/EventLogDataCollectorTests.cs +++ b/test/DataCollectors/Microsoft.TestPlatform.Extensions.EventLogCollector.UnitTests/EventLogDataCollectorTests.cs @@ -220,7 +220,7 @@ public void InitializeShouldSubscribeToDataCollectionEvents() Assert.AreEqual(1, testableDataCollectionEvents.GetTestCaseStartInvocationList()?.Length, "GetTestCaseStartInvocationList"); Assert.AreEqual(1, testableDataCollectionEvents.GetTestCaseEndInvocationList()?.Length, "GetTestCaseEndInvocationList"); Assert.AreEqual(1, testableDataCollectionEvents.GetTestSessionEndInvocationList()?.Length, "GetTestSessionEndInvocationList"); - Assert.AreEqual(1, testableDataCollectionEvents.GetTestSessionStartInvocationList().Length, "GetTestSessionStartInvocationList"); + Assert.AreEqual(1, testableDataCollectionEvents.GetTestSessionStartInvocationList()?.Length, "GetTestSessionStartInvocationList"); } [TestMethod] From 98fa6b91517e3497291d022d18f132d806d33863 Mon Sep 17 00:00:00 2001 From: nohwnd Date: Fri, 4 Mar 2022 13:03:59 +0100 Subject: [PATCH 027/112] Revert incorrect rename --- scripts/build.ps1 | 1 - .../AppDomainTests.cs | 4 ++-- .../BlameDataCollectorTests.cs | 4 ++-- .../CodeCoverageAcceptanceTestBase.cs | 4 ++-- .../CodeCoverageTests.cs | 8 ++++---- .../DisableAppdomainTests.cs | 4 ++-- .../EventLogCollectorTests.cs | 8 ++++---- .../RunsettingsTests.cs | 4 ++-- .../TelemetryTests.cs | 8 ++++---- 9 files changed, 22 insertions(+), 23 deletions(-) diff --git a/scripts/build.ps1 b/scripts/build.ps1 index c108e8eced..4fe054862f 100644 --- a/scripts/build.ps1 +++ b/scripts/build.ps1 @@ -140,7 +140,6 @@ if ((Test-Path $procdumpDir) -and 2 -ne @(Get-Item "$procdumpDir\0.0.1\bin").Len Remove-Item -Recurse -Force $procdumpDir } - function Invoke-Build { $timer = Start-Timer diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/AppDomainTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/AppDomainTests.cs index 235aa4c59f..82676c5ae2 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/AppDomainTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/AppDomainTests.cs @@ -73,9 +73,9 @@ private static bool IsFilesContentEqual(string filePath1, string filePath2) return string.Equals(content1, content2, StringComparison.Ordinal); } - private string GetInProcDataCollectionRunsettingsFile(bool disableAppDomain, TempDirectory TempDirectory) + private string GetInProcDataCollectionRunsettingsFile(bool disableAppDomain, TempDirectory tempDirectory) { - var runSettings = Path.Combine(TempDirectory.Path, "test_" + Guid.NewGuid() + ".runsettings"); + var runSettings = Path.Combine(tempDirectory.Path, "test_" + Guid.NewGuid() + ".runsettings"); var inprocasm = _testEnvironment.GetTestAsset("SimpleDataCollector.dll"); #if !NET451 var assemblyName = AssemblyLoadContext.GetAssemblyName(inprocasm); diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/BlameDataCollectorTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/BlameDataCollectorTests.cs index fc00b22d7a..93a88e9894 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/BlameDataCollectorTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/BlameDataCollectorTests.cs @@ -323,14 +323,14 @@ private void ValidateDump(int expectedDumpCount = 1) } } - private void VaildateOutput(TempDirectory TempDirectory, string testName, bool validateDumpFile = false) + private void VaildateOutput(TempDirectory tempDirectory, string testName, bool validateDumpFile = false) { bool isSequenceAttachmentReceived = false; bool isDumpAttachmentReceived = false; bool isValid = false; StdErrorContains(testName); StdOutputContains("Sequence_"); - var resultFiles = Directory.GetFiles(TempDirectory.Path, "*", SearchOption.AllDirectories); + var resultFiles = Directory.GetFiles(tempDirectory.Path, "*", SearchOption.AllDirectories); foreach (var file in resultFiles) { diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/CodeCoverageAcceptanceTestBase.cs b/test/Microsoft.TestPlatform.AcceptanceTests/CodeCoverageAcceptanceTestBase.cs index b8a44734f8..2081de653b 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/CodeCoverageAcceptanceTestBase.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/CodeCoverageAcceptanceTestBase.cs @@ -59,7 +59,7 @@ protected XmlNode GetNode(XmlNode node, string type, string name) return node.SelectSingleNode($"//{type}[@name='{name}']") ?? node.SelectSingleNode($"//{type}[@name='{name.ToLower()}']"); } - protected XmlDocument GetXmlCoverage(string coverageResult, TempDirectory TempDirectory) + protected XmlDocument GetXmlCoverage(string coverageResult, TempDirectory tempDirectory) { var coverage = new XmlDocument(); @@ -70,7 +70,7 @@ protected XmlDocument GetXmlCoverage(string coverageResult, TempDirectory TempDi } var codeCoverageExe = GetCodeCoverageExePath(); - var output = Path.Combine(TempDirectory.Path, Guid.NewGuid().ToString() + ".xml"); + var output = Path.Combine(tempDirectory.Path, Guid.NewGuid().ToString() + ".xml"); var watch = new Stopwatch(); diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/CodeCoverageTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/CodeCoverageTests.cs index 1dc4c3f618..b426fc7fd1 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/CodeCoverageTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/CodeCoverageTests.cs @@ -246,7 +246,7 @@ private void CollectCodeCoverage(RunnerInfo runnerInfo, TestParameters testParam } private string CreateArguments( - TempDirectory TempDirectory, + TempDirectory tempDirectory, RunnerInfo runnerInfo, TestParameters testParameters, out string trxFilePath) @@ -256,14 +256,14 @@ private string CreateArguments( string traceDataCollectorDir = Path.Combine(IntegrationTestEnvironment.TestPlatformRootDirectory, "artifacts", IntegrationTestEnvironment.BuildConfiguration, "Microsoft.CodeCoverage"); - string diagFileName = Path.Combine(TempDirectory.Path, "diaglog.txt"); + string diagFileName = Path.Combine(tempDirectory.Path, "diaglog.txt"); var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, - FrameworkArgValue, runnerInfo.InIsolationValue, TempDirectory.Path); + FrameworkArgValue, runnerInfo.InIsolationValue, tempDirectory.Path); arguments = string.Concat(arguments, $" /Diag:{diagFileName}", $" /TestAdapterPath:{traceDataCollectorDir}"); arguments = string.Concat(arguments, $" /Platform:{testParameters.TargetPlatform}"); - trxFilePath = Path.Combine(TempDirectory.Path, Guid.NewGuid() + ".trx"); + trxFilePath = Path.Combine(tempDirectory.Path, Guid.NewGuid() + ".trx"); arguments = string.Concat(arguments, " /logger:trx;logfilename=" + trxFilePath); var defaultRunSettingsPath = Path.Combine( diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/DisableAppdomainTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/DisableAppdomainTests.cs index 78add6fe64..4136000193 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/DisableAppdomainTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/DisableAppdomainTests.cs @@ -64,9 +64,9 @@ private void RunTests(RunnerInfo runnerInfo, string testAssembly, int passedTest ValidateSummaryStatus(passedTestCount, 0, 0); } - private string GetRunsettingsFilePath(TempDirectory TempDirectory, Dictionary runConfigurationDictionary) + private string GetRunsettingsFilePath(TempDirectory tempDirectory, Dictionary runConfigurationDictionary) { - var runsettingsPath = Path.Combine(TempDirectory.Path, "test_" + Guid.NewGuid() + ".runsettings"); + var runsettingsPath = Path.Combine(tempDirectory.Path, "test_" + Guid.NewGuid() + ".runsettings"); CreateRunSettingsFile(runsettingsPath, runConfigurationDictionary); return runsettingsPath; } diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/EventLogCollectorTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/EventLogCollectorTests.cs index b7e80ebf5e..12cee6580a 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/EventLogCollectorTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/EventLogCollectorTests.cs @@ -61,9 +61,9 @@ public void EventLogDataCollectorShoudCreateLogFileWithoutEventsIfEventsAreNotLo StdErrorDoesNotContains("Unable to read event log"); } - private string GetRunsettingsFilePath(TempDirectory TempDirectory) + private string GetRunsettingsFilePath(TempDirectory tempDirectory) { - var runsettingsPath = Path.Combine(TempDirectory.Path, "test_" + Guid.NewGuid() + ".runsettings"); + var runsettingsPath = Path.Combine(tempDirectory.Path, "test_" + Guid.NewGuid() + ".runsettings"); string runSettingsXml = @" @@ -85,10 +85,10 @@ private string GetRunsettingsFilePath(TempDirectory TempDirectory) return runsettingsPath; } - private void VaildateDataCollectorOutput(TempDirectory TempDirectory) + private void VaildateDataCollectorOutput(TempDirectory tempDirectory) { // Verify attachments - var di = new DirectoryInfo(TempDirectory.Path); + var di = new DirectoryInfo(tempDirectory.Path); var resultFiles = di.EnumerateFiles("Event Log.xml", SearchOption.AllDirectories) .OrderBy(d => d.CreationTime) .Select(d => d.FullName) diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/RunsettingsTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/RunsettingsTests.cs index 8ea5a48b5c..edfa13bcad 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/RunsettingsTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/RunsettingsTests.cs @@ -522,9 +522,9 @@ public void RunSettingsAreLoadedFromProject(RunnerInfo runnerInfo) #endregion - private string GetRunsettingsFilePath(Dictionary runConfigurationDictionary, TempDirectory TempDirectory) + private string GetRunsettingsFilePath(Dictionary runConfigurationDictionary, TempDirectory tempDirectory) { - var runsettingsPath = Path.Combine(TempDirectory.Path, "test_" + Guid.NewGuid() + ".runsettings"); + var runsettingsPath = Path.Combine(tempDirectory.Path, "test_" + Guid.NewGuid() + ".runsettings"); if (runConfigurationDictionary != null) { CreateRunSettingsFile(runsettingsPath, runConfigurationDictionary); diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/TelemetryTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/TelemetryTests.cs index 2083b84bde..fdf49097ce 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/TelemetryTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/TelemetryTests.cs @@ -83,15 +83,15 @@ private void DiscoverTests(RunnerInfo runnerInfo) ValidateOutput("Discovery", TempDirectory); } - private void ValidateOutput(string command, TempDirectory TempDirectory) + private void ValidateOutput(string command, TempDirectory tempDirectory) { - if (!Directory.Exists(TempDirectory.Path)) + if (!Directory.Exists(tempDirectory.Path)) { - Assert.Fail("Could not find the telemetry logs folder at {0}", TempDirectory.Path); + Assert.Fail("Could not find the telemetry logs folder at {0}", tempDirectory.Path); } bool isValid = false; - var directory = new DirectoryInfo(TempDirectory.Path); + var directory = new DirectoryInfo(tempDirectory.Path); var file = directory.GetFiles().OrderByDescending(f => f.CreationTime).First(); string[] lines = File.ReadAllLines(file.FullName); From 6342ee43ad25f54698b8fbe33248fc38015b2be4 Mon Sep 17 00:00:00 2001 From: nohwnd Date: Fri, 4 Mar 2022 13:20:59 +0100 Subject: [PATCH 028/112] Fix review remarks --- .../BlameDataCollectorTests.cs | 4 ++-- .../DifferentTestFrameworkSimpleTests.cs | 12 ++++++------ .../Extension/RunnnerInfo.cs | 6 ++++-- .../RunsettingsTests.cs | 4 ++-- .../TestCaseFilterTests.cs | 4 ++-- .../IntegrationTestBase.cs | 10 +++++----- 6 files changed, 21 insertions(+), 19 deletions(-) diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/BlameDataCollectorTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/BlameDataCollectorTests.cs index 93a88e9894..671a91618d 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/BlameDataCollectorTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/BlameDataCollectorTests.cs @@ -33,8 +33,8 @@ public BlameDataCollectorTests() if (!File.Exists(procDumpExePath)) { throw new InvalidOperationException($"Procdump path {procDumpExePath} does not exist. " - + $"It is possible that antivirus deleted it from your nuget cache. " - + $"Delete the whole procdump folder in your nuget cache, and run build, or restore"); + + "It is possible that antivirus deleted it from your nuget cache. " + + "Delete the whole procdump folder in your nuget cache, and run build, or restore"); } } diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/DifferentTestFrameworkSimpleTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/DifferentTestFrameworkSimpleTests.cs index f360caeb84..26dfa7ab26 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/DifferentTestFrameworkSimpleTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/DifferentTestFrameworkSimpleTests.cs @@ -83,7 +83,7 @@ public void WebTestRunAllTestsWithRunSettings(RunnerInfo runnerInfo) var minWebTestResultFileSizeInKB = 150; if (runnerInfo.IsNetRunner) { - Assert.Inconclusive("WebTests tests not supported with .Netcore runner."); + Assert.Inconclusive("WebTests tests not supported with .NET Core runner."); return; } @@ -120,7 +120,7 @@ public void CodedWebTestRunAllTests(RunnerInfo runnerInfo) SetTestEnvironment(_testEnvironment, runnerInfo); if (runnerInfo.IsNetRunner) { - Assert.Inconclusive("WebTests tests not supported with .Netcore runner."); + Assert.Inconclusive("WebTests tests not supported with .NET Core runner."); return; } @@ -143,10 +143,10 @@ public void NUnitRunAllTestExecution(RunnerInfo runnerInfo) SetTestEnvironment(_testEnvironment, runnerInfo); var arguments = PrepareArguments( -GetAssetFullPath("NUTestProject.dll"), -GetTestAdapterPath(UnitTestFramework.NUnit), -string.Empty, FrameworkArgValue, -runnerInfo.InIsolationValue, TempDirectory.Path); + GetAssetFullPath("NUTestProject.dll"), + GetTestAdapterPath(UnitTestFramework.NUnit), + string.Empty, FrameworkArgValue, + runnerInfo.InIsolationValue, TempDirectory.Path); InvokeVsTest(arguments); ValidateSummaryStatus(1, 1, 0); } diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/RunnnerInfo.cs b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/RunnnerInfo.cs index bf7c89fb1d..5357734779 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/RunnnerInfo.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/RunnnerInfo.cs @@ -3,6 +3,8 @@ #nullable disable +using System; + namespace Microsoft.TestPlatform.AcceptanceTests; public class RunnerInfo @@ -17,10 +19,10 @@ public RunnerInfo(string runnerType, string targetFramework, string inIsolation) TargetFramework = targetFramework; InIsolationValue = inIsolation; // The value is netcoreapp2.1. - IsNetRunner = RunnerFramework.StartsWith("netcoreapp"); + IsNetRunner = RunnerFramework.StartsWith("netcoreapp", StringComparison.InvariantCultureIgnoreCase); // The value is net451. IsNetFrameworkRunner = !IsNetRunner; - IsNetTarget = TargetFramework.StartsWith("netcoreapp"); + IsNetTarget = TargetFramework.StartsWith("netcoreapp", StringComparison.InvariantCultureIgnoreCase); IsNetFrameworkTarget = !IsNetTarget; } diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/RunsettingsTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/RunsettingsTests.cs index edfa13bcad..2a0809cbaa 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/RunsettingsTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/RunsettingsTests.cs @@ -242,7 +242,7 @@ public void RunSettingsWithInvalidValueShouldLogError(RunnerInfo runnerInfo) var runConfigurationDictionary = new Dictionary { - { "TargetPlatform", "123" } + { "TargetPlatform", "123" } }; var runsettingsFilePath = GetRunsettingsFilePath(runConfigurationDictionary, TempDirectory); var arguments = PrepareArguments( @@ -263,7 +263,7 @@ public void TestAdapterPathFromRunSettings(RunnerInfo runnerInfo) var runConfigurationDictionary = new Dictionary { - { "TestAdaptersPaths", GetTestAdapterPath() } + { "TestAdaptersPaths", GetTestAdapterPath() } }; var runsettingsFilePath = GetRunsettingsFilePath(runConfigurationDictionary, TempDirectory); var arguments = PrepareArguments( diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/TestCaseFilterTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/TestCaseFilterTests.cs index 9f90e13949..c4363cf825 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/TestCaseFilterTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/TestCaseFilterTests.cs @@ -165,7 +165,7 @@ public void DiscoverMstestV1TestsWithAndOperatorTrait(RunnerInfo runnerInfo) { if (runnerInfo.IsNetRunner) { - Assert.Inconclusive("Mstest v1 tests not supported with .Netcore runner."); + Assert.Inconclusive("Mstest v1 tests not supported with .NET Core runner."); return; } @@ -197,7 +197,7 @@ public void DiscoverTmiTestsWithOnlyPropertyValue(RunnerInfo runnerInfo) { if (runnerInfo.IsNetRunner) { - Assert.Inconclusive("Tmi tests not supported with .Netcore runner."); + Assert.Inconclusive("Tmi tests not supported with .NET Core runner."); return; } diff --git a/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestBase.cs b/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestBase.cs index 8dd9c262f7..9f6b567252 100644 --- a/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestBase.cs +++ b/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestBase.cs @@ -42,8 +42,6 @@ public class IntegrationTestBase protected readonly IntegrationTestEnvironment _testEnvironment; - public string BuildConfiguration { get; } - private readonly string _testAdapterRelativePath = @"mstest.testadapter\{0}\build\_common".Replace('\\', Path.DirectorySeparatorChar); private readonly string _nUnitTestAdapterRelativePath = @"nunit3testadapter\{0}\build".Replace('\\', Path.DirectorySeparatorChar); private readonly string _xUnitTestAdapterRelativePath = @"xunit.runner.visualstudio\{0}\build\_common".Replace('\\', Path.DirectorySeparatorChar); @@ -73,6 +71,8 @@ public IntegrationTestBase() public TestContext TestContext { get; set; } + public string BuildConfiguration { get; } + [TestCleanup] public void TempDirectoryCleanup() { @@ -765,12 +765,12 @@ protected static string GetDiagArg(string rootDir) /// /// Counts the number of logs following the '*.host.*' pattern in the given folder. /// - protected static int CountTestHostLogs(string diagLogsDir, IEnumerable testHostProcessNames) - => Directory.GetFiles(diagLogsDir, "*.host.*").Count(); + protected static int CountTestHostLogs(string diagLogsDir) + => Directory.GetFiles(diagLogsDir, "*.host.*").Length; protected static void AssertExpectedNumberOfHostProcesses(int expectedNumOfProcessCreated, string diagLogsDir, IEnumerable testHostProcessNames, string arguments = null, string runnerPath = null) { - var processCreatedCount = CountTestHostLogs(diagLogsDir, testHostProcessNames); + var processCreatedCount = CountTestHostLogs(diagLogsDir); Assert.AreEqual( expectedNumOfProcessCreated, processCreatedCount, From 4f95d819eb975c5212fab64d91c05a8ecc9201a5 Mon Sep 17 00:00:00 2001 From: nohwnd Date: Fri, 4 Mar 2022 13:44:59 +0100 Subject: [PATCH 029/112] Build failures --- .../DotnetArchitectureSwitchTests.Windows.cs | 8 ++++---- .../Extension/RunnnerInfo.cs | 2 +- .../TranslationLayerTests/CodeCoverageTests.cs | 4 ++-- .../DotnetHostArchitectureVerifierTests.cs | 8 ++++---- 4 files changed, 11 insertions(+), 11 deletions(-) diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/DotnetArchitectureSwitchTests.Windows.cs b/test/Microsoft.TestPlatform.AcceptanceTests/DotnetArchitectureSwitchTests.Windows.cs index daa6c32b05..e94f1b83b3 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/DotnetArchitectureSwitchTests.Windows.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/DotnetArchitectureSwitchTests.Windows.cs @@ -43,7 +43,7 @@ public void Use_EnvironmentVariables(string architectureFrom, string architectur [$"DOTNET_ROOT_{architectureTo}"] = Path.GetDirectoryName(dotnetPathTo), ["ExpectedArchitecture"] = architectureTo }; - ExecuteApplication(dotnetPath, "new mstest", out string stdOut, out string stdError, out int exitCode, environmentVariables, TempDirectory.Path); + ExecuteApplication(dotnetPath, "new mstest", out _, out string _, out _, environmentVariables, TempDirectory.Path); // Patch test file File.WriteAllText(Path.Combine(TempDirectory.Path, "UnitTest1.cs"), @@ -65,7 +65,7 @@ public void TestMethod1() } }"); - ExecuteApplication(dotnetPath, $"test -p:VsTestConsolePath=\"{Path.Combine(dotnetRunnerPath.FullName, Path.GetFileName(vstestConsolePath))}\" --arch {architectureTo.ToLower()} --diag:log.txt", out stdOut, out stdError, out exitCode, environmentVariables, TempDirectory.Path); + ExecuteApplication(dotnetPath, $"test -p:VsTestConsolePath=\"{Path.Combine(dotnetRunnerPath.FullName, Path.GetFileName(vstestConsolePath))}\" --arch {architectureTo.ToLower()} --diag:log.txt", out string stdOut, out _, out int exitCode, environmentVariables, TempDirectory.Path); Assert.AreEqual(0, exitCode, stdOut); environmentVariables = new Dictionary @@ -74,7 +74,7 @@ public void TestMethod1() ["DOTNET_ROOT"] = Path.GetDirectoryName(dotnetPathTo), ["ExpectedArchitecture"] = architectureTo }; - ExecuteApplication(dotnetPath, $"test -p:VsTestConsolePath=\"{Path.Combine(dotnetRunnerPath.FullName, Path.GetFileName(vstestConsolePath))}\" --arch {architectureTo.ToLower()} --diag:log.txt", out stdOut, out stdError, out exitCode, environmentVariables, TempDirectory.Path); + ExecuteApplication(dotnetPath, $"test -p:VsTestConsolePath=\"{Path.Combine(dotnetRunnerPath.FullName, Path.GetFileName(vstestConsolePath))}\" --arch {architectureTo.ToLower()} --diag:log.txt", out stdOut, out _, out exitCode, environmentVariables, TempDirectory.Path); Assert.AreEqual(0, exitCode, stdOut); environmentVariables = new Dictionary @@ -84,7 +84,7 @@ public void TestMethod1() ["DOTNET_ROOT"] = "WE SHOULD PICK THE ABOVE ONE BEFORE FALLBACK TO DOTNET_ROOT", ["ExpectedArchitecture"] = architectureTo }; - ExecuteApplication(dotnetPath, $"test -p:VsTestConsolePath=\"{Path.Combine(dotnetRunnerPath.FullName, Path.GetFileName(vstestConsolePath))}\" --arch {architectureTo.ToLower()} --diag:log.txt", out stdOut, out stdError, out exitCode, environmentVariables, TempDirectory.Path); + ExecuteApplication(dotnetPath, $"test -p:VsTestConsolePath=\"{Path.Combine(dotnetRunnerPath.FullName, Path.GetFileName(vstestConsolePath))}\" --arch {architectureTo.ToLower()} --diag:log.txt", out stdOut, out _, out exitCode, environmentVariables, TempDirectory.Path); Assert.AreEqual(0, exitCode, stdOut); } diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/RunnnerInfo.cs b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/RunnnerInfo.cs index 5357734779..d9f9a14a18 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/RunnnerInfo.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/RunnnerInfo.cs @@ -24,8 +24,8 @@ public RunnerInfo(string runnerType, string targetFramework, string inIsolation) IsNetFrameworkRunner = !IsNetRunner; IsNetTarget = TargetFramework.StartsWith("netcoreapp", StringComparison.InvariantCultureIgnoreCase); IsNetFrameworkTarget = !IsNetTarget; - } + /// /// Gets the target framework. /// diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/CodeCoverageTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/CodeCoverageTests.cs index dded0392f4..e11224712c 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/CodeCoverageTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/CodeCoverageTests.cs @@ -29,7 +29,7 @@ namespace Microsoft.TestPlatform.AcceptanceTests.TranslationLayerTests; public class CodeCoverageTests : CodeCoverageAcceptanceTestBase { private IVsTestConsoleWrapper _vstestConsoleWrapper; - private TempDirectory _TempDirectory; + private TempDirectory _tempDirectory; private RunEventHandler _runEventHandler; private TestRunAttachmentsProcessingEventHandler _testRunAttachmentsProcessingEventHandler; @@ -42,7 +42,7 @@ static CodeCoverageTests() private void Setup() { - _vstestConsoleWrapper = GetVsTestConsoleWrapper(out _TempDirectory); + _vstestConsoleWrapper = GetVsTestConsoleWrapper(out _tempDirectory); _runEventHandler = new RunEventHandler(); _testRunAttachmentsProcessingEventHandler = new TestRunAttachmentsProcessingEventHandler(); } diff --git a/test/Microsoft.TestPlatform.SmokeTests/DotnetHostArchitectureVerifierTests.cs b/test/Microsoft.TestPlatform.SmokeTests/DotnetHostArchitectureVerifierTests.cs index 8dfe7f2258..676d55c50c 100644 --- a/test/Microsoft.TestPlatform.SmokeTests/DotnetHostArchitectureVerifierTests.cs +++ b/test/Microsoft.TestPlatform.SmokeTests/DotnetHostArchitectureVerifierTests.cs @@ -26,7 +26,7 @@ public void VerifyHostArchitecture(string architecture) { string dotnetPath = GetDownloadedDotnetMuxerFromTools(architecture); var vstestConsolePath = GetDotnetRunnerPath(); - var dotnetRunnerPath = workSpace.CreateDirectory("dotnetrunner"); + var dotnetRunnerPath = TempDirectory.CreateDirectory("dotnetrunner"); TempDirectory.CopyDirectory(new DirectoryInfo(Path.GetDirectoryName(vstestConsolePath)), dotnetRunnerPath); // Patch the runner @@ -42,10 +42,10 @@ public void VerifyHostArchitecture(string architecture) ["ExpectedArchitecture"] = architecture }; - ExecuteApplication(dotnetPath, "new mstest", out string stdOut, out string stdError, out int exitCode, environmentVariables, workSpace.Path); + ExecuteApplication(dotnetPath, "new mstest", out _, out _, out _, environmentVariables, TempDirectory.Path); // Patch test file - File.WriteAllText(Path.Combine(workSpace.Path, "UnitTest1.cs"), + File.WriteAllText(Path.Combine(TempDirectory.Path, "UnitTest1.cs"), @" using Microsoft.VisualStudio.TestTools.UnitTesting; using System; @@ -62,7 +62,7 @@ public void TestMethod1() } }"); - ExecuteApplication(dotnetPath, $"test -p:VsTestConsolePath=\"{Path.Combine(dotnetRunnerPath.FullName, Path.GetFileName(vstestConsolePath))}\"", out stdOut, out stdError, out exitCode, environmentVariables, workSpace.Path); + ExecuteApplication(dotnetPath, $"test -p:VsTestConsolePath=\"{Path.Combine(dotnetRunnerPath.FullName, Path.GetFileName(vstestConsolePath))}\"", out string stdOut, out _, out int exitCode, environmentVariables, TempDirectory.Path); Assert.AreEqual(0, exitCode, stdOut); } From b62a26de2064a874437f452aecf35f35ce92e5c4 Mon Sep 17 00:00:00 2001 From: nohwnd Date: Fri, 4 Mar 2022 14:57:54 +0100 Subject: [PATCH 030/112] Upgrade --- ...rm.Extensions.EventLogCollector.UnitTests.csproj | 13 ++++++------- .../AppDomainTests.cs | 4 ++-- .../DotnetArchitectureSwitchTests.Windows.cs | 2 +- .../DotnetArchitectureSwitchTests.cs | 2 +- .../Microsoft.TestPlatform.AcceptanceTests.csproj | 6 +++--- .../PostProcessingTests.cs | 2 ++ ...t.TestPlatform.AdapterUtilities.UnitTests.csproj | 2 +- .../Microsoft.TestPlatform.Build.UnitTests.csproj | 2 +- .../Microsoft.TestPlatform.Client.UnitTests.csproj | 4 ++-- ...crosoft.TestPlatform.Common.PlatformTests.csproj | 4 ++-- .../Microsoft.TestPlatform.Common.UnitTests.csproj | 4 ++-- ...form.CommunicationUtilities.PlatformTests.csproj | 4 ++-- ...Platform.CommunicationUtilities.UnitTests.csproj | 4 ++-- ...soft.TestPlatform.CoreUtilities.UnitTests.csproj | 6 +++--- ...ft.TestPlatform.CrossPlatEngine.UnitTests.csproj | 4 ++-- ...m.Extensions.BlameDataCollector.UnitTests.csproj | 2 +- ...tPlatform.Extensions.HtmlLogger.UnitTests.csproj | 4 ++-- ...stPlatform.Extensions.TrxLogger.UnitTests.csproj | 4 ++-- ...ft.TestPlatform.ObjectModel.PlatformTests.csproj | 5 +++-- .../FrameworkTests.cs | 2 +- ...rosoft.TestPlatform.ObjectModel.UnitTests.csproj | 4 ++-- .../Microsoft.TestPlatform.PerformanceTests.csproj | 6 +++--- .../Microsoft.TestPlatform.SmokeTests.csproj | 6 +++--- ...t.TestPlatform.TestHostProvider.UnitTests.csproj | 2 +- .../AssemblyUtility.cs | 4 ++-- .../Microsoft.TestPlatform.TestUtilities.csproj | 4 ++-- ...icrosoft.TestPlatform.Utilities.UnitTests.csproj | 4 ++-- .../SettingsMigrator.UnitTests.csproj | 3 ++- .../TranslationLayer.UnitTests.csproj | 6 +++--- test/coverlet.collector/coverlet.collector.csproj | 2 +- .../datacollector.PlatformTests.csproj | 2 +- .../datacollector.UnitTests.csproj | 2 +- test/testhost.UnitTests/testhost.UnitTests.csproj | 6 +++--- .../vstest.ProgrammerTests.csproj | 2 +- .../vstest.console.PlatformTests.csproj | 4 ++-- .../vstest.console.UnitTests.csproj | 4 ++-- 36 files changed, 72 insertions(+), 69 deletions(-) diff --git a/test/DataCollectors/Microsoft.TestPlatform.Extensions.EventLogCollector.UnitTests/Microsoft.TestPlatform.Extensions.EventLogCollector.UnitTests.csproj b/test/DataCollectors/Microsoft.TestPlatform.Extensions.EventLogCollector.UnitTests/Microsoft.TestPlatform.Extensions.EventLogCollector.UnitTests.csproj index 6015fe0778..51460fe09c 100644 --- a/test/DataCollectors/Microsoft.TestPlatform.Extensions.EventLogCollector.UnitTests/Microsoft.TestPlatform.Extensions.EventLogCollector.UnitTests.csproj +++ b/test/DataCollectors/Microsoft.TestPlatform.Extensions.EventLogCollector.UnitTests/Microsoft.TestPlatform.Extensions.EventLogCollector.UnitTests.csproj @@ -10,19 +10,18 @@ Microsoft.TestPlatform.Extensions.EventLogCollector.UnitTests - net451 - - - net451 + + net48 Microsoft.TestPlatform.Extensions.EventLogCollector.UnitTests - - + - + + + diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/AppDomainTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/AppDomainTests.cs index 442b24b0d4..5e79b4b26f 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/AppDomainTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/AppDomainTests.cs @@ -4,7 +4,7 @@ using System; using System.Collections.Generic; using System.IO; -#if !NET451 +#if !NETFRAMEWORK using System.Runtime.Loader; #else @@ -78,7 +78,7 @@ private string GetInProcDataCollectionRunsettingsFile(bool disableAppDomain, Tem { var runSettings = Path.Combine(tempDirectory.Path, "test_" + Guid.NewGuid() + ".runsettings"); var inprocasm = _testEnvironment.GetTestAsset("SimpleDataCollector.dll"); -#if !NET451 +#if !NETFRAMEWORK var assemblyName = AssemblyLoadContext.GetAssemblyName(inprocasm); #else var assemblyName = AssemblyName.GetAssemblyName(inprocasm); diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/DotnetArchitectureSwitchTests.Windows.cs b/test/Microsoft.TestPlatform.AcceptanceTests/DotnetArchitectureSwitchTests.Windows.cs index 2e7a2a0807..17fb42a8e8 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/DotnetArchitectureSwitchTests.Windows.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/DotnetArchitectureSwitchTests.Windows.cs @@ -1,7 +1,7 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT license. See LICENSE file in the project root for full license information. -#if !NET451 +#if !NETFRAMEWORK using System.Collections.Generic; using System.IO; diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/DotnetArchitectureSwitchTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/DotnetArchitectureSwitchTests.cs index e2d11506cf..e60f46ed94 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/DotnetArchitectureSwitchTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/DotnetArchitectureSwitchTests.cs @@ -1,7 +1,7 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT license. See LICENSE file in the project root for full license information. -#if !NET451 +#if !NETFRAMEWORK using System; using System.Collections.Generic; diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/Microsoft.TestPlatform.AcceptanceTests.csproj b/test/Microsoft.TestPlatform.AcceptanceTests/Microsoft.TestPlatform.AcceptanceTests.csproj index 31d535775e..be86e43d04 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/Microsoft.TestPlatform.AcceptanceTests.csproj +++ b/test/Microsoft.TestPlatform.AcceptanceTests/Microsoft.TestPlatform.AcceptanceTests.csproj @@ -8,7 +8,7 @@ Exe - netcoreapp2.1;net451 + net6.0;net48 netcoreapp3.1 Microsoft.TestPlatform.AcceptanceTests @@ -36,11 +36,11 @@ - + - + diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/PostProcessingTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/PostProcessingTests.cs index 2204859563..12f873b1f6 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/PostProcessingTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/PostProcessingTests.cs @@ -14,6 +14,8 @@ using Microsoft.VisualStudio.TestPlatform.Utilities.Helpers; using Microsoft.VisualStudio.TestTools.UnitTesting; +# nullable disable + namespace Microsoft.TestPlatform.AcceptanceTests; [TestClass] diff --git a/test/Microsoft.TestPlatform.AdapterUtilities.UnitTests/Microsoft.TestPlatform.AdapterUtilities.UnitTests.csproj b/test/Microsoft.TestPlatform.AdapterUtilities.UnitTests/Microsoft.TestPlatform.AdapterUtilities.UnitTests.csproj index a6973865bd..1f257d556d 100644 --- a/test/Microsoft.TestPlatform.AdapterUtilities.UnitTests/Microsoft.TestPlatform.AdapterUtilities.UnitTests.csproj +++ b/test/Microsoft.TestPlatform.AdapterUtilities.UnitTests/Microsoft.TestPlatform.AdapterUtilities.UnitTests.csproj @@ -6,7 +6,7 @@ - netcoreapp2.1 + net6.0;net48 netcoreapp3.1 Exe Microsoft.TestPlatform.AdapterUtilities.UnitTests diff --git a/test/Microsoft.TestPlatform.Build.UnitTests/Microsoft.TestPlatform.Build.UnitTests.csproj b/test/Microsoft.TestPlatform.Build.UnitTests/Microsoft.TestPlatform.Build.UnitTests.csproj index 3524adc1a0..5987088a8b 100644 --- a/test/Microsoft.TestPlatform.Build.UnitTests/Microsoft.TestPlatform.Build.UnitTests.csproj +++ b/test/Microsoft.TestPlatform.Build.UnitTests/Microsoft.TestPlatform.Build.UnitTests.csproj @@ -7,7 +7,7 @@ - netcoreapp2.1 + net6.0;net48 netcoreapp3.1 Exe Microsoft.TestPlatform.Build.UnitTests diff --git a/test/Microsoft.TestPlatform.Client.UnitTests/Microsoft.TestPlatform.Client.UnitTests.csproj b/test/Microsoft.TestPlatform.Client.UnitTests/Microsoft.TestPlatform.Client.UnitTests.csproj index cd47aeb6bf..516aafb5e1 100644 --- a/test/Microsoft.TestPlatform.Client.UnitTests/Microsoft.TestPlatform.Client.UnitTests.csproj +++ b/test/Microsoft.TestPlatform.Client.UnitTests/Microsoft.TestPlatform.Client.UnitTests.csproj @@ -7,7 +7,7 @@ - netcoreapp2.1;net451 + net6.0;net48 netcoreapp3.1 Exe Microsoft.TestPlatform.Client.UnitTests @@ -15,7 +15,7 @@ - + diff --git a/test/Microsoft.TestPlatform.Common.PlatformTests/Microsoft.TestPlatform.Common.PlatformTests.csproj b/test/Microsoft.TestPlatform.Common.PlatformTests/Microsoft.TestPlatform.Common.PlatformTests.csproj index 9e8b4eec12..0603c1a3c9 100644 --- a/test/Microsoft.TestPlatform.Common.PlatformTests/Microsoft.TestPlatform.Common.PlatformTests.csproj +++ b/test/Microsoft.TestPlatform.Common.PlatformTests/Microsoft.TestPlatform.Common.PlatformTests.csproj @@ -7,7 +7,7 @@ - netcoreapp2.1;net451 + net6.0;net48 netcoreapp3.1 Exe Microsoft.TestPlatform.Common.PlatformTests @@ -18,7 +18,7 @@ 2.0.0 - + diff --git a/test/Microsoft.TestPlatform.Common.UnitTests/Microsoft.TestPlatform.Common.UnitTests.csproj b/test/Microsoft.TestPlatform.Common.UnitTests/Microsoft.TestPlatform.Common.UnitTests.csproj index f72b78c876..299201bc61 100644 --- a/test/Microsoft.TestPlatform.Common.UnitTests/Microsoft.TestPlatform.Common.UnitTests.csproj +++ b/test/Microsoft.TestPlatform.Common.UnitTests/Microsoft.TestPlatform.Common.UnitTests.csproj @@ -7,12 +7,12 @@ - netcoreapp2.1;net451 + net6.0;net48 netcoreapp3.1 Exe Microsoft.TestPlatform.Common.UnitTests - + diff --git a/test/Microsoft.TestPlatform.CommunicationUtilities.PlatformTests/Microsoft.TestPlatform.CommunicationUtilities.PlatformTests.csproj b/test/Microsoft.TestPlatform.CommunicationUtilities.PlatformTests/Microsoft.TestPlatform.CommunicationUtilities.PlatformTests.csproj index 7c5eae08da..a390f3afd0 100644 --- a/test/Microsoft.TestPlatform.CommunicationUtilities.PlatformTests/Microsoft.TestPlatform.CommunicationUtilities.PlatformTests.csproj +++ b/test/Microsoft.TestPlatform.CommunicationUtilities.PlatformTests/Microsoft.TestPlatform.CommunicationUtilities.PlatformTests.csproj @@ -7,12 +7,12 @@ - netcoreapp2.1;net451 + net6.0;net48 netcoreapp3.1 Exe Microsoft.TestPlatform.CommunicationUtilities.PlatformTests - + diff --git a/test/Microsoft.TestPlatform.CommunicationUtilities.UnitTests/Microsoft.TestPlatform.CommunicationUtilities.UnitTests.csproj b/test/Microsoft.TestPlatform.CommunicationUtilities.UnitTests/Microsoft.TestPlatform.CommunicationUtilities.UnitTests.csproj index 33121ffb44..a08483e0fa 100644 --- a/test/Microsoft.TestPlatform.CommunicationUtilities.UnitTests/Microsoft.TestPlatform.CommunicationUtilities.UnitTests.csproj +++ b/test/Microsoft.TestPlatform.CommunicationUtilities.UnitTests/Microsoft.TestPlatform.CommunicationUtilities.UnitTests.csproj @@ -7,12 +7,12 @@ - netcoreapp2.1;net451 + net6.0;net48 netcoreapp3.1 Exe Microsoft.TestPlatform.CommunicationUtilities.UnitTests - + diff --git a/test/Microsoft.TestPlatform.CoreUtilities.UnitTests/Microsoft.TestPlatform.CoreUtilities.UnitTests.csproj b/test/Microsoft.TestPlatform.CoreUtilities.UnitTests/Microsoft.TestPlatform.CoreUtilities.UnitTests.csproj index afe4bf2809..a3193683f2 100644 --- a/test/Microsoft.TestPlatform.CoreUtilities.UnitTests/Microsoft.TestPlatform.CoreUtilities.UnitTests.csproj +++ b/test/Microsoft.TestPlatform.CoreUtilities.UnitTests/Microsoft.TestPlatform.CoreUtilities.UnitTests.csproj @@ -7,12 +7,12 @@ - netcoreapp2.1;net451 + net6.0;net48 netcoreapp3.1 Exe Microsoft.TestPlatform.CoreUtilities.UnitTests - + 4.3.0 @@ -26,7 +26,7 @@ 4.3.0 - + diff --git a/test/Microsoft.TestPlatform.CrossPlatEngine.UnitTests/Microsoft.TestPlatform.CrossPlatEngine.UnitTests.csproj b/test/Microsoft.TestPlatform.CrossPlatEngine.UnitTests/Microsoft.TestPlatform.CrossPlatEngine.UnitTests.csproj index f9e9226d18..85e5b5fd38 100644 --- a/test/Microsoft.TestPlatform.CrossPlatEngine.UnitTests/Microsoft.TestPlatform.CrossPlatEngine.UnitTests.csproj +++ b/test/Microsoft.TestPlatform.CrossPlatEngine.UnitTests/Microsoft.TestPlatform.CrossPlatEngine.UnitTests.csproj @@ -8,7 +8,7 @@ Microsoft.TestPlatform.CrossPlatEngine.UnitTests - netcoreapp2.1;net451 + net6.0;net48 netcoreapp3.1 Exe @@ -20,7 +20,7 @@ - + diff --git a/test/Microsoft.TestPlatform.Extensions.BlameDataCollector.UnitTests/Microsoft.TestPlatform.Extensions.BlameDataCollector.UnitTests.csproj b/test/Microsoft.TestPlatform.Extensions.BlameDataCollector.UnitTests/Microsoft.TestPlatform.Extensions.BlameDataCollector.UnitTests.csproj index 0024aa19b4..6f1d2c843a 100644 --- a/test/Microsoft.TestPlatform.Extensions.BlameDataCollector.UnitTests/Microsoft.TestPlatform.Extensions.BlameDataCollector.UnitTests.csproj +++ b/test/Microsoft.TestPlatform.Extensions.BlameDataCollector.UnitTests/Microsoft.TestPlatform.Extensions.BlameDataCollector.UnitTests.csproj @@ -19,7 +19,7 @@ Microsoft.TestPlatform.Extensions.BlameDataCollector.UnitTests - netcoreapp2.1;net472 + net6.0;net48 netcoreapp3.1 Exe Microsoft.TestPlatform.Extensions.BlameDataCollector.UnitTests diff --git a/test/Microsoft.TestPlatform.Extensions.HtmlLogger.UnitTests/Microsoft.TestPlatform.Extensions.HtmlLogger.UnitTests.csproj b/test/Microsoft.TestPlatform.Extensions.HtmlLogger.UnitTests/Microsoft.TestPlatform.Extensions.HtmlLogger.UnitTests.csproj index 25d350e476..409b518469 100644 --- a/test/Microsoft.TestPlatform.Extensions.HtmlLogger.UnitTests/Microsoft.TestPlatform.Extensions.HtmlLogger.UnitTests.csproj +++ b/test/Microsoft.TestPlatform.Extensions.HtmlLogger.UnitTests/Microsoft.TestPlatform.Extensions.HtmlLogger.UnitTests.csproj @@ -7,14 +7,14 @@ - netcoreapp2.1;net451 + net6.0;net48 netcoreapp3.1 Exe - + diff --git a/test/Microsoft.TestPlatform.Extensions.TrxLogger.UnitTests/Microsoft.TestPlatform.Extensions.TrxLogger.UnitTests.csproj b/test/Microsoft.TestPlatform.Extensions.TrxLogger.UnitTests/Microsoft.TestPlatform.Extensions.TrxLogger.UnitTests.csproj index 9b53f2f32d..7a764e60db 100644 --- a/test/Microsoft.TestPlatform.Extensions.TrxLogger.UnitTests/Microsoft.TestPlatform.Extensions.TrxLogger.UnitTests.csproj +++ b/test/Microsoft.TestPlatform.Extensions.TrxLogger.UnitTests/Microsoft.TestPlatform.Extensions.TrxLogger.UnitTests.csproj @@ -7,7 +7,7 @@ - netcoreapp2.1;net451 + net6.0;net48 netcoreapp3.1 Exe Microsoft.TestPlatform.Extensions.TrxLogger.UnitTests @@ -15,7 +15,7 @@ - + diff --git a/test/Microsoft.TestPlatform.ObjectModel.PlatformTests/Microsoft.TestPlatform.ObjectModel.PlatformTests.csproj b/test/Microsoft.TestPlatform.ObjectModel.PlatformTests/Microsoft.TestPlatform.ObjectModel.PlatformTests.csproj index 3e84049474..c271a4d4b3 100644 --- a/test/Microsoft.TestPlatform.ObjectModel.PlatformTests/Microsoft.TestPlatform.ObjectModel.PlatformTests.csproj +++ b/test/Microsoft.TestPlatform.ObjectModel.PlatformTests/Microsoft.TestPlatform.ObjectModel.PlatformTests.csproj @@ -6,7 +6,7 @@ - netcoreapp2.1;net451 + net6.0;net48 netcoreapp3.1 Exe Microsoft.TestPlatform.ObjectModel.PlatformTests @@ -14,7 +14,7 @@ - + @@ -22,6 +22,7 @@ + diff --git a/test/Microsoft.TestPlatform.ObjectModel.UnitTests/FrameworkTests.cs b/test/Microsoft.TestPlatform.ObjectModel.UnitTests/FrameworkTests.cs index 0c792d18fb..84a71688ea 100644 --- a/test/Microsoft.TestPlatform.ObjectModel.UnitTests/FrameworkTests.cs +++ b/test/Microsoft.TestPlatform.ObjectModel.UnitTests/FrameworkTests.cs @@ -81,7 +81,7 @@ public void DefaultFrameworkShouldBeNet40OnDesktop() [TestMethod] public void DefaultFrameworkShouldBeNetCoreApp10OnNonDesktop() { -#if !NET451 +#if !NETFRAMEWORK Assert.AreEqual(".NETCoreApp,Version=v1.0", Framework.DefaultFramework.Name); #endif } diff --git a/test/Microsoft.TestPlatform.ObjectModel.UnitTests/Microsoft.TestPlatform.ObjectModel.UnitTests.csproj b/test/Microsoft.TestPlatform.ObjectModel.UnitTests/Microsoft.TestPlatform.ObjectModel.UnitTests.csproj index eca84c2c95..decbf5ef22 100644 --- a/test/Microsoft.TestPlatform.ObjectModel.UnitTests/Microsoft.TestPlatform.ObjectModel.UnitTests.csproj +++ b/test/Microsoft.TestPlatform.ObjectModel.UnitTests/Microsoft.TestPlatform.ObjectModel.UnitTests.csproj @@ -7,7 +7,7 @@ - netcoreapp2.1;net451 + net6.0;net48 netcoreapp3.1 Exe Microsoft.TestPlatform.ObjectModel.UnitTests @@ -17,7 +17,7 @@ $(JsonNetVersion) - + diff --git a/test/Microsoft.TestPlatform.PerformanceTests/Microsoft.TestPlatform.PerformanceTests.csproj b/test/Microsoft.TestPlatform.PerformanceTests/Microsoft.TestPlatform.PerformanceTests.csproj index f5c20febb5..02254dbfb6 100644 --- a/test/Microsoft.TestPlatform.PerformanceTests/Microsoft.TestPlatform.PerformanceTests.csproj +++ b/test/Microsoft.TestPlatform.PerformanceTests/Microsoft.TestPlatform.PerformanceTests.csproj @@ -7,8 +7,8 @@ - Exe - netcoreapp2.1;net451 + Exe + net6.0;net48 Microsoft.TestPlatform.PerformanceTests @@ -19,7 +19,7 @@ - + diff --git a/test/Microsoft.TestPlatform.SmokeTests/Microsoft.TestPlatform.SmokeTests.csproj b/test/Microsoft.TestPlatform.SmokeTests/Microsoft.TestPlatform.SmokeTests.csproj index 9948d46e82..336ccaee62 100644 --- a/test/Microsoft.TestPlatform.SmokeTests/Microsoft.TestPlatform.SmokeTests.csproj +++ b/test/Microsoft.TestPlatform.SmokeTests/Microsoft.TestPlatform.SmokeTests.csproj @@ -8,19 +8,19 @@ Microsoft.TestPlatform.SmokeTests - netcoreapp2.1;net451 + net6.0;net48 netcoreapp3.1 Exe - + - + 4.3.0 diff --git a/test/Microsoft.TestPlatform.TestHostProvider.UnitTests/Microsoft.TestPlatform.TestHostProvider.UnitTests.csproj b/test/Microsoft.TestPlatform.TestHostProvider.UnitTests/Microsoft.TestPlatform.TestHostProvider.UnitTests.csproj index e098a6461c..7fa225328d 100644 --- a/test/Microsoft.TestPlatform.TestHostProvider.UnitTests/Microsoft.TestPlatform.TestHostProvider.UnitTests.csproj +++ b/test/Microsoft.TestPlatform.TestHostProvider.UnitTests/Microsoft.TestPlatform.TestHostProvider.UnitTests.csproj @@ -8,7 +8,7 @@ Microsoft.TestPlatform.TestHostProvider.UnitTests - netcoreapp2.1;net472 + net6.0;net48 netcoreapp3.1 Exe diff --git a/test/Microsoft.TestPlatform.TestUtilities/AssemblyUtility.cs b/test/Microsoft.TestPlatform.TestUtilities/AssemblyUtility.cs index e0702a44f7..a3c67178a7 100644 --- a/test/Microsoft.TestPlatform.TestUtilities/AssemblyUtility.cs +++ b/test/Microsoft.TestPlatform.TestUtilities/AssemblyUtility.cs @@ -2,7 +2,7 @@ // Licensed under the MIT license. See LICENSE file in the project root for full license information. using System.Reflection; -#if !NET451 +#if !NETFRAMEWORK using System.Runtime.Loader; #endif @@ -22,7 +22,7 @@ public class AssemblyUtility /// public static AssemblyName GetAssemblyName(string assemblyPath) { -#if !NET451 +#if !NETFRAMEWORK return AssemblyLoadContext.GetAssemblyName(assemblyPath); #else return AssemblyName.GetAssemblyName(assemblyPath); diff --git a/test/Microsoft.TestPlatform.TestUtilities/Microsoft.TestPlatform.TestUtilities.csproj b/test/Microsoft.TestPlatform.TestUtilities/Microsoft.TestPlatform.TestUtilities.csproj index a91d6a82ca..ccb1f81b3f 100644 --- a/test/Microsoft.TestPlatform.TestUtilities/Microsoft.TestPlatform.TestUtilities.csproj +++ b/test/Microsoft.TestPlatform.TestUtilities/Microsoft.TestPlatform.TestUtilities.csproj @@ -6,7 +6,7 @@ Microsoft.TestPlatform.TestUtilities - netcoreapp2.1;net451 + net6.0;net48 netcoreapp3.1 false @@ -22,7 +22,7 @@ - + diff --git a/test/Microsoft.TestPlatform.Utilities.UnitTests/Microsoft.TestPlatform.Utilities.UnitTests.csproj b/test/Microsoft.TestPlatform.Utilities.UnitTests/Microsoft.TestPlatform.Utilities.UnitTests.csproj index b1ada1e9c5..aa15f291aa 100644 --- a/test/Microsoft.TestPlatform.Utilities.UnitTests/Microsoft.TestPlatform.Utilities.UnitTests.csproj +++ b/test/Microsoft.TestPlatform.Utilities.UnitTests/Microsoft.TestPlatform.Utilities.UnitTests.csproj @@ -8,7 +8,7 @@ - netcoreapp2.1;net451 + net6.0;net48 netcoreapp3.1 Exe Microsoft.TestPlatform.Utilities.UnitTests @@ -22,7 +22,7 @@ - + diff --git a/test/SettingsMigrator.UnitTests/SettingsMigrator.UnitTests.csproj b/test/SettingsMigrator.UnitTests/SettingsMigrator.UnitTests.csproj index 5450d2cc23..c40955205f 100644 --- a/test/SettingsMigrator.UnitTests/SettingsMigrator.UnitTests.csproj +++ b/test/SettingsMigrator.UnitTests/SettingsMigrator.UnitTests.csproj @@ -7,7 +7,8 @@ - net451 + + net48 SettingsMigrator.UnitTests diff --git a/test/TranslationLayer.UnitTests/TranslationLayer.UnitTests.csproj b/test/TranslationLayer.UnitTests/TranslationLayer.UnitTests.csproj index 1b33ca81a5..4efa60842d 100644 --- a/test/TranslationLayer.UnitTests/TranslationLayer.UnitTests.csproj +++ b/test/TranslationLayer.UnitTests/TranslationLayer.UnitTests.csproj @@ -7,15 +7,15 @@ - netcoreapp2.1;net451 + net6.0;net48 netcoreapp3.1 - Exe + Exe TranslationLayer.UnitTests - + diff --git a/test/coverlet.collector/coverlet.collector.csproj b/test/coverlet.collector/coverlet.collector.csproj index 7966d51861..700ccdceb0 100644 --- a/test/coverlet.collector/coverlet.collector.csproj +++ b/test/coverlet.collector/coverlet.collector.csproj @@ -1,6 +1,6 @@ - netcoreapp2.1;net451 + net6.0;net48 netcoreapp3.1 false false diff --git a/test/datacollector.PlatformTests/datacollector.PlatformTests.csproj b/test/datacollector.PlatformTests/datacollector.PlatformTests.csproj index 0b8bfdcc3b..ade87ab0ee 100644 --- a/test/datacollector.PlatformTests/datacollector.PlatformTests.csproj +++ b/test/datacollector.PlatformTests/datacollector.PlatformTests.csproj @@ -9,7 +9,7 @@ Microsoft.VisualStudio.TestPlatform.DataCollector.PlatformTests - netcoreapp2.1;net472 + net6.0;net48 netcoreapp3.1 Exe datacollector.PlatformTests diff --git a/test/datacollector.UnitTests/datacollector.UnitTests.csproj b/test/datacollector.UnitTests/datacollector.UnitTests.csproj index 5e89f02902..2ce91cd9c1 100644 --- a/test/datacollector.UnitTests/datacollector.UnitTests.csproj +++ b/test/datacollector.UnitTests/datacollector.UnitTests.csproj @@ -11,7 +11,7 @@ Exe - netcoreapp2.1;net472 + net6.0;net48 netcoreapp3.1 datacollector.UnitTests diff --git a/test/testhost.UnitTests/testhost.UnitTests.csproj b/test/testhost.UnitTests/testhost.UnitTests.csproj index b434796078..4487c864f9 100644 --- a/test/testhost.UnitTests/testhost.UnitTests.csproj +++ b/test/testhost.UnitTests/testhost.UnitTests.csproj @@ -7,13 +7,13 @@ - netcoreapp2.1;net451 + net6.0;net48 netcoreapp3.1 Exe testhost.UnitTests - x64 + x64 - + diff --git a/test/vstest.ProgrammerTests/vstest.ProgrammerTests.csproj b/test/vstest.ProgrammerTests/vstest.ProgrammerTests.csproj index 21a59d0ab8..4b5bd1a84c 100644 --- a/test/vstest.ProgrammerTests/vstest.ProgrammerTests.csproj +++ b/test/vstest.ProgrammerTests/vstest.ProgrammerTests.csproj @@ -29,7 +29,7 @@ - + diff --git a/test/vstest.console.PlatformTests/vstest.console.PlatformTests.csproj b/test/vstest.console.PlatformTests/vstest.console.PlatformTests.csproj index f1c029346a..88fec59e33 100644 --- a/test/vstest.console.PlatformTests/vstest.console.PlatformTests.csproj +++ b/test/vstest.console.PlatformTests/vstest.console.PlatformTests.csproj @@ -7,7 +7,7 @@ - netcoreapp2.1;net451 + net6.0;net48 netcoreapp3.1 Exe vstest.console.PlatformTests @@ -16,7 +16,7 @@ - + diff --git a/test/vstest.console.UnitTests/vstest.console.UnitTests.csproj b/test/vstest.console.UnitTests/vstest.console.UnitTests.csproj index d41ebf1187..137205af5d 100644 --- a/test/vstest.console.UnitTests/vstest.console.UnitTests.csproj +++ b/test/vstest.console.UnitTests/vstest.console.UnitTests.csproj @@ -7,7 +7,7 @@ - netcoreapp2.1;net451 + net6.0;net48 netcoreapp3.1 Exe vstest.console.UnitTests @@ -22,7 +22,7 @@ true - + From f15ade9c47e1d7be54c95f7e4d1b916404833221 Mon Sep 17 00:00:00 2001 From: nohwnd Date: Fri, 4 Mar 2022 16:21:49 +0100 Subject: [PATCH 031/112] Upgrade tests --- scripts/test.ps1 | 20 ++++++++++--------- ...ensions.EventLogCollector.UnitTests.csproj | 1 - ...rosoft.TestPlatform.AcceptanceTests.csproj | 2 +- .../ProcessesInteractionTests.cs | 2 +- ...Platform.AdapterUtilities.UnitTests.csproj | 2 +- ...rosoft.TestPlatform.Build.UnitTests.csproj | 2 +- ...osoft.TestPlatform.Client.UnitTests.csproj | 2 +- ...t.TestPlatform.Common.PlatformTests.csproj | 2 +- ...osoft.TestPlatform.Common.UnitTests.csproj | 2 +- ...ommunicationUtilities.PlatformTests.csproj | 2 +- ...rm.CommunicationUtilities.UnitTests.csproj | 2 +- ...estPlatform.CoreUtilities.UnitTests.csproj | 2 +- ...tPlatform.CrossPlatEngine.UnitTests.csproj | 2 +- ...nsions.BlameDataCollector.UnitTests.csproj | 2 +- .../XmlReaderWriterTests.cs | 9 ++++++--- ...orm.Extensions.HtmlLogger.UnitTests.csproj | 2 +- ...form.Extensions.TrxLogger.UnitTests.csproj | 2 +- ...tPlatform.ObjectModel.PlatformTests.csproj | 2 +- ....TestPlatform.ObjectModel.UnitTests.csproj | 2 +- .../Microsoft.TestPlatform.SmokeTests.csproj | 2 +- .../Hosting/DotnetTestHostManagerTests.cs | 9 +++++++++ ...Platform.TestHostProvider.UnitTests.csproj | 4 ++-- ...ft.TestPlatform.Utilities.UnitTests.csproj | 2 +- .../datacollector.PlatformTests.csproj | 2 +- .../datacollector.UnitTests.csproj | 2 +- .../testhost.UnitTests.csproj | 2 +- .../vstest.ProgrammerTests.csproj | 2 +- .../vstest.console.PlatformTests.csproj | 2 +- .../vstest.console.UnitTests.csproj | 2 +- 29 files changed, 52 insertions(+), 39 deletions(-) diff --git a/scripts/test.ps1 b/scripts/test.ps1 index d0022db77b..c6e30a6113 100644 --- a/scripts/test.ps1 +++ b/scripts/test.ps1 @@ -14,7 +14,7 @@ Param( [System.String] $TargetRuntime = "win7-x64", [Parameter(Mandatory=$false)] - [ValidateSet("net451", "netcoreapp2.1")] + [ValidateSet("net48", "net6.0")] [Alias("f")] [System.String] $TargetFramework, @@ -94,12 +94,14 @@ $env:NUGET_PACKAGES = $env:TP_PACKAGES_DIR # # Test configuration # -$TPT_TargetFrameworkFullCLR = "net451" -$TPT_TargetFrameworkCore20 = "netcoreapp2.1" +$TPT_TargetFrameworkNet451 = "net451" +$TPT_TargetFrameworkNet48 = "net48" +$TPT_TargetFrameworkCore21 = "netcoreapp2.1" +$TPT_TargetFrameworkNet60 = "net6.0" Write-Verbose "Setup build configuration." $Script:TPT_Configuration = $Configuration $Script:TPT_SourceFolders = @("test") -$Script:TPT_TargetFrameworks =@($TPT_TargetFrameworkFullCLR, $TPT_TargetFrameworkCore20) +$Script:TPT_TargetFrameworks =@($TPT_TargetFrameworkNet48, $TPT_TargetFrameworkNet60) $Script:TPT_TargetFramework = $TargetFramework $Script:TPT_TargetRuntime = $TargetRuntime $Script:TPT_SkipProjects = @("_none_"); @@ -225,17 +227,17 @@ function Invoke-Test $testFilter = "/testCaseFilter:`"$TPT_TestFilter`"" } - if($fx -eq $TPT_TargetFrameworkCore20) + if($fx -eq $TPT_TargetFrameworkNet60) { $vstestConsoleFileName = "vstest.console.dll" $targetRunTime = "" - $vstestConsolePath = Join-Path (Get-PackageDirectory $TPT_TargetFrameworkCore20 $targetRuntime) $vstestConsoleFileName + $vstestConsolePath = Join-Path (Get-PackageDirectory $TPT_TargetFrameworkCore21 $targetRuntime) $vstestConsoleFileName } else { $vstestConsoleFileName = "vstest.console.exe" $targetRunTime = $Script:TPT_TargetRuntime - $vstestConsolePath = Join-Path (Get-PackageDirectory $TPT_TargetFrameworkFullCLR $targetRuntime) $vstestConsoleFileName + $vstestConsolePath = Join-Path (Get-PackageDirectory $TPT_TargetFrameworkNet451 $targetRuntime) $vstestConsoleFileName } if (!(Test-Path $vstestConsolePath)) @@ -264,7 +266,7 @@ function Invoke-Test } Set-TestEnvironment - if($fx -eq $TPT_TargetFrameworkFullCLR) + if($fx -eq $TPT_TargetFrameworkNet48) { Write-Verbose "$vstestConsolePath $testContainerSet /parallel /logger:`"trx;LogFileName=$trxLogFileName`" $testFilter $ConsoleLogger" @@ -305,7 +307,7 @@ function Invoke-Test Set-TestEnvironment - if($fx -eq $TPT_TargetFrameworkFullCLR) + if($fx -eq $TPT_TargetFrameworkNet48) { Write-Verbose "$vstestConsolePath $testContainer /logger:`"trx;LogFileName=$trxLogFileName`" $ConsoleLogger $testFilter" diff --git a/test/DataCollectors/Microsoft.TestPlatform.Extensions.EventLogCollector.UnitTests/Microsoft.TestPlatform.Extensions.EventLogCollector.UnitTests.csproj b/test/DataCollectors/Microsoft.TestPlatform.Extensions.EventLogCollector.UnitTests/Microsoft.TestPlatform.Extensions.EventLogCollector.UnitTests.csproj index 51460fe09c..51e4d723fe 100644 --- a/test/DataCollectors/Microsoft.TestPlatform.Extensions.EventLogCollector.UnitTests/Microsoft.TestPlatform.Extensions.EventLogCollector.UnitTests.csproj +++ b/test/DataCollectors/Microsoft.TestPlatform.Extensions.EventLogCollector.UnitTests/Microsoft.TestPlatform.Extensions.EventLogCollector.UnitTests.csproj @@ -20,7 +20,6 @@ - diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/Microsoft.TestPlatform.AcceptanceTests.csproj b/test/Microsoft.TestPlatform.AcceptanceTests/Microsoft.TestPlatform.AcceptanceTests.csproj index be86e43d04..9a87dad715 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/Microsoft.TestPlatform.AcceptanceTests.csproj +++ b/test/Microsoft.TestPlatform.AcceptanceTests/Microsoft.TestPlatform.AcceptanceTests.csproj @@ -7,7 +7,7 @@ - Exe + Exe net6.0;net48 netcoreapp3.1 Microsoft.TestPlatform.AcceptanceTests diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/ProcessesInteractionTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/ProcessesInteractionTests.cs index b4fe07e22c..e82ddc9520 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/ProcessesInteractionTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/ProcessesInteractionTests.cs @@ -37,7 +37,7 @@ static void UpdateRuntimeConfigJsonWithInvalidFramework(string assemblyPath, str // On the contrary to other tests, we need to modify the test asset we are using to replace // the target framework with an invalid framework. This is why we have a specific test asset // that's only meant to be used by this project. - var runtimeConfigJson = Path.Combine(Path.GetDirectoryName(assemblyPath), testAssetProjectName + ".runtimeconfig.json"); + var runtimeConfigJson = Path.Combine(Path.GetDirectoryName(assemblyPath)!, testAssetProjectName + ".runtimeconfig.json"); var fileContent = File.ReadAllText(runtimeConfigJson); var updatedContent = fileContent.Replace("\"version\": \"2.1.0\"", "\"version\": \"0.0.0\""); File.WriteAllText(runtimeConfigJson, updatedContent); diff --git a/test/Microsoft.TestPlatform.AdapterUtilities.UnitTests/Microsoft.TestPlatform.AdapterUtilities.UnitTests.csproj b/test/Microsoft.TestPlatform.AdapterUtilities.UnitTests/Microsoft.TestPlatform.AdapterUtilities.UnitTests.csproj index 1f257d556d..d69a372fc5 100644 --- a/test/Microsoft.TestPlatform.AdapterUtilities.UnitTests/Microsoft.TestPlatform.AdapterUtilities.UnitTests.csproj +++ b/test/Microsoft.TestPlatform.AdapterUtilities.UnitTests/Microsoft.TestPlatform.AdapterUtilities.UnitTests.csproj @@ -8,7 +8,7 @@ net6.0;net48 netcoreapp3.1 - Exe + Exe Microsoft.TestPlatform.AdapterUtilities.UnitTests true $(NoWarn);RS1024 diff --git a/test/Microsoft.TestPlatform.Build.UnitTests/Microsoft.TestPlatform.Build.UnitTests.csproj b/test/Microsoft.TestPlatform.Build.UnitTests/Microsoft.TestPlatform.Build.UnitTests.csproj index 5987088a8b..022d772011 100644 --- a/test/Microsoft.TestPlatform.Build.UnitTests/Microsoft.TestPlatform.Build.UnitTests.csproj +++ b/test/Microsoft.TestPlatform.Build.UnitTests/Microsoft.TestPlatform.Build.UnitTests.csproj @@ -9,7 +9,7 @@ net6.0;net48 netcoreapp3.1 - Exe + Exe Microsoft.TestPlatform.Build.UnitTests diff --git a/test/Microsoft.TestPlatform.Client.UnitTests/Microsoft.TestPlatform.Client.UnitTests.csproj b/test/Microsoft.TestPlatform.Client.UnitTests/Microsoft.TestPlatform.Client.UnitTests.csproj index 516aafb5e1..3638bd290d 100644 --- a/test/Microsoft.TestPlatform.Client.UnitTests/Microsoft.TestPlatform.Client.UnitTests.csproj +++ b/test/Microsoft.TestPlatform.Client.UnitTests/Microsoft.TestPlatform.Client.UnitTests.csproj @@ -9,7 +9,7 @@ net6.0;net48 netcoreapp3.1 - Exe + Exe Microsoft.TestPlatform.Client.UnitTests diff --git a/test/Microsoft.TestPlatform.Common.PlatformTests/Microsoft.TestPlatform.Common.PlatformTests.csproj b/test/Microsoft.TestPlatform.Common.PlatformTests/Microsoft.TestPlatform.Common.PlatformTests.csproj index 0603c1a3c9..d0cdd8d84f 100644 --- a/test/Microsoft.TestPlatform.Common.PlatformTests/Microsoft.TestPlatform.Common.PlatformTests.csproj +++ b/test/Microsoft.TestPlatform.Common.PlatformTests/Microsoft.TestPlatform.Common.PlatformTests.csproj @@ -9,7 +9,7 @@ net6.0;net48 netcoreapp3.1 - Exe + Exe Microsoft.TestPlatform.Common.PlatformTests diff --git a/test/Microsoft.TestPlatform.Common.UnitTests/Microsoft.TestPlatform.Common.UnitTests.csproj b/test/Microsoft.TestPlatform.Common.UnitTests/Microsoft.TestPlatform.Common.UnitTests.csproj index 299201bc61..26cf91d771 100644 --- a/test/Microsoft.TestPlatform.Common.UnitTests/Microsoft.TestPlatform.Common.UnitTests.csproj +++ b/test/Microsoft.TestPlatform.Common.UnitTests/Microsoft.TestPlatform.Common.UnitTests.csproj @@ -9,7 +9,7 @@ net6.0;net48 netcoreapp3.1 - Exe + Exe Microsoft.TestPlatform.Common.UnitTests diff --git a/test/Microsoft.TestPlatform.CommunicationUtilities.PlatformTests/Microsoft.TestPlatform.CommunicationUtilities.PlatformTests.csproj b/test/Microsoft.TestPlatform.CommunicationUtilities.PlatformTests/Microsoft.TestPlatform.CommunicationUtilities.PlatformTests.csproj index a390f3afd0..b0098fff88 100644 --- a/test/Microsoft.TestPlatform.CommunicationUtilities.PlatformTests/Microsoft.TestPlatform.CommunicationUtilities.PlatformTests.csproj +++ b/test/Microsoft.TestPlatform.CommunicationUtilities.PlatformTests/Microsoft.TestPlatform.CommunicationUtilities.PlatformTests.csproj @@ -9,7 +9,7 @@ net6.0;net48 netcoreapp3.1 - Exe + Exe Microsoft.TestPlatform.CommunicationUtilities.PlatformTests diff --git a/test/Microsoft.TestPlatform.CommunicationUtilities.UnitTests/Microsoft.TestPlatform.CommunicationUtilities.UnitTests.csproj b/test/Microsoft.TestPlatform.CommunicationUtilities.UnitTests/Microsoft.TestPlatform.CommunicationUtilities.UnitTests.csproj index a08483e0fa..7e35f37f32 100644 --- a/test/Microsoft.TestPlatform.CommunicationUtilities.UnitTests/Microsoft.TestPlatform.CommunicationUtilities.UnitTests.csproj +++ b/test/Microsoft.TestPlatform.CommunicationUtilities.UnitTests/Microsoft.TestPlatform.CommunicationUtilities.UnitTests.csproj @@ -9,7 +9,7 @@ net6.0;net48 netcoreapp3.1 - Exe + Exe Microsoft.TestPlatform.CommunicationUtilities.UnitTests diff --git a/test/Microsoft.TestPlatform.CoreUtilities.UnitTests/Microsoft.TestPlatform.CoreUtilities.UnitTests.csproj b/test/Microsoft.TestPlatform.CoreUtilities.UnitTests/Microsoft.TestPlatform.CoreUtilities.UnitTests.csproj index a3193683f2..12d5ab3331 100644 --- a/test/Microsoft.TestPlatform.CoreUtilities.UnitTests/Microsoft.TestPlatform.CoreUtilities.UnitTests.csproj +++ b/test/Microsoft.TestPlatform.CoreUtilities.UnitTests/Microsoft.TestPlatform.CoreUtilities.UnitTests.csproj @@ -9,7 +9,7 @@ net6.0;net48 netcoreapp3.1 - Exe + Exe Microsoft.TestPlatform.CoreUtilities.UnitTests diff --git a/test/Microsoft.TestPlatform.CrossPlatEngine.UnitTests/Microsoft.TestPlatform.CrossPlatEngine.UnitTests.csproj b/test/Microsoft.TestPlatform.CrossPlatEngine.UnitTests/Microsoft.TestPlatform.CrossPlatEngine.UnitTests.csproj index 85e5b5fd38..4379157066 100644 --- a/test/Microsoft.TestPlatform.CrossPlatEngine.UnitTests/Microsoft.TestPlatform.CrossPlatEngine.UnitTests.csproj +++ b/test/Microsoft.TestPlatform.CrossPlatEngine.UnitTests/Microsoft.TestPlatform.CrossPlatEngine.UnitTests.csproj @@ -10,7 +10,7 @@ Microsoft.TestPlatform.CrossPlatEngine.UnitTests net6.0;net48 netcoreapp3.1 - Exe + Exe diff --git a/test/Microsoft.TestPlatform.Extensions.BlameDataCollector.UnitTests/Microsoft.TestPlatform.Extensions.BlameDataCollector.UnitTests.csproj b/test/Microsoft.TestPlatform.Extensions.BlameDataCollector.UnitTests/Microsoft.TestPlatform.Extensions.BlameDataCollector.UnitTests.csproj index 6f1d2c843a..8331cbedd0 100644 --- a/test/Microsoft.TestPlatform.Extensions.BlameDataCollector.UnitTests/Microsoft.TestPlatform.Extensions.BlameDataCollector.UnitTests.csproj +++ b/test/Microsoft.TestPlatform.Extensions.BlameDataCollector.UnitTests/Microsoft.TestPlatform.Extensions.BlameDataCollector.UnitTests.csproj @@ -21,7 +21,7 @@ net6.0;net48 netcoreapp3.1 - Exe + Exe Microsoft.TestPlatform.Extensions.BlameDataCollector.UnitTests diff --git a/test/Microsoft.TestPlatform.Extensions.BlameDataCollector.UnitTests/XmlReaderWriterTests.cs b/test/Microsoft.TestPlatform.Extensions.BlameDataCollector.UnitTests/XmlReaderWriterTests.cs index 7bc177cfe1..4dd782d5e0 100644 --- a/test/Microsoft.TestPlatform.Extensions.BlameDataCollector.UnitTests/XmlReaderWriterTests.cs +++ b/test/Microsoft.TestPlatform.Extensions.BlameDataCollector.UnitTests/XmlReaderWriterTests.cs @@ -5,6 +5,7 @@ using System.Collections.Generic; using System.IO; using System.Linq; +using System.Text; using Microsoft.VisualStudio.TestPlatform.ObjectModel; using Microsoft.VisualStudio.TestPlatform.Utilities.Helpers.Interfaces; @@ -120,7 +121,8 @@ public void WriteTestSequenceShouldWriteFileStream() { // Setup _mockFileHelper.Setup(m => m.Exists(It.IsAny())).Returns(true); - _mockFileHelper.Setup(m => m.GetStream("path.xml", FileMode.Create, FileAccess.ReadWrite)).Returns(_mockStream.Object); + using var stream = new MemoryStream(); + _mockFileHelper.Setup(m => m.GetStream("path.xml", FileMode.Create, FileAccess.ReadWrite)).Returns(stream); _mockStream.Setup(x => x.CanWrite).Returns(true); _mockStream.Setup(x => x.Write(It.IsAny(), It.IsAny(), It.IsAny())); @@ -129,8 +131,9 @@ public void WriteTestSequenceShouldWriteFileStream() // Verify Call to fileHelper _mockFileHelper.Verify(x => x.GetStream("path.xml", FileMode.Create, FileAccess.ReadWrite)); - // Verify Call to stream write - _mockStream.Verify(x => x.Write(It.IsAny(), It.IsAny(), It.IsAny())); + // Assert it has some data + var data = Encoding.UTF8.GetString(stream.ToArray()); + Assert.IsTrue(data.Length > 0, "Stream should have some data."); } /// diff --git a/test/Microsoft.TestPlatform.Extensions.HtmlLogger.UnitTests/Microsoft.TestPlatform.Extensions.HtmlLogger.UnitTests.csproj b/test/Microsoft.TestPlatform.Extensions.HtmlLogger.UnitTests/Microsoft.TestPlatform.Extensions.HtmlLogger.UnitTests.csproj index 409b518469..5ab7c3446c 100644 --- a/test/Microsoft.TestPlatform.Extensions.HtmlLogger.UnitTests/Microsoft.TestPlatform.Extensions.HtmlLogger.UnitTests.csproj +++ b/test/Microsoft.TestPlatform.Extensions.HtmlLogger.UnitTests/Microsoft.TestPlatform.Extensions.HtmlLogger.UnitTests.csproj @@ -9,7 +9,7 @@ net6.0;net48 netcoreapp3.1 - Exe + Exe diff --git a/test/Microsoft.TestPlatform.Extensions.TrxLogger.UnitTests/Microsoft.TestPlatform.Extensions.TrxLogger.UnitTests.csproj b/test/Microsoft.TestPlatform.Extensions.TrxLogger.UnitTests/Microsoft.TestPlatform.Extensions.TrxLogger.UnitTests.csproj index 7a764e60db..349996bbd3 100644 --- a/test/Microsoft.TestPlatform.Extensions.TrxLogger.UnitTests/Microsoft.TestPlatform.Extensions.TrxLogger.UnitTests.csproj +++ b/test/Microsoft.TestPlatform.Extensions.TrxLogger.UnitTests/Microsoft.TestPlatform.Extensions.TrxLogger.UnitTests.csproj @@ -9,7 +9,7 @@ net6.0;net48 netcoreapp3.1 - Exe + Exe Microsoft.TestPlatform.Extensions.TrxLogger.UnitTests diff --git a/test/Microsoft.TestPlatform.ObjectModel.PlatformTests/Microsoft.TestPlatform.ObjectModel.PlatformTests.csproj b/test/Microsoft.TestPlatform.ObjectModel.PlatformTests/Microsoft.TestPlatform.ObjectModel.PlatformTests.csproj index c271a4d4b3..45a2a1778a 100644 --- a/test/Microsoft.TestPlatform.ObjectModel.PlatformTests/Microsoft.TestPlatform.ObjectModel.PlatformTests.csproj +++ b/test/Microsoft.TestPlatform.ObjectModel.PlatformTests/Microsoft.TestPlatform.ObjectModel.PlatformTests.csproj @@ -8,7 +8,7 @@ net6.0;net48 netcoreapp3.1 - Exe + Exe Microsoft.TestPlatform.ObjectModel.PlatformTests diff --git a/test/Microsoft.TestPlatform.ObjectModel.UnitTests/Microsoft.TestPlatform.ObjectModel.UnitTests.csproj b/test/Microsoft.TestPlatform.ObjectModel.UnitTests/Microsoft.TestPlatform.ObjectModel.UnitTests.csproj index decbf5ef22..5b34507354 100644 --- a/test/Microsoft.TestPlatform.ObjectModel.UnitTests/Microsoft.TestPlatform.ObjectModel.UnitTests.csproj +++ b/test/Microsoft.TestPlatform.ObjectModel.UnitTests/Microsoft.TestPlatform.ObjectModel.UnitTests.csproj @@ -9,7 +9,7 @@ net6.0;net48 netcoreapp3.1 - Exe + Exe Microsoft.TestPlatform.ObjectModel.UnitTests diff --git a/test/Microsoft.TestPlatform.SmokeTests/Microsoft.TestPlatform.SmokeTests.csproj b/test/Microsoft.TestPlatform.SmokeTests/Microsoft.TestPlatform.SmokeTests.csproj index 336ccaee62..f5a1ab7451 100644 --- a/test/Microsoft.TestPlatform.SmokeTests/Microsoft.TestPlatform.SmokeTests.csproj +++ b/test/Microsoft.TestPlatform.SmokeTests/Microsoft.TestPlatform.SmokeTests.csproj @@ -10,7 +10,7 @@ Microsoft.TestPlatform.SmokeTests net6.0;net48 netcoreapp3.1 - Exe + Exe diff --git a/test/Microsoft.TestPlatform.TestHostProvider.UnitTests/Hosting/DotnetTestHostManagerTests.cs b/test/Microsoft.TestPlatform.TestHostProvider.UnitTests/Hosting/DotnetTestHostManagerTests.cs index 1b130db836..7d4ce97a00 100644 --- a/test/Microsoft.TestPlatform.TestHostProvider.UnitTests/Hosting/DotnetTestHostManagerTests.cs +++ b/test/Microsoft.TestPlatform.TestHostProvider.UnitTests/Hosting/DotnetTestHostManagerTests.cs @@ -616,6 +616,8 @@ public void GetTestHostProcessStartInfoShouldIncludeTestHostPathFromSourceDirect StringAssert.Contains(startInfo.Arguments, expectedTestHostPath); } + // TODO: Not sure why but this worked before on .NET451, but now I can't get it to work anywhere. Assembly.GetEntryAssembly().Location is null because of running in app domain. +#if NET [TestMethod] public void GetTestHostProcessStartInfoShouldIncludeTestHostPathNextToTestRunnerIfTesthostDllIsNoFoundAndDepsFileNotFound() { @@ -639,6 +641,11 @@ public void GetTestHostProcessStartInfoShouldIncludeTestHostPathNextToTestRunner StringAssert.Contains(startInfo.Arguments, $"--runtimeconfig \"{expectedRuntimeConfigPath}\""); } +#endif + + // TODO: Not sure why but this worked before on .NET451, but now I can't get it to work anywhere. Assembly.GetEntryAssembly().Location is null because of running in app domain. +#if NET + [TestMethod] // we can't put in a "default" value, and we don't have other way to determine if this provided value is the @@ -671,6 +678,8 @@ public void GetTestHostProcessStartInfoShouldIncludeTestHostPathNextToTestRunner StringAssert.Contains(startInfo.Arguments, $"--runtimeconfig \"{expectedRuntimeConfigPath}\""); } +#endif + [TestMethod] public void GetTestHostProcessStartInfoShouldIncludeTestHostPathFromSourceDirectoryIfRunConfigDevFileNotFound() { diff --git a/test/Microsoft.TestPlatform.TestHostProvider.UnitTests/Microsoft.TestPlatform.TestHostProvider.UnitTests.csproj b/test/Microsoft.TestPlatform.TestHostProvider.UnitTests/Microsoft.TestPlatform.TestHostProvider.UnitTests.csproj index 7fa225328d..40aeb778f2 100644 --- a/test/Microsoft.TestPlatform.TestHostProvider.UnitTests/Microsoft.TestPlatform.TestHostProvider.UnitTests.csproj +++ b/test/Microsoft.TestPlatform.TestHostProvider.UnitTests/Microsoft.TestPlatform.TestHostProvider.UnitTests.csproj @@ -8,9 +8,9 @@ Microsoft.TestPlatform.TestHostProvider.UnitTests - net6.0;net48 + net6.0;net48;net451 netcoreapp3.1 - Exe + Exe diff --git a/test/Microsoft.TestPlatform.Utilities.UnitTests/Microsoft.TestPlatform.Utilities.UnitTests.csproj b/test/Microsoft.TestPlatform.Utilities.UnitTests/Microsoft.TestPlatform.Utilities.UnitTests.csproj index aa15f291aa..ac2e969a99 100644 --- a/test/Microsoft.TestPlatform.Utilities.UnitTests/Microsoft.TestPlatform.Utilities.UnitTests.csproj +++ b/test/Microsoft.TestPlatform.Utilities.UnitTests/Microsoft.TestPlatform.Utilities.UnitTests.csproj @@ -10,7 +10,7 @@ net6.0;net48 netcoreapp3.1 - Exe + Exe Microsoft.TestPlatform.Utilities.UnitTests diff --git a/test/datacollector.PlatformTests/datacollector.PlatformTests.csproj b/test/datacollector.PlatformTests/datacollector.PlatformTests.csproj index ade87ab0ee..935e9ba419 100644 --- a/test/datacollector.PlatformTests/datacollector.PlatformTests.csproj +++ b/test/datacollector.PlatformTests/datacollector.PlatformTests.csproj @@ -11,7 +11,7 @@ net6.0;net48 netcoreapp3.1 - Exe + Exe datacollector.PlatformTests diff --git a/test/datacollector.UnitTests/datacollector.UnitTests.csproj b/test/datacollector.UnitTests/datacollector.UnitTests.csproj index 2ce91cd9c1..9810f40bff 100644 --- a/test/datacollector.UnitTests/datacollector.UnitTests.csproj +++ b/test/datacollector.UnitTests/datacollector.UnitTests.csproj @@ -10,7 +10,7 @@ Microsoft.VisualStudio.TestPlatform.DataCollector.UnitTests - Exe + Exe net6.0;net48 netcoreapp3.1 datacollector.UnitTests diff --git a/test/testhost.UnitTests/testhost.UnitTests.csproj b/test/testhost.UnitTests/testhost.UnitTests.csproj index 4487c864f9..843ee2e344 100644 --- a/test/testhost.UnitTests/testhost.UnitTests.csproj +++ b/test/testhost.UnitTests/testhost.UnitTests.csproj @@ -9,7 +9,7 @@ net6.0;net48 netcoreapp3.1 - Exe + Exe testhost.UnitTests x64 diff --git a/test/vstest.ProgrammerTests/vstest.ProgrammerTests.csproj b/test/vstest.ProgrammerTests/vstest.ProgrammerTests.csproj index 4b5bd1a84c..793d988e7d 100644 --- a/test/vstest.ProgrammerTests/vstest.ProgrammerTests.csproj +++ b/test/vstest.ProgrammerTests/vstest.ProgrammerTests.csproj @@ -12,7 +12,7 @@ preview net6.0 netcoreapp3.1 - Exe + Exe Exe diff --git a/test/vstest.console.PlatformTests/vstest.console.PlatformTests.csproj b/test/vstest.console.PlatformTests/vstest.console.PlatformTests.csproj index 88fec59e33..9e48311a14 100644 --- a/test/vstest.console.PlatformTests/vstest.console.PlatformTests.csproj +++ b/test/vstest.console.PlatformTests/vstest.console.PlatformTests.csproj @@ -9,7 +9,7 @@ net6.0;net48 netcoreapp3.1 - Exe + Exe vstest.console.PlatformTests diff --git a/test/vstest.console.UnitTests/vstest.console.UnitTests.csproj b/test/vstest.console.UnitTests/vstest.console.UnitTests.csproj index 137205af5d..d0b644826b 100644 --- a/test/vstest.console.UnitTests/vstest.console.UnitTests.csproj +++ b/test/vstest.console.UnitTests/vstest.console.UnitTests.csproj @@ -9,7 +9,7 @@ net6.0;net48 netcoreapp3.1 - Exe + Exe vstest.console.UnitTests From 100b4ddd4835024ff45c4bdf537d676db5077e11 Mon Sep 17 00:00:00 2001 From: nohwnd Date: Fri, 4 Mar 2022 16:43:59 +0100 Subject: [PATCH 032/112] Skip web tests locally --- .../DifferentTestFrameworkSimpleTests.cs | 11 +++++++++++ .../IntegrationTestBase.cs | 3 +++ .../IntegrationTestEnvironment.cs | 3 +++ 3 files changed, 17 insertions(+) diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/DifferentTestFrameworkSimpleTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/DifferentTestFrameworkSimpleTests.cs index 26dfa7ab26..846fd07816 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/DifferentTestFrameworkSimpleTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/DifferentTestFrameworkSimpleTests.cs @@ -15,6 +15,7 @@ namespace Microsoft.TestPlatform.AcceptanceTests; [TestCategory("Windows-Review")] public class DifferentTestFrameworkSimpleTests : AcceptanceTestBase { + [TestMethod] [NetFullTargetFrameworkDataSource(inIsolation: true, inProcess: true)] public void ChutzpahRunAllTestExecution(RunnerInfo runnerInfo) @@ -65,6 +66,11 @@ public void CPPRunAllTestExecutionPlatformx64Net(RunnerInfo runnerInfo) [NetFullTargetFrameworkDataSource] public void WebTestRunAllTestsWithRunSettings(RunnerInfo runnerInfo) { + if (IsCI) + { + Assert.Inconclusive("This works on server but not locally, because locall it grabs dll from my GAC and that is old, but has 10.0.0 version as the one in our package."); + } + SetTestEnvironment(_testEnvironment, runnerInfo); var runSettingsFilePath = Path.Combine(TempDirectory.Path, Guid.NewGuid() + ".runsettings"); @@ -117,6 +123,11 @@ public void WebTestRunAllTestsWithRunSettings(RunnerInfo runnerInfo) [NetFullTargetFrameworkDataSource] public void CodedWebTestRunAllTests(RunnerInfo runnerInfo) { + if (IsCI) + { + Assert.Inconclusive("This works on server but not locally, because locall it grabs dll from my GAC and that is old, but has 10.0.0 version as the one in our package."); + } + SetTestEnvironment(_testEnvironment, runnerInfo); if (runnerInfo.IsNetRunner) { diff --git a/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestBase.cs b/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestBase.cs index 9f6b567252..abbbbf0967 100644 --- a/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestBase.cs +++ b/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestBase.cs @@ -59,6 +59,7 @@ public IntegrationTestBase() _testEnvironment = new IntegrationTestEnvironment(); BuildConfiguration = IntegrationTestEnvironment.BuildConfiguration; TempDirectory = new TempDirectory(); + IsCI = IntegrationTestEnvironment.IsCI; } public string StdOut => _standardTestOutput; @@ -73,6 +74,8 @@ public IntegrationTestBase() public string BuildConfiguration { get; } + public bool IsCI { get; } + [TestCleanup] public void TempDirectoryCleanup() { diff --git a/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestEnvironment.cs b/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestEnvironment.cs index 8fe9e840c2..fdded084bf 100644 --- a/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestEnvironment.cs +++ b/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestEnvironment.cs @@ -195,6 +195,9 @@ public string RunnerFramework set; } + // A known AzureDevOps env variable meaning we are running in CI. + public static bool IsCI { get; } = Environment.GetEnvironmentVariable("TF_BUILD") == "True"; + /// /// Gets the full path to a test asset. /// From cdaf3d053f9ecff66b24dc07570c6316557c4760 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Jare=C5=A1?= Date: Fri, 4 Mar 2022 16:51:50 +0100 Subject: [PATCH 033/112] Apply suggestions from code review --- .../Hosting/DotnetTestHostManagerTests.cs | 6 ++++-- ...Microsoft.TestPlatform.TestHostProvider.UnitTests.csproj | 2 +- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/test/Microsoft.TestPlatform.TestHostProvider.UnitTests/Hosting/DotnetTestHostManagerTests.cs b/test/Microsoft.TestPlatform.TestHostProvider.UnitTests/Hosting/DotnetTestHostManagerTests.cs index 7d4ce97a00..8e917e1046 100644 --- a/test/Microsoft.TestPlatform.TestHostProvider.UnitTests/Hosting/DotnetTestHostManagerTests.cs +++ b/test/Microsoft.TestPlatform.TestHostProvider.UnitTests/Hosting/DotnetTestHostManagerTests.cs @@ -616,7 +616,8 @@ public void GetTestHostProcessStartInfoShouldIncludeTestHostPathFromSourceDirect StringAssert.Contains(startInfo.Arguments, expectedTestHostPath); } - // TODO: Not sure why but this worked before on .NET451, but now I can't get it to work anywhere. Assembly.GetEntryAssembly().Location is null because of running in app domain. + // TODO: This assembly was previously compiled as net472 and so it was skipped and only ran as netcoreapp2.1. This fails in test, but works in code that is not isolated in appdomain. Might be worth fixing because we get one null here, and another in DotnetTestHostManager. + // Assembly.GetEntryAssembly().Location is null because of running in app domain. #if NET [TestMethod] public void GetTestHostProcessStartInfoShouldIncludeTestHostPathNextToTestRunnerIfTesthostDllIsNoFoundAndDepsFileNotFound() @@ -643,7 +644,8 @@ public void GetTestHostProcessStartInfoShouldIncludeTestHostPathNextToTestRunner #endif - // TODO: Not sure why but this worked before on .NET451, but now I can't get it to work anywhere. Assembly.GetEntryAssembly().Location is null because of running in app domain. + // TODO: This assembly was previously compiled as net472 and so it was skipped and only ran as netcoreapp2.1. This fails in test, but works in code that is not isolated in appdomain. Might be worth fixing because we get one null here, and another in DotnetTestHostManager. + // Assembly.GetEntryAssembly().Location is null because of running in app domain. #if NET [TestMethod] diff --git a/test/Microsoft.TestPlatform.TestHostProvider.UnitTests/Microsoft.TestPlatform.TestHostProvider.UnitTests.csproj b/test/Microsoft.TestPlatform.TestHostProvider.UnitTests/Microsoft.TestPlatform.TestHostProvider.UnitTests.csproj index 40aeb778f2..c2ee1ef0a6 100644 --- a/test/Microsoft.TestPlatform.TestHostProvider.UnitTests/Microsoft.TestPlatform.TestHostProvider.UnitTests.csproj +++ b/test/Microsoft.TestPlatform.TestHostProvider.UnitTests/Microsoft.TestPlatform.TestHostProvider.UnitTests.csproj @@ -8,7 +8,7 @@ Microsoft.TestPlatform.TestHostProvider.UnitTests - net6.0;net48;net451 + net6.0;net48 netcoreapp3.1 Exe From 390d80b52fc43f2afd9695d7aec985d754cfe178 Mon Sep 17 00:00:00 2001 From: nohwnd Date: Fri, 4 Mar 2022 16:53:44 +0100 Subject: [PATCH 034/112] Skip in non-ci --- .../DifferentTestFrameworkSimpleTests.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/DifferentTestFrameworkSimpleTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/DifferentTestFrameworkSimpleTests.cs index 846fd07816..221d806917 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/DifferentTestFrameworkSimpleTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/DifferentTestFrameworkSimpleTests.cs @@ -66,7 +66,7 @@ public void CPPRunAllTestExecutionPlatformx64Net(RunnerInfo runnerInfo) [NetFullTargetFrameworkDataSource] public void WebTestRunAllTestsWithRunSettings(RunnerInfo runnerInfo) { - if (IsCI) + if (!IsCI) { Assert.Inconclusive("This works on server but not locally, because locall it grabs dll from my GAC and that is old, but has 10.0.0 version as the one in our package."); } @@ -123,7 +123,7 @@ public void WebTestRunAllTestsWithRunSettings(RunnerInfo runnerInfo) [NetFullTargetFrameworkDataSource] public void CodedWebTestRunAllTests(RunnerInfo runnerInfo) { - if (IsCI) + if (!IsCI) { Assert.Inconclusive("This works on server but not locally, because locall it grabs dll from my GAC and that is old, but has 10.0.0 version as the one in our package."); } From 543c21c47406d74ed48eba70028f759279c43bf9 Mon Sep 17 00:00:00 2001 From: nohwnd Date: Fri, 4 Mar 2022 17:22:12 +0100 Subject: [PATCH 035/112] Change order of tests to run from fastest Use net48 for acceptance --- azure-pipelines.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 62576ab4c0..998b4ffca9 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -97,26 +97,26 @@ jobs: reg ADD "HKLM\Software\Wow6432Node\Microsoft\StrongName\Verification\*,*" /f - task: BatchScript@1 - displayName: 'Run Acceptance Tests' + displayName: 'Run Platform Tests' inputs: filename: test.cmd - arguments: '-verbose -configuration $(buildConfiguration) -p AcceptanceTests -f net451' + arguments: '-verbose -configuration $(buildConfiguration) -p platformtests' modifyEnvironment: false failOnStandardError: true - task: BatchScript@1 - displayName: 'Run Platform Tests' + displayName: 'Run Smoke Tests' inputs: filename: test.cmd - arguments: '-verbose -configuration $(buildConfiguration) -p platformtests' + arguments: '-verbose -configuration $(buildConfiguration) -p smoke' modifyEnvironment: false failOnStandardError: true - task: BatchScript@1 - displayName: 'Run Smoke Tests' + displayName: 'Run Acceptance Tests' inputs: filename: test.cmd - arguments: '-verbose -configuration $(buildConfiguration) -p smoke' + arguments: '-verbose -configuration $(buildConfiguration) -p AcceptanceTests -f net48' modifyEnvironment: false failOnStandardError: true From 76021d2c54ac48b91c30b260311e3ffbe0905d3b Mon Sep 17 00:00:00 2001 From: nohwnd Date: Mon, 7 Mar 2022 07:37:01 +0100 Subject: [PATCH 036/112] Newer approach to reference assemblies on MacOS and Linux --- playground/MSTest1/MSTest1.csproj | 1 - .../TestPlatform.Playground/TestPlatform.Playground.csproj | 1 - scripts/build.sh | 2 -- scripts/build/TestPlatform.Settings.targets | 5 +++++ ...crosoft.TestPlatform.Extensions.BlameDataCollector.csproj | 1 - src/package/external/external.csproj | 5 ----- src/testhost.arm64/testhost.arm64.csproj | 1 - src/testhost.x86/testhost.x86.csproj | 1 - src/testhost/testhost.csproj | 1 - ...stPlatform.Extensions.BlameDataCollector.UnitTests.csproj | 1 - .../Microsoft.TestPlatform.TestHostProvider.UnitTests.csproj | 1 - 11 files changed, 5 insertions(+), 15 deletions(-) diff --git a/playground/MSTest1/MSTest1.csproj b/playground/MSTest1/MSTest1.csproj index e952c0fa11..c1a72605a4 100644 --- a/playground/MSTest1/MSTest1.csproj +++ b/playground/MSTest1/MSTest1.csproj @@ -19,7 +19,6 @@ - diff --git a/playground/TestPlatform.Playground/TestPlatform.Playground.csproj b/playground/TestPlatform.Playground/TestPlatform.Playground.csproj index d73746f16e..77ccfd0b7e 100644 --- a/playground/TestPlatform.Playground/TestPlatform.Playground.csproj +++ b/playground/TestPlatform.Playground/TestPlatform.Playground.csproj @@ -30,7 +30,6 @@ - diff --git a/scripts/build.sh b/scripts/build.sh index d6e2e6bf73..9268940821 100755 --- a/scripts/build.sh +++ b/scripts/build.sh @@ -267,8 +267,6 @@ function invoke_build() local start=$SECONDS log ".. .. Build: Source: $TPB_Solution" - # Workaround for https://github.com/dotnet/sdk/issues/335 - export FrameworkPathOverride=$TP_PACKAGES_DIR/microsoft.targetingpack.netframework.v4.7.2/1.0.0/lib/net472/ if [ -z "$PROJECT_NAME_PATTERNS" ] then if [[ $TP_USE_REPO_API = 0 ]]; then diff --git a/scripts/build/TestPlatform.Settings.targets b/scripts/build/TestPlatform.Settings.targets index 36a9550e8a..e883bf8085 100644 --- a/scripts/build/TestPlatform.Settings.targets +++ b/scripts/build/TestPlatform.Settings.targets @@ -49,6 +49,11 @@ + + + + + - diff --git a/src/testhost.arm64/testhost.arm64.csproj b/src/testhost.arm64/testhost.arm64.csproj index 87affaafaf..d3e4c7da5c 100644 --- a/src/testhost.arm64/testhost.arm64.csproj +++ b/src/testhost.arm64/testhost.arm64.csproj @@ -50,7 +50,6 @@ - diff --git a/src/testhost.x86/testhost.x86.csproj b/src/testhost.x86/testhost.x86.csproj index 06e2ad30a7..a9e8ea85ba 100644 --- a/src/testhost.x86/testhost.x86.csproj +++ b/src/testhost.x86/testhost.x86.csproj @@ -49,7 +49,6 @@ - diff --git a/src/testhost/testhost.csproj b/src/testhost/testhost.csproj index cd8477bd59..ac2c7a02cf 100644 --- a/src/testhost/testhost.csproj +++ b/src/testhost/testhost.csproj @@ -50,7 +50,6 @@ - diff --git a/test/Microsoft.TestPlatform.Extensions.BlameDataCollector.UnitTests/Microsoft.TestPlatform.Extensions.BlameDataCollector.UnitTests.csproj b/test/Microsoft.TestPlatform.Extensions.BlameDataCollector.UnitTests/Microsoft.TestPlatform.Extensions.BlameDataCollector.UnitTests.csproj index 0024aa19b4..1edb0499f2 100644 --- a/test/Microsoft.TestPlatform.Extensions.BlameDataCollector.UnitTests/Microsoft.TestPlatform.Extensions.BlameDataCollector.UnitTests.csproj +++ b/test/Microsoft.TestPlatform.Extensions.BlameDataCollector.UnitTests/Microsoft.TestPlatform.Extensions.BlameDataCollector.UnitTests.csproj @@ -37,7 +37,6 @@ - diff --git a/test/Microsoft.TestPlatform.TestHostProvider.UnitTests/Microsoft.TestPlatform.TestHostProvider.UnitTests.csproj b/test/Microsoft.TestPlatform.TestHostProvider.UnitTests/Microsoft.TestPlatform.TestHostProvider.UnitTests.csproj index e098a6461c..a6d24cf837 100644 --- a/test/Microsoft.TestPlatform.TestHostProvider.UnitTests/Microsoft.TestPlatform.TestHostProvider.UnitTests.csproj +++ b/test/Microsoft.TestPlatform.TestHostProvider.UnitTests/Microsoft.TestPlatform.TestHostProvider.UnitTests.csproj @@ -20,7 +20,6 @@ - From de7aefccfb09b96ffc80600af97216c97d51291c Mon Sep 17 00:00:00 2001 From: nohwnd Date: Mon, 7 Mar 2022 09:22:54 +0100 Subject: [PATCH 037/112] Revert "Change order of tests to run from fastest" This reverts commit 543c21c47406d74ed48eba70028f759279c43bf9. --- azure-pipelines.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 998b4ffca9..62576ab4c0 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -97,26 +97,26 @@ jobs: reg ADD "HKLM\Software\Wow6432Node\Microsoft\StrongName\Verification\*,*" /f - task: BatchScript@1 - displayName: 'Run Platform Tests' + displayName: 'Run Acceptance Tests' inputs: filename: test.cmd - arguments: '-verbose -configuration $(buildConfiguration) -p platformtests' + arguments: '-verbose -configuration $(buildConfiguration) -p AcceptanceTests -f net451' modifyEnvironment: false failOnStandardError: true - task: BatchScript@1 - displayName: 'Run Smoke Tests' + displayName: 'Run Platform Tests' inputs: filename: test.cmd - arguments: '-verbose -configuration $(buildConfiguration) -p smoke' + arguments: '-verbose -configuration $(buildConfiguration) -p platformtests' modifyEnvironment: false failOnStandardError: true - task: BatchScript@1 - displayName: 'Run Acceptance Tests' + displayName: 'Run Smoke Tests' inputs: filename: test.cmd - arguments: '-verbose -configuration $(buildConfiguration) -p AcceptanceTests -f net48' + arguments: '-verbose -configuration $(buildConfiguration) -p smoke' modifyEnvironment: false failOnStandardError: true From 1bddbc87589ef060d891cdc392abb6ca0cd10e23 Mon Sep 17 00:00:00 2001 From: nohwnd Date: Mon, 7 Mar 2022 09:23:28 +0100 Subject: [PATCH 038/112] Acceptance for net48 --- azure-pipelines.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 62576ab4c0..8df66a8969 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -100,7 +100,7 @@ jobs: displayName: 'Run Acceptance Tests' inputs: filename: test.cmd - arguments: '-verbose -configuration $(buildConfiguration) -p AcceptanceTests -f net451' + arguments: '-verbose -configuration $(buildConfiguration) -p AcceptanceTests -f net48' modifyEnvironment: false failOnStandardError: true From c4e7ff1bd86f1e53208a1d11dc448da5b9f171fe Mon Sep 17 00:00:00 2001 From: nohwnd Date: Mon, 7 Mar 2022 09:26:15 +0100 Subject: [PATCH 039/112] Fix filter --- scripts/test.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/test.sh b/scripts/test.sh index a28a96f3ce..9d988cfd52 100644 --- a/scripts/test.sh +++ b/scripts/test.sh @@ -57,7 +57,7 @@ done # # Variables # -PROJECT_NAME_PATTERNS=**$PROJECT_NAME_PATTERNS*bin*$CONFIGURATION*netcoreapp2.1*$PROJECT_NAME_PATTERNS*Tests*dll +PROJECT_NAME_PATTERNS=**$PROJECT_NAME_PATTERNS*bin*$CONFIGURATION*net6.0*$PROJECT_NAME_PATTERNS*Tests*dll TP_ROOT_DIR=$(cd "$(dirname "$0")"; pwd -P) TP_TOOLS_DIR="$TP_ROOT_DIR/tools" TP_PACKAGES_DIR="$TP_ROOT_DIR/packages" From bbd4473c8f5f7fa179c24dbb437a716a29608f0f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Jare=C5=A1?= Date: Mon, 7 Mar 2022 09:36:02 +0100 Subject: [PATCH 040/112] Apply suggestions from code review MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Amaury Levé --- .../DifferentTestFrameworkSimpleTests.cs | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/DifferentTestFrameworkSimpleTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/DifferentTestFrameworkSimpleTests.cs index 221d806917..faaeb1fd7e 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/DifferentTestFrameworkSimpleTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/DifferentTestFrameworkSimpleTests.cs @@ -15,7 +15,6 @@ namespace Microsoft.TestPlatform.AcceptanceTests; [TestCategory("Windows-Review")] public class DifferentTestFrameworkSimpleTests : AcceptanceTestBase { - [TestMethod] [NetFullTargetFrameworkDataSource(inIsolation: true, inProcess: true)] public void ChutzpahRunAllTestExecution(RunnerInfo runnerInfo) @@ -68,7 +67,7 @@ public void WebTestRunAllTestsWithRunSettings(RunnerInfo runnerInfo) { if (!IsCI) { - Assert.Inconclusive("This works on server but not locally, because locall it grabs dll from my GAC and that is old, but has 10.0.0 version as the one in our package."); + Assert.Inconclusive("This works on server but not locally, because locally it grabs old dll from GAC, but has version 10.0.0 as the one in our package."); } SetTestEnvironment(_testEnvironment, runnerInfo); @@ -125,7 +124,7 @@ public void CodedWebTestRunAllTests(RunnerInfo runnerInfo) { if (!IsCI) { - Assert.Inconclusive("This works on server but not locally, because locall it grabs dll from my GAC and that is old, but has 10.0.0 version as the one in our package."); + Assert.Inconclusive("This works on server but not locally, because locally it grabs old dll from GAC, but has version 10.0.0 as the one in our package."); } SetTestEnvironment(_testEnvironment, runnerInfo); From 7c6d9462d86c76af032b0708c2f24270e3c7e079 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Jare=C5=A1?= Date: Mon, 7 Mar 2022 10:24:48 +0100 Subject: [PATCH 041/112] Update scripts/build/TestPlatform.Settings.targets MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Amaury Levé --- scripts/build/TestPlatform.Settings.targets | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/build/TestPlatform.Settings.targets b/scripts/build/TestPlatform.Settings.targets index e883bf8085..b297c84ab9 100644 --- a/scripts/build/TestPlatform.Settings.targets +++ b/scripts/build/TestPlatform.Settings.targets @@ -51,7 +51,7 @@ - + 17.2.0-dev - 2.1.0 - 2.1.0 + 2.2.8 + 2.2.8 1.0.3-preview + + [2.2.8] + [2.2.9-preview-20220210-07] + [2.1.0] + [2.1.0] + [1.4.0] + 2.3.1 - 2.3.1 - 2.3.1 + $(XUnitFrameworkVersion) + $(XUnitFrameworkVersion) 3.10.1 - 3.10.0 + $(NUnit3FrameworkVersion) 3.8.0 4.4.12 diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/AcceptanceTestBase.cs b/test/Microsoft.TestPlatform.AcceptanceTests/AcceptanceTestBase.cs index 4829c3edcb..55b41da95f 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/AcceptanceTestBase.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/AcceptanceTestBase.cs @@ -45,12 +45,18 @@ public class AcceptanceTestBase : IntegrationTestBase public const string DesktopRunnerTargetRuntime = "win7-x64"; public const string CoreRunnerTargetRuntime = ""; public const string InIsolation = "/InIsolation"; - + public const string NETFX452_48 = "net452;net461;net472;net48"; public const string NETFX451_48 = "net452;net461;net472;net48"; public const string NETCORE21_50 = "netcoreapp2.1;netcoreapp3.1;net5.0"; public const string NETFX452_NET50 = "net452;net461;net472;net48;netcoreapp2.1;netcoreapp3.1;net5.0"; public const string NETFX452_NET31 = "net452;net461;net472;net48;netcoreapp2.1;netcoreapp3.1"; + /// + /// Our current defaults for .NET and .NET Framework. + /// + public const string DEFAULT_NETFX_AND_NET = "net451;netcoreapp2.1"; + public const string LATEST_LEGACY = "Latest;LatestStable;LatestPreview;MostDownloaded;PreviousStable;LegacyStable"; + public const string LATESTSTABLE_LEGACY = "LatestStable;LatestPreview;MostDownloaded;PreviousStable;LegacyStable"; public static string And(string left, string right) { diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/ExecutionTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/ExecutionTests.cs index b88bf2e02d..8a2dbcfa97 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/ExecutionTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/ExecutionTests.cs @@ -31,6 +31,20 @@ public void RunMultipleTestAssemblies(RunnerInfo runnerInfo) ExitCodeEquals(1); // failing tests } + [TestMethod] + [MSTestCompatibilityDataSource] + public void RunMultipleTestAssemblies2(RunnerInfo runnerInfo, MSTestInfo msTestInfo) + { + SetTestEnvironment(_testEnvironment, runnerInfo); + + var assemblyPaths = BuildMultipleAssemblyPath(msTestInfo, "SimpleTestProject.dll").Trim('\"'); + + InvokeVsTestForExecution(assemblyPaths, testAdapterPath: null, FrameworkArgValue, string.Empty); + + ValidateSummaryStatus(2, 2, 2); + ExitCodeEquals(1); // failing tests + } + [TestMethod] [NetFullTargetFrameworkDataSource(inIsolation: true, inProcess: true)] [NetCoreTargetFrameworkDataSource] @@ -59,12 +73,13 @@ public void RunMultipleTestAssembliesWithoutTestAdapterPath(RunnerInfo runnerInf public void RunMultipleTestAssembliesInParallel(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); + using var tempDir = new TempDirectory(); var assemblyPaths = BuildMultipleAssemblyPath("SimpleTestProject.dll", "SimpleTestProject2.dll").Trim('\"'); - var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: TempDirectory.Path); + var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: tempDir.Path); arguments = string.Concat(arguments, " /Parallel"); arguments = string.Concat(arguments, " /Platform:x86"); - arguments += GetDiagArg(TempDirectory.Path); + arguments += GetDiagArg(tempDir.Path); // for the desktop we will run testhost.x86 in two copies, but for core // we will run a combination of testhost.x86 and dotnet, where the dotnet will be @@ -77,7 +92,7 @@ public void RunMultipleTestAssembliesInParallel(RunnerInfo runnerInfo) InvokeVsTest(arguments); - AssertExpectedNumberOfHostProcesses(expectedNumOfProcessCreated, TempDirectory.Path, testHostProcessNames); + AssertExpectedNumberOfHostProcesses(expectedNumOfProcessCreated, tempDir.Path, testHostProcessNames); ValidateSummaryStatus(2, 2, 2); ExitCodeEquals(1); // failing tests } @@ -88,10 +103,11 @@ public void RunMultipleTestAssembliesInParallel(RunnerInfo runnerInfo) public void TestSessionTimeOutTests(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); + using var tempDir = new TempDirectory(); var assemblyPaths = BuildMultipleAssemblyPath("SimpleTestProject3.dll").Trim('\"'); - var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: TempDirectory.Path); + var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: tempDir.Path); arguments = string.Concat(arguments, " /TestCaseFilter:TestSessionTimeoutTest"); // set TestSessionTimeOut = 7 sec @@ -109,9 +125,10 @@ public void TestSessionTimeOutTests(RunnerInfo runnerInfo) public void TestPlatformShouldBeCompatibleWithOldTestHost(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); + using var tempDir = new TempDirectory(); var assemblyPaths = BuildMultipleAssemblyPath("SampleProjectWithOldTestHost.dll").Trim('\"'); - var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: TempDirectory.Path); + var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: tempDir.Path); InvokeVsTest(arguments); @@ -125,9 +142,10 @@ public void TestPlatformShouldBeCompatibleWithOldTestHost(RunnerInfo runnerInfo) public void WorkingDirectoryIsSourceDirectory(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); + using var tempDir = new TempDirectory(); var assemblyPaths = BuildMultipleAssemblyPath("SimpleTestProject3.dll").Trim('\"'); - var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: TempDirectory.Path); + var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: tempDir.Path); arguments = string.Concat(arguments, " /tests:WorkingDirectoryTest"); InvokeVsTest(arguments); @@ -142,6 +160,7 @@ public void WorkingDirectoryIsSourceDirectory(RunnerInfo runnerInfo) public void StackOverflowExceptionShouldBeLoggedToConsoleAndDiagLogFile(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); + using var tempDir = new TempDirectory(); if (IntegrationTestEnvironment.BuildConfiguration.Equals("release", StringComparison.OrdinalIgnoreCase)) { @@ -150,11 +169,11 @@ public void StackOverflowExceptionShouldBeLoggedToConsoleAndDiagLogFile(RunnerIn return; } - var diagLogFilePath = Path.Combine(TempDirectory.Path, $"std_error_log_{Guid.NewGuid()}.txt"); + var diagLogFilePath = Path.Combine(tempDir.Path, $"std_error_log_{Guid.NewGuid()}.txt"); File.Delete(diagLogFilePath); var assemblyPaths = BuildMultipleAssemblyPath("SimpleTestProject3.dll").Trim('\"'); - var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: TempDirectory.Path); + var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: tempDir.Path); arguments = string.Concat(arguments, " /testcasefilter:ExitWithStackoverFlow"); arguments = string.Concat(arguments, $" /diag:{diagLogFilePath}"); @@ -178,13 +197,14 @@ public void StackOverflowExceptionShouldBeLoggedToConsoleAndDiagLogFile(RunnerIn public void UnhandleExceptionExceptionShouldBeLoggedToDiagLogFile(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); + using var tempDir = new TempDirectory(); - var diagLogFilePath = Path.Combine(TempDirectory.Path, $"std_error_log_{Guid.NewGuid()}.txt"); + var diagLogFilePath = Path.Combine(tempDir.Path, $"std_error_log_{Guid.NewGuid()}.txt"); File.Delete(diagLogFilePath); var assemblyPaths = BuildMultipleAssemblyPath("SimpleTestProject3.dll").Trim('\"'); - var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: TempDirectory.Path); + var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: tempDir.Path); arguments = string.Concat(arguments, " /testcasefilter:ExitwithUnhandleException"); arguments = string.Concat(arguments, $" /diag:{diagLogFilePath}"); @@ -201,11 +221,12 @@ public void UnhandleExceptionExceptionShouldBeLoggedToDiagLogFile(RunnerInfo run public void IncompatibleSourcesWarningShouldBeDisplayedInTheConsole(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); + using var tempDir = new TempDirectory(); var expectedWarningContains = @"Following DLL(s) do not match current settings, which are .NETFramework,Version=v4.5.1 framework and X86 platform. SimpleTestProject3.dll is built for Framework .NETFramework,Version=v4.5.1 and Platform X64"; var assemblyPaths = BuildMultipleAssemblyPath("SimpleTestProject3.dll", "SimpleTestProjectx86.dll").Trim('\"'); - var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: TempDirectory.Path); + var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: tempDir.Path); arguments = string.Concat(arguments, " /testcasefilter:PassingTestx86"); InvokeVsTest(arguments); @@ -223,11 +244,12 @@ public void IncompatibleSourcesWarningShouldBeDisplayedInTheConsole(RunnerInfo r public void NoIncompatibleSourcesWarningShouldBeDisplayedInTheConsole(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); + using var tempDir = new TempDirectory(); var expectedWarningContains = @"Following DLL(s) do not match current settings, which are .NETFramework,Version=v4.5.1 framework and X86 platform. SimpleTestProjectx86 is built for Framework .NETFramework,Version=v4.5.1 and Platform X86"; var assemblyPaths = BuildMultipleAssemblyPath("SimpleTestProjectx86.dll"); - var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: TempDirectory.Path); + var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: tempDir.Path); InvokeVsTest(arguments); @@ -243,11 +265,12 @@ public void NoIncompatibleSourcesWarningShouldBeDisplayedInTheConsole(RunnerInfo public void IncompatibleSourcesWarningShouldBeDisplayedInTheConsoleOnlyWhenRunningIn32BitOS(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); + using var tempDir = new TempDirectory(); var expectedWarningContains = @"Following DLL(s) do not match current settings, which are .NETFramework,Version=v4.5.1 framework and X86 platform. SimpleTestProject2.dll is built for Framework .NETFramework,Version=v4.5.1 and Platform X64"; var assemblyPaths = BuildMultipleAssemblyPath("SimpleTestProject2.dll"); - var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: TempDirectory.Path); + var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: tempDir.Path); InvokeVsTest(arguments); @@ -272,10 +295,11 @@ public void IncompatibleSourcesWarningShouldBeDisplayedInTheConsoleOnlyWhenRunni public void ExitCodeShouldReturnOneWhenTreatNoTestsAsErrorParameterSetToTrueAndNoTestMatchesFilter(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); + using var tempDir = new TempDirectory(); var assemblyPaths = BuildMultipleAssemblyPath("SimpleTestProject2.dll").Trim('\"'); - var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: TempDirectory.Path); + var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: tempDir.Path); // Setting /TestCaseFilter to the test name, which does not exists in the assembly, so we will have 0 tests executed arguments = string.Concat(arguments, " /TestCaseFilter:TestNameThatMatchesNoTestInTheAssembly"); @@ -292,9 +316,10 @@ public void ExitCodeShouldReturnOneWhenTreatNoTestsAsErrorParameterSetToTrueAndN public void ExitCodeShouldReturnZeroWhenTreatNoTestsAsErrorParameterSetToFalseAndNoTestMatchesFilter(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); + using var tempDir = new TempDirectory(); var assemblyPaths = BuildMultipleAssemblyPath("SimpleTestProject2.dll").Trim('\"'); - var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: TempDirectory.Path); + var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: tempDir.Path); // Setting /TestCaseFilter to the test name, which does not exists in the assembly, so we will have 0 tests executed arguments = string.Concat(arguments, " /TestCaseFilter:TestNameThatMatchesNoTestInTheAssembly"); @@ -311,10 +336,11 @@ public void ExitCodeShouldReturnZeroWhenTreatNoTestsAsErrorParameterSetToFalseAn public void ExitCodeShouldNotDependOnTreatNoTestsAsErrorTrueValueWhenThereAreAnyTestsToRun(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); + using var tempDir = new TempDirectory(); var assemblyPaths = BuildMultipleAssemblyPath("SimpleTestProject2.dll").Trim('\"'); - var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: TempDirectory.Path); + var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: tempDir.Path); arguments = string.Concat(arguments, " -- RunConfiguration.TreatNoTestsAsError=true"); InvokeVsTest(arguments); @@ -329,9 +355,10 @@ public void ExitCodeShouldNotDependOnTreatNoTestsAsErrorTrueValueWhenThereAreAny public void ExitCodeShouldNotDependOnFailTreatNoTestsAsErrorFalseValueWhenThereAreAnyTestsToRun(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); + using var tempDir = new TempDirectory(); var assemblyPaths = BuildMultipleAssemblyPath("SimpleTestProject2.dll").Trim('\"'); - var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: TempDirectory.Path); + var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: tempDir.Path); arguments = string.Concat(arguments, " -- RunConfiguration.TreatNoTestsAsError=false"); InvokeVsTest(arguments); diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/NetCoreTargetFrameworkDataSource.cs b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/NetCoreTargetFrameworkDataSource.cs index 4e8ae4ae9d..b9ee94a80e 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/NetCoreTargetFrameworkDataSource.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/NetCoreTargetFrameworkDataSource.cs @@ -4,7 +4,12 @@ using System; using System.Collections.Generic; using System.Globalization; +using System.IO; +using System.Linq; using System.Reflection; +using System.Text; +using System.Text.RegularExpressions; +using System.Xml; using Microsoft.TestPlatform.TestUtilities; @@ -84,3 +89,77 @@ public string GetDisplayName(MethodInfo methodInfo, object[] data) return string.Format(CultureInfo.CurrentCulture, "{0} ({1})", methodInfo.Name, string.Join(",", data)); } } + +public class MSTestCompatibilityDataSource : Attribute, ITestDataSource +{ + /// + /// Initializes a new instance of the class. + /// + /// To run tests with desktop runner(vstest.console.exe), use AcceptanceTestBase.Net452TargetFramework or alike values. + public MSTestCompatibilityDataSource(string runners = AcceptanceTestBase.DEFAULT_NETFX_AND_NET, string targetFrameworks = AcceptanceTestBase.DEFAULT_NETFX_AND_NET, string msTestVersions = AcceptanceTestBase.LATESTSTABLE_LEGACY) + { + var runnersFrameworks = runners.Split(';'); + var testhostFrameworks = targetFrameworks.Split(';'); + var msTestVersionsToRun = msTestVersions.Split(';'); + + var isWindows = Environment.OSVersion.Platform.ToString().StartsWith("Win"); + + // Only run .NET Framework tests on Windows. + Func filter = tfm => isWindows || !tfm.StartsWith("net4"); + + foreach (var runner in runnersFrameworks.Where(filter)) + { + foreach (var fmw in testhostFrameworks.Where(filter)) + { + foreach (var msTestVersion in msTestVersionsToRun) + { + _dataRows.Add(new object[] { new RunnerInfo(runner, fmw), GetMSTestInfo(msTestVersion) }); + } + } + } + } + + private readonly List _dataRows = new(); + private static XmlDocument _depsXml; + + public IEnumerable GetData(MethodInfo methodInfo) + { + return _dataRows; + } + + public string GetDisplayName(MethodInfo methodInfo, object[] data) + { + return string.Format(CultureInfo.CurrentCulture, "{0} ({1})", methodInfo.Name, string.Join(",", data)); + } + + private MSTestInfo GetMSTestInfo(string msTestVersion) + { + // TODO: replacing in the result string is lame, but I am not going to fight 20 GetAssetFullPath method overloads right now + // TODO: this could also be cached of course. + + var depsXml = GetDependenciesXml(); + + XmlNode node = depsXml.DocumentElement.SelectSingleNode($"PropertyGroup/MSTestFramework{msTestVersion}Version"); + var version = node?.InnerText.Replace("[", "").Replace("]", ""); + var slash = Path.DirectorySeparatorChar; + var versionSpecificBinPath = $"{slash}bin{slash}MSTest{msTestVersion}-{version}{slash}"; + + return new MSTestInfo(msTestVersion, version, versionSpecificBinPath); + } + + private static XmlDocument GetDependenciesXml() + { + if (_depsXml != null) + return _depsXml; + + var depsXmlPath = Path.Combine(IntegrationTestEnvironment.TestPlatformRootDirectory, "scripts", "build", "TestPlatform.Dependencies.props"); + var fileStream = File.OpenRead(depsXmlPath); + var xmlTextReader = new XmlTextReader(fileStream) { Namespaces = false }; + var depsXml = new XmlDocument(); + depsXml.Load(xmlTextReader); + + _depsXml = depsXml; + return depsXml; + } +} + diff --git a/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestBase.cs b/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestBase.cs index abbbbf0967..0277197da9 100644 --- a/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestBase.cs +++ b/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestBase.cs @@ -753,13 +753,29 @@ public static void CreateRunSettingsFile(string destinationRunsettingsPath, stri protected string BuildMultipleAssemblyPath(params string[] assetNames) { - var assertFullPaths = new string[assetNames.Length]; + var assetFullPath = new string[assetNames.Length]; for (var i = 0; i < assetNames.Length; i++) { - assertFullPaths[i] = GetAssetFullPath(assetNames[i]).AddDoubleQuote(); + assetFullPath[i] = GetAssetFullPath(assetNames[i]).AddDoubleQuote(); } - return string.Join(" ", assertFullPaths); + return string.Join(" ", assetFullPath); + } + + protected string BuildMultipleAssemblyPath(MSTestInfo msTestInfo, params string[] assetNames) + { + var assetFullPaths = new string[assetNames.Length]; + var slash = Path.DirectorySeparatorChar; + for (var i = 0; i < assetNames.Length; i++) + { + var path = GetAssetFullPath(assetNames[i]); + var updatedPath = msTestInfo.UpdatePath(path); + Assert.IsTrue(File.Exists(updatedPath), "GetTestAsset: Path not found: {0}. Most likely you need to build using build.cmd -s PrepareAcceptanceTests.", updatedPath); + + assetFullPaths[i] = updatedPath.AddDoubleQuote(); + } + + return string.Join(" ", assetFullPaths); } protected static string GetDiagArg(string rootDir) @@ -800,3 +816,29 @@ protected static string GetDownloadedDotnetMuxerFromTools(string architecture) protected static string GetDotnetRunnerPath() => Path.Combine(IntegrationTestEnvironment.TestPlatformRootDirectory, "artifacts", IntegrationTestEnvironment.BuildConfiguration, "netcoreapp2.1", "vstest.console.dll"); } + +public class MSTestInfo +{ + public MSTestInfo(string versionType, string version, string path) + { + VersionType = versionType; + Version = version; + Path = path; + } + + public string VersionType { get; } + public string Version { get; } + public string Path { get; } + + public override string ToString() => $" MSTest = {Version} [{VersionType}]"; + + public string UpdatePath(string path) + { + // Version is not directly used, below, but if it is not populated the path will be incorrect. + // We don't want to throw when creating MSTestInfo because that is happening too early, and has worse error reporting. + if (Version == null) + throw new InvalidOperationException($"Version was not correctly populated from TestPlatform.Dependencies.props, review that there is entry for MSTestFramework{VersionType}Version."); + + return path.Replace($"{System.IO.Path.DirectorySeparatorChar}bin{System.IO.Path.DirectorySeparatorChar}", Path); + } +} diff --git a/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestEnvironment.cs b/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestEnvironment.cs index 2de0b4dd03..51008831c2 100644 --- a/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestEnvironment.cs +++ b/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestEnvironment.cs @@ -235,7 +235,7 @@ public string GetTestAsset(string assetName, string targetFramework) targetFramework, assetName); - Assert.IsTrue(File.Exists(assetPath), "GetTestAsset: Path not found: {0}.", assetPath); + Assert.IsTrue(File.Exists(assetPath), "GetTestAsset: Path not found: {0}. Most likely you need to build using build.cmd -s PrepareAcceptanceTests.", assetPath); return assetPath; } From ecdb18eddf4099708f9f8c24bf4a44cc49d1acc3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Jare=C5=A1?= Date: Thu, 10 Mar 2022 10:51:11 +0100 Subject: [PATCH 044/112] MSTest adapter tests --- scripts/build.ps1 | 28 ++-- scripts/build/TestPlatform.Dependencies.props | 7 +- .../AcceptanceTestBase.cs | 8 +- .../ExecutionTests.cs | 36 ++--- .../MSTestCompatibilityDataSource.cs | 118 +++++++++++++++ .../Extension/NetCoreRunner.cs | 25 ++-- .../NetCoreTargetFrameworkDataSource.cs | 136 +++++------------- .../Extension/NetFrameworkRunner.cs | 27 ++-- .../NetFullTargetFrameworkDataSource.cs | 67 ++++----- .../Extension/RunnnerInfo.cs | 15 +- .../Extension/TestDataSource.cs | 71 +++++++++ .../IntegrationTestBase.cs | 58 ++++---- .../IntegrationTestEnvironment.cs | 4 + .../MSTestInfo.cs | 34 +++++ 14 files changed, 405 insertions(+), 229 deletions(-) create mode 100644 test/Microsoft.TestPlatform.AcceptanceTests/Extension/MSTestCompatibilityDataSource.cs create mode 100644 test/Microsoft.TestPlatform.AcceptanceTests/Extension/TestDataSource.cs create mode 100644 test/Microsoft.TestPlatform.TestUtilities/MSTestInfo.cs diff --git a/scripts/build.ps1 b/scripts/build.ps1 index 625a7838bc..81475960c4 100644 --- a/scripts/build.ps1 +++ b/scripts/build.ps1 @@ -174,7 +174,11 @@ function Invoke-TestAssetsBuild $dependenciesPath = "$env:TP_ROOT_DIR\scripts\build\TestPlatform.Dependencies.props" $dependenciesXml = [xml](Get-Content -Raw -Encoding UTF8 $dependenciesPath) - $project = "$env:TP_ROOT_DIR\test\TestAssets\SimpleTestProject\SimpleTestProject.csproj" + $projects = @( + "$env:TP_ROOT_DIR\test\TestAssets\SimpleTestProject\SimpleTestProject.csproj" + "$env:TP_ROOT_DIR\test\TestAssets\SimpleTestProject2\SimpleTestProject2.csproj" + ) + $versionProperties = @( "MSTestFrameworkLatestStableVersion" "MSTestFrameworkLatestPreviewVersion" @@ -182,18 +186,20 @@ function Invoke-TestAssetsBuild "MSTestFrameworkPreviousStableVersion" "MSTestFrameworkLegacyStableVersion" ) + + foreach ($project in $projects) { + foreach ($propertyName in $versionProperties) { + $mstestVersion = $dependenciesXml.Project.PropertyGroup.$propertyName - foreach ($propertyName in $versionProperties) { - $mstestVersion = $dependenciesXml.Project.PropertyGroup.$propertyName - - if (-not $mstestVersion) - { - throw "MSTestVersion for $propertyName is empty." + if (-not $mstestVersion) + { + throw "MSTestVersion for $propertyName is empty." + } + + $dirVersion = $mstestVersion -replace "\[|\]" + $dirPropertyName = $propertyName -replace "Framework" -replace "Version" + Invoke-Exe $dotnetExe -Arguments "build $project --configuration $TPB_Configuration -v:minimal -p:CIBuild=$TPB_CIBuild -p:LocalizedBuild=$TPB_LocalizedBuild -p:MSTestFrameworkVersion=$mstestVersion -p:MSTestAdapterVersion=$mstestVersion -p:BaseOutputPath=""bin\$dirPropertyName-$dirVersion\\""" } - - $dirVersion = $mstestVersion -replace "\[|\]" - $dirPropertyName = $propertyName -replace "Framework" -replace "Version" - Invoke-Exe $dotnetExe -Arguments "build $project --configuration $TPB_Configuration -v:minimal -p:CIBuild=$TPB_CIBuild -p:LocalizedBuild=$TPB_LocalizedBuild -bl:""$env:TP_OUT_DIR\log\$Configuration\TestAssets.binlog"" -p:MSTestFrameworkVersion=$mstestVersion -p:MSTestAdapterVersion=$mstestVersion -p:BaseOutputPath=""bin\$dirPropertyName-$dirVersion\\""" } } finally { diff --git a/scripts/build/TestPlatform.Dependencies.props b/scripts/build/TestPlatform.Dependencies.props index 4cc9cca978..6d505c671e 100644 --- a/scripts/build/TestPlatform.Dependencies.props +++ b/scripts/build/TestPlatform.Dependencies.props @@ -13,11 +13,14 @@ from a build parameter would not be available, so I am writing this version from the build.ps1 script to keep it in sync --> 17.2.0-dev + 2.2.8 2.2.8 1.0.3-preview - + [2.2.8] [2.2.9-preview-20220210-07] [2.1.0] @@ -29,7 +32,7 @@ $(XUnitFrameworkVersion) 3.10.1 - $(NUnit3FrameworkVersion) + 3.11.0 3.8.0 4.4.12 diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/AcceptanceTestBase.cs b/test/Microsoft.TestPlatform.AcceptanceTests/AcceptanceTestBase.cs index 55b41da95f..3a108e5cf7 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/AcceptanceTestBase.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/AcceptanceTestBase.cs @@ -51,10 +51,12 @@ public class AcceptanceTestBase : IntegrationTestBase public const string NETCORE21_50 = "netcoreapp2.1;netcoreapp3.1;net5.0"; public const string NETFX452_NET50 = "net452;net461;net472;net48;netcoreapp2.1;netcoreapp3.1;net5.0"; public const string NETFX452_NET31 = "net452;net461;net472;net48;netcoreapp2.1;netcoreapp3.1"; + public const string DEFAULT_RUNNER_NETFX = "net451"; /// /// Our current defaults for .NET and .NET Framework. /// - public const string DEFAULT_NETFX_AND_NET = "net451;netcoreapp2.1"; + public const string DEFAULT_RUNNER_NETFX_AND_NET = $"{DEFAULT_RUNNER_NETFX};netcoreapp2.1"; + public const string DEFAULT_HOST_NETFX_AND_NET = "net451;netcoreapp2.1"; public const string LATEST_LEGACY = "Latest;LatestStable;LatestPreview;MostDownloaded;PreviousStable;LegacyStable"; public const string LATESTSTABLE_LEGACY = "LatestStable;LatestPreview;MostDownloaded;PreviousStable;LegacyStable"; @@ -70,6 +72,10 @@ protected static void SetTestEnvironment(IntegrationTestEnvironment testEnvironm testEnvironment.RunnerFramework = runnerInfo.RunnerFramework; testEnvironment.TargetFramework = runnerInfo.TargetFramework; testEnvironment.InIsolationValue = runnerInfo.InIsolationValue; + testEnvironment.DebugVSTestConsole = runnerInfo.DebugVSTestConsole; + testEnvironment.DebugTesthost = runnerInfo.DebugTesthost; + testEnvironment.DebugDataCollector = runnerInfo.DebugDataCollector; + testEnvironment.NoDefaultBreakpoints = runnerInfo.NoDefaultBreakpoints; } protected static string DeriveFrameworkArgValue(IntegrationTestEnvironment testEnvironment) diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/ExecutionTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/ExecutionTests.cs index 8a2dbcfa97..9a7f2b275e 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/ExecutionTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/ExecutionTests.cs @@ -16,28 +16,15 @@ namespace Microsoft.TestPlatform.AcceptanceTests; [TestClass] public class ExecutionTests : AcceptanceTestBase { - [TestMethod] - [NetFullTargetFrameworkDataSource(inIsolation: true, inProcess: true)] - [NetCoreTargetFrameworkDataSource] - public void RunMultipleTestAssemblies(RunnerInfo runnerInfo) - { - SetTestEnvironment(_testEnvironment, runnerInfo); - - var assemblyPaths = BuildMultipleAssemblyPath("SimpleTestProject.dll", "SimpleTestProject2.dll").Trim('\"'); - - InvokeVsTestForExecution(assemblyPaths, GetTestAdapterPath(), FrameworkArgValue, string.Empty); - - ValidateSummaryStatus(2, 2, 2); - ExitCodeEquals(1); // failing tests - } + // TODO: It looks like the first 3 tests would be useful to multiply by all 3 test frameworks, should we make the test even more generic, or duplicate them? [TestMethod] - [MSTestCompatibilityDataSource] - public void RunMultipleTestAssemblies2(RunnerInfo runnerInfo, MSTestInfo msTestInfo) + [MSTestCompatibilityDataSource(InProcess = true)] + public void RunMultipleTestAssemblies(RunnerInfo runnerInfo, MSTestInfo msTestInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - var assemblyPaths = BuildMultipleAssemblyPath(msTestInfo, "SimpleTestProject.dll").Trim('\"'); + var assemblyPaths = BuildMultipleAssemblyPath(msTestInfo, "SimpleTestProject.dll", "SimpleTestProject2.dll").Trim('\"'); InvokeVsTestForExecution(assemblyPaths, testAdapterPath: null, FrameworkArgValue, string.Empty); @@ -45,6 +32,10 @@ public void RunMultipleTestAssemblies2(RunnerInfo runnerInfo, MSTestInfo msTestI ExitCodeEquals(1); // failing tests } + + // TODO: This one mixes different frameworks, I can make it work, but it is worth it? We are going to test + // the two respective versions together (e.g. latest xunit and latest mstest), but does using two different test + // frameworks have any added value over using 2 mstest dlls? [TestMethod] [NetFullTargetFrameworkDataSource(inIsolation: true, inProcess: true)] [NetCoreTargetFrameworkDataSource] @@ -58,7 +49,7 @@ public void RunMultipleTestAssembliesWithoutTestAdapterPath(RunnerInfo runnerInf _testEnvironment.GetTestAsset("XUTestProject.dll"); assemblyPaths = string.Concat(assemblyPaths, "\" \"", xunitAssemblyPath); - InvokeVsTestForExecution(assemblyPaths, string.Empty, FrameworkArgValue, string.Empty); + InvokeVsTestForExecution(assemblyPaths, testAdapterPath: string.Empty, FrameworkArgValue, string.Empty); ValidateSummaryStatus(2, 2, 1); ExitCodeEquals(1); // failing tests @@ -68,15 +59,14 @@ public void RunMultipleTestAssembliesWithoutTestAdapterPath(RunnerInfo runnerInf // and after --arch feature implementation we won't find correct muxer on CI. [TestCategory("Windows")] [TestMethod] - [NetFullTargetFrameworkDataSource] - [NetCoreTargetFrameworkDataSource] - public void RunMultipleTestAssembliesInParallel(RunnerInfo runnerInfo) + [MSTestCompatibilityDataSource] + public void RunMultipleTestAssembliesInParallel(RunnerInfo runnerInfo, MSTestInfo msTestInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); using var tempDir = new TempDirectory(); - var assemblyPaths = BuildMultipleAssemblyPath("SimpleTestProject.dll", "SimpleTestProject2.dll").Trim('\"'); - var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: tempDir.Path); + var assemblyPaths = BuildMultipleAssemblyPath(msTestInfo,"SimpleTestProject.dll", "SimpleTestProject2.dll").Trim('\"'); + var arguments = PrepareArguments(assemblyPaths, testAdapterPath: null, runSettings: null, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: tempDir.Path); arguments = string.Concat(arguments, " /Parallel"); arguments = string.Concat(arguments, " /Platform:x86"); arguments += GetDiagArg(tempDir.Path); diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/MSTestCompatibilityDataSource.cs b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/MSTestCompatibilityDataSource.cs new file mode 100644 index 0000000000..d560945109 --- /dev/null +++ b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/MSTestCompatibilityDataSource.cs @@ -0,0 +1,118 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using System; +using System.Collections.Generic; +using System.Globalization; +using System.IO; +using System.Linq; +using System.Reflection; +using System.Xml; + +using Microsoft.TestPlatform.TestUtilities; + + +namespace Microsoft.TestPlatform.AcceptanceTests; + +public sealed class MSTestCompatibilityDataSource : TestDataSource +{ + private static XmlDocument? _depsXml; + private readonly string[] _runnerFrameworks; + private readonly string[] _targetFrameworks; + private readonly string[] _msTestVersions; + + /// + /// Initializes a new instance. + /// + /// To run tests with desktop runner(vstest.console.exe), use AcceptanceTestBase.Net452TargetFramework or alike values. + public MSTestCompatibilityDataSource(string runners = AcceptanceTestBase.DEFAULT_RUNNER_NETFX_AND_NET, string targetFrameworks = AcceptanceTestBase.DEFAULT_HOST_NETFX_AND_NET, string msTestVersions = AcceptanceTestBase.LATESTSTABLE_LEGACY) + { + _runnerFrameworks = runners.Split(';'); + _targetFrameworks = targetFrameworks.Split(';'); + _msTestVersions = msTestVersions.Split(';'); + + // Do not generate the data rows here, properties (e.g. InProcess) are not populated until after constructor is done. + } + + /// + /// Add also run for in-process using the runner. + /// + // TODO: Can we somehow assert that we actually ran in process? + public bool InProcess { get; set; } + public bool DebugVSTestConsole { get; set; } + public bool DebugTesthost { get; set; } + public bool DebugDataCollector { get; set; } + public bool NoDefaultBreakpoints { get; set; } = true; + + public override void CreateData(MethodInfo methodInfo) + { + var isWindows = Environment.OSVersion.Platform.ToString().StartsWith("Win"); + // Only run .NET Framework tests on Windows. + Func filter = tfm => isWindows || !tfm.StartsWith("net4"); + + if (InProcess) + { + foreach (var msTestVersion in _msTestVersions) + { + var runnerInfo = new RunnerInfo(AcceptanceTestBase.DEFAULT_RUNNER_NETFX, AcceptanceTestBase.DEFAULT_RUNNER_NETFX, inIsolation: null, + DebugVSTestConsole, DebugTesthost, DebugDataCollector, NoDefaultBreakpoints); + var msTestInfo = GetMSTestInfo(msTestVersion); + // We run in the .NET Framework runner process, the runner and target framework must agree. + AddData(runnerInfo, msTestInfo); + } + } + + foreach (var runner in _runnerFrameworks.Where(filter)) + { + foreach (var fmw in _targetFrameworks.Where(filter)) + { + foreach (var msTestVersion in _msTestVersions) + { + var runnerInfo = new RunnerInfo(runner, fmw, AcceptanceTestBase.InIsolation, + DebugVSTestConsole, DebugTesthost, DebugDataCollector, NoDefaultBreakpoints); + var msTestInfo = GetMSTestInfo(msTestVersion); + + AddData(runnerInfo, msTestInfo); + } + } + } + } + + public string GetDisplayName(MethodInfo methodInfo, object[] data) + { + return string.Format(CultureInfo.CurrentCulture, "{0} ({1})", methodInfo.Name, string.Join(",", data)); + } + + private MSTestInfo GetMSTestInfo(string msTestVersion) + { + // TODO: replacing in the result string is lame, but I am not going to fight 20 GetAssetFullPath method overloads right now + // TODO: this could also be cached of course. + + var depsXml = GetDependenciesXml(); + + // It is okay when node is null, we check that Version has value when we update paths by using MSTestInfo, and throw. + // This way it throws in the body of the test which has better error reporting than throwing in the data source. + XmlNode? node = depsXml.DocumentElement?.SelectSingleNode($"PropertyGroup/MSTestFramework{msTestVersion}Version"); + var version = node?.InnerText.Replace("[", "").Replace("]", ""); + var slash = Path.DirectorySeparatorChar; + var versionSpecificBinPath = $"{slash}bin{slash}MSTest{msTestVersion}-{version}{slash}"; + + return new MSTestInfo(msTestVersion, version, versionSpecificBinPath); + } + + private static XmlDocument GetDependenciesXml() + { + if (_depsXml != null) + return _depsXml; + + var depsXmlPath = Path.Combine(IntegrationTestEnvironment.TestPlatformRootDirectory, "scripts", "build", "TestPlatform.Dependencies.props"); + var fileStream = File.OpenRead(depsXmlPath); + var xmlTextReader = new XmlTextReader(fileStream) { Namespaces = false }; + var depsXml = new XmlDocument(); + depsXml.Load(xmlTextReader); + + _depsXml = depsXml; + return depsXml; + } +} + diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/NetCoreRunner.cs b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/NetCoreRunner.cs index 67cc4fdc68..86f2b05b33 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/NetCoreRunner.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/NetCoreRunner.cs @@ -23,27 +23,34 @@ namespace Microsoft.TestPlatform.AcceptanceTests; /// public class NetCoreRunner : Attribute, ITestDataSource { + private readonly string _targetFrameworks; + /// /// Initializes a new instance of the class. /// /// To run tests with desktop runner(vstest.console.exe), use AcceptanceTestBase.Net452TargetFramework or alike values. public NetCoreRunner(string targetFrameworks = AcceptanceTestBase.NETFX452_NET50) { + _targetFrameworks = targetFrameworks; + } + + public bool DebugVSTestConsole { get; set; } + public bool DebugTesthost { get; set; } + public bool DebugDataCollector { get; set; } + public bool NoDefaultBreakpoints { get; set; } = true; + + public IEnumerable GetData(MethodInfo methodInfo) + { + var dataRows = new List(); var isWindows = Environment.OSVersion.Platform.ToString().StartsWith("Win"); // on non-windows we want to filter down only to netcoreapp runner, and net5.0 and newer. Func filter = tfm => isWindows || !tfm.StartsWith("net4"); - foreach (var fmw in targetFrameworks.Split(';').Where(filter)) + foreach (var fmw in _targetFrameworks.Split(';').Where(filter)) { - _dataRows.Add(new object[] { new RunnerInfo(IntegrationTestBase.CoreRunnerFramework, fmw) }); + dataRows.Add(new object[] { new RunnerInfo(IntegrationTestBase.CoreRunnerFramework, fmw, inIsolation: null, DebugVSTestConsole, DebugTesthost, DebugDataCollector, NoDefaultBreakpoints) }); } - } - - private readonly List _dataRows = new(); - - public IEnumerable GetData(MethodInfo methodInfo) - { - return _dataRows; + return dataRows; } public string GetDisplayName(MethodInfo methodInfo, object[] data) diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/NetCoreTargetFrameworkDataSource.cs b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/NetCoreTargetFrameworkDataSource.cs index b9ee94a80e..786b477bbc 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/NetCoreTargetFrameworkDataSource.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/NetCoreTargetFrameworkDataSource.cs @@ -4,12 +4,7 @@ using System; using System.Collections.Generic; using System.Globalization; -using System.IO; -using System.Linq; using System.Reflection; -using System.Text; -using System.Text.RegularExpressions; -using System.Xml; using Microsoft.TestPlatform.TestUtilities; @@ -27,7 +22,11 @@ namespace Microsoft.TestPlatform.AcceptanceTests; /// public class NetCoreTargetFrameworkDataSource : Attribute, ITestDataSource { - private readonly List _dataRows = new(); + private readonly bool _useDesktopRunner; + private readonly bool _useCoreRunner; + private readonly bool _useNetCore21Target; + private readonly bool _useNetCore31Target; + /// /// Initializes a new instance of the class. /// @@ -43,45 +42,56 @@ public NetCoreTargetFrameworkDataSource( // all tests to avoid changing all acceptance tests right now bool useNetCore31Target = false) { + _useDesktopRunner = useDesktopRunner; + _useCoreRunner = useCoreRunner; + _useNetCore21Target = useNetCore21Target; + _useNetCore31Target = useNetCore31Target; + } + + public bool DebugVSTestConsole { get; set; } + public bool DebugTesthost { get; set; } + public bool DebugDataCollector { get; set; } + public bool NoDefaultBreakpoints { get; set; } = true; + + private void AddRunnerDataRow(List dataRows, string runnerFramework, string targetFramework) + { + var runnerInfo = new RunnerInfo(runnerFramework, targetFramework, inIsolation: null, DebugVSTestConsole, DebugTesthost, DebugDataCollector, NoDefaultBreakpoints); + dataRows.Add(new object[] { runnerInfo }); + } + + public IEnumerable GetData(MethodInfo methodInfo) + { + var dataRows = new List(); var isWindows = Environment.OSVersion.Platform.ToString().StartsWith("Win"); - if (useDesktopRunner && isWindows) + if (_useDesktopRunner && isWindows) { var runnerFramework = IntegrationTestBase.DesktopRunnerFramework; - if (useNetCore21Target) + if (_useNetCore21Target) { - AddRunnerDataRow(runnerFramework, AcceptanceTestBase.Core21TargetFramework); + AddRunnerDataRow(dataRows, runnerFramework, AcceptanceTestBase.Core21TargetFramework); } - if (useNetCore31Target) + if (_useNetCore31Target) { - AddRunnerDataRow(runnerFramework, AcceptanceTestBase.Core31TargetFramework); + AddRunnerDataRow(dataRows, runnerFramework, AcceptanceTestBase.Core31TargetFramework); } } - if (useCoreRunner) + if (_useCoreRunner) { var runnerFramework = IntegrationTestBase.CoreRunnerFramework; - if (useNetCore21Target) + if (_useNetCore21Target) { - AddRunnerDataRow(runnerFramework, AcceptanceTestBase.Core21TargetFramework); + AddRunnerDataRow(dataRows, runnerFramework, AcceptanceTestBase.Core21TargetFramework); } - if (useNetCore31Target) + if (_useNetCore31Target) { - AddRunnerDataRow(runnerFramework, AcceptanceTestBase.Core31TargetFramework); + AddRunnerDataRow(dataRows, runnerFramework, AcceptanceTestBase.Core31TargetFramework); } } - } - private void AddRunnerDataRow(string runnerFramework, string targetFramework) - { - var runnerInfo = new RunnerInfo(runnerFramework, targetFramework); - _dataRows.Add(new object[] { runnerInfo }); - } - - public IEnumerable GetData(MethodInfo methodInfo) - { - return _dataRows; + return dataRows; } public string GetDisplayName(MethodInfo methodInfo, object[] data) @@ -89,77 +99,3 @@ public string GetDisplayName(MethodInfo methodInfo, object[] data) return string.Format(CultureInfo.CurrentCulture, "{0} ({1})", methodInfo.Name, string.Join(",", data)); } } - -public class MSTestCompatibilityDataSource : Attribute, ITestDataSource -{ - /// - /// Initializes a new instance of the class. - /// - /// To run tests with desktop runner(vstest.console.exe), use AcceptanceTestBase.Net452TargetFramework or alike values. - public MSTestCompatibilityDataSource(string runners = AcceptanceTestBase.DEFAULT_NETFX_AND_NET, string targetFrameworks = AcceptanceTestBase.DEFAULT_NETFX_AND_NET, string msTestVersions = AcceptanceTestBase.LATESTSTABLE_LEGACY) - { - var runnersFrameworks = runners.Split(';'); - var testhostFrameworks = targetFrameworks.Split(';'); - var msTestVersionsToRun = msTestVersions.Split(';'); - - var isWindows = Environment.OSVersion.Platform.ToString().StartsWith("Win"); - - // Only run .NET Framework tests on Windows. - Func filter = tfm => isWindows || !tfm.StartsWith("net4"); - - foreach (var runner in runnersFrameworks.Where(filter)) - { - foreach (var fmw in testhostFrameworks.Where(filter)) - { - foreach (var msTestVersion in msTestVersionsToRun) - { - _dataRows.Add(new object[] { new RunnerInfo(runner, fmw), GetMSTestInfo(msTestVersion) }); - } - } - } - } - - private readonly List _dataRows = new(); - private static XmlDocument _depsXml; - - public IEnumerable GetData(MethodInfo methodInfo) - { - return _dataRows; - } - - public string GetDisplayName(MethodInfo methodInfo, object[] data) - { - return string.Format(CultureInfo.CurrentCulture, "{0} ({1})", methodInfo.Name, string.Join(",", data)); - } - - private MSTestInfo GetMSTestInfo(string msTestVersion) - { - // TODO: replacing in the result string is lame, but I am not going to fight 20 GetAssetFullPath method overloads right now - // TODO: this could also be cached of course. - - var depsXml = GetDependenciesXml(); - - XmlNode node = depsXml.DocumentElement.SelectSingleNode($"PropertyGroup/MSTestFramework{msTestVersion}Version"); - var version = node?.InnerText.Replace("[", "").Replace("]", ""); - var slash = Path.DirectorySeparatorChar; - var versionSpecificBinPath = $"{slash}bin{slash}MSTest{msTestVersion}-{version}{slash}"; - - return new MSTestInfo(msTestVersion, version, versionSpecificBinPath); - } - - private static XmlDocument GetDependenciesXml() - { - if (_depsXml != null) - return _depsXml; - - var depsXmlPath = Path.Combine(IntegrationTestEnvironment.TestPlatformRootDirectory, "scripts", "build", "TestPlatform.Dependencies.props"); - var fileStream = File.OpenRead(depsXmlPath); - var xmlTextReader = new XmlTextReader(fileStream) { Namespaces = false }; - var depsXml = new XmlDocument(); - depsXml.Load(xmlTextReader); - - _depsXml = depsXml; - return depsXml; - } -} - diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/NetFrameworkRunner.cs b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/NetFrameworkRunner.cs index 49f33f8d02..b8386149f0 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/NetFrameworkRunner.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/NetFrameworkRunner.cs @@ -28,24 +28,31 @@ public class NetFrameworkRunner : Attribute, ITestDataSource /// To run tests with desktop runner(vstest.console.exe), use AcceptanceTestBase.Net452TargetFramework or alike values. public NetFrameworkRunner(string targetFrameworks = AcceptanceTestBase.NETFX452_NET50) { + _targetFrameworks = targetFrameworks; + } + + public bool DebugVSTestConsole { get; set; } + public bool DebugTesthost { get; set; } + public bool DebugDataCollector { get; set; } + public bool NoDefaultBreakpoints { get; set; } = true; + + private readonly string _targetFrameworks; + + public IEnumerable GetData(MethodInfo methodInfo) + { + var dataRows = new List(); var isWindows = Environment.OSVersion.Platform.ToString().StartsWith("Win"); if (!isWindows) { - return; + return dataRows; } - foreach (var fmw in targetFrameworks.Split(';')) + foreach (var fmw in _targetFrameworks.Split(';')) { - _dataRows.Add(new object[] { new RunnerInfo(IntegrationTestBase.DesktopRunnerFramework, fmw, AcceptanceTestBase.InIsolation) }); + dataRows.Add(new object[] { new RunnerInfo(IntegrationTestBase.DesktopRunnerFramework, fmw, AcceptanceTestBase.InIsolation, DebugVSTestConsole, DebugTesthost, DebugDataCollector, NoDefaultBreakpoints) }); } - } - - private readonly List _dataRows = new(); - - public IEnumerable GetData(MethodInfo methodInfo) - { - return _dataRows; + return dataRows; } public string GetDisplayName(MethodInfo methodInfo, object[] data) diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/NetFullTargetFrameworkDataSource.cs b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/NetFullTargetFrameworkDataSource.cs index 626d11422f..a18eeb8271 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/NetFullTargetFrameworkDataSource.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/NetFullTargetFrameworkDataSource.cs @@ -22,6 +22,11 @@ namespace Microsoft.TestPlatform.AcceptanceTests; /// public class NetFullTargetFrameworkDataSource : Attribute, ITestDataSource { + private readonly bool _inIsolation; + private readonly bool _inProcess; + private readonly bool _useDesktopRunner; + private readonly bool _useCoreRunner; + /// /// Initializes a new instance of the class. /// @@ -31,60 +36,40 @@ public class NetFullTargetFrameworkDataSource : Attribute, ITestDataSource /// To run tests with core runner(dotnet vstest.console.dll) public NetFullTargetFrameworkDataSource(bool inIsolation = true, bool inProcess = false, bool useDesktopRunner = true, bool useCoreRunner = true) { - _dataRows = new List(); + _inIsolation = inIsolation; + _inProcess = inProcess; + _useDesktopRunner = useDesktopRunner; + _useCoreRunner = useCoreRunner; + } - var isWindows = Environment.OSVersion.Platform.ToString().StartsWith("Win"); - if (useCoreRunner && isWindows) - { - _dataRows.Add(new object[] { new RunnerInfo(IntegrationTestBase.CoreRunnerFramework, AcceptanceTestBase.DesktopTargetFramework) }); - } + public bool DebugVSTestConsole { get; set; } + public bool DebugTesthost { get; set; } + public bool DebugDataCollector { get; set; } + public bool NoDefaultBreakpoints { get; set; } = true; - if (useDesktopRunner && isWindows) + public IEnumerable GetData(MethodInfo methodInfo) + { + var dataRows = new List(); + var isWindows = Environment.OSVersion.Platform.ToString().StartsWith("Win"); + if (_useCoreRunner && isWindows) { - if (inIsolation) - { - _dataRows.Add(new object[] { new RunnerInfo(IntegrationTestBase.DesktopRunnerFramework, AcceptanceTestBase.DesktopTargetFramework, AcceptanceTestBase.InIsolation) }); - } - - if (inProcess) - { - _dataRows.Add(new object[] { new RunnerInfo(IntegrationTestBase.DesktopRunnerFramework, AcceptanceTestBase.DesktopTargetFramework) }); - } + dataRows.Add(new object[] { new RunnerInfo(IntegrationTestBase.CoreRunnerFramework, AcceptanceTestBase.DesktopTargetFramework, inIsolation: null, DebugVSTestConsole, DebugTesthost, DebugDataCollector, NoDefaultBreakpoints) }); } - } - - /// - /// Initializes a new instance of the class. - /// - /// To run tests with desktop runner(vstest.console.exe), use AcceptanceTestBase.Net452TargetFramework or alike values. - public NetFullTargetFrameworkDataSource(string[] targetFrameworks, bool inIsolation = true, bool inProcess = false) - { - if (inIsolation) + if (_useDesktopRunner && isWindows) { - foreach (var fmw in targetFrameworks) + if (_inIsolation) { - _dataRows.Add(new object[] { new RunnerInfo(IntegrationTestBase.DesktopRunnerFramework, fmw, AcceptanceTestBase.InIsolation) }); + dataRows.Add(new object[] { new RunnerInfo(IntegrationTestBase.DesktopRunnerFramework, AcceptanceTestBase.DesktopTargetFramework, AcceptanceTestBase.InIsolation, DebugVSTestConsole, DebugTesthost, DebugDataCollector, NoDefaultBreakpoints) }); } - } - if (inProcess) - { - foreach (var fmw in targetFrameworks) + if (_inProcess) { - _dataRows.Add(new object[] { new RunnerInfo(IntegrationTestBase.DesktopRunnerFramework, fmw) }); + dataRows.Add(new object[] { new RunnerInfo(IntegrationTestBase.DesktopRunnerFramework, AcceptanceTestBase.DesktopTargetFramework, inIsolation: null, DebugVSTestConsole, DebugTesthost, DebugDataCollector, NoDefaultBreakpoints) }); } } - } - - /// - /// Gets or sets the data rows. - /// - private readonly List _dataRows = new(); - public IEnumerable GetData(MethodInfo methodInfo) - { - return _dataRows; + return dataRows; } public string GetDisplayName(MethodInfo methodInfo, object[] data) diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/RunnnerInfo.cs b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/RunnnerInfo.cs index d9f9a14a18..5f73d8a0c5 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/RunnnerInfo.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/RunnnerInfo.cs @@ -9,15 +9,16 @@ namespace Microsoft.TestPlatform.AcceptanceTests; public class RunnerInfo { - public RunnerInfo(string runnerType, string targetFramework) : this(runnerType, targetFramework, "") - { - } - - public RunnerInfo(string runnerType, string targetFramework, string inIsolation) + public RunnerInfo(string runnerType, string targetFramework, string inIsolation, + bool debugVSTestConsole, bool debugTesthost, bool debugDataCollector, bool noDefaultBreakpoints) { RunnerFramework = runnerType; TargetFramework = targetFramework; InIsolationValue = inIsolation; + DebugVSTestConsole = debugVSTestConsole; + DebugTesthost = debugTesthost; + DebugDataCollector = debugDataCollector; + NoDefaultBreakpoints = noDefaultBreakpoints; // The value is netcoreapp2.1. IsNetRunner = RunnerFramework.StartsWith("netcoreapp", StringComparison.InvariantCultureIgnoreCase); // The value is net451. @@ -43,6 +44,10 @@ public string InIsolationValue { get; set; } + public bool DebugVSTestConsole { get; } + public bool DebugTesthost { get; } + public bool DebugDataCollector { get; } + public bool NoDefaultBreakpoints { get; } /// /// Gets the application type. diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/TestDataSource.cs b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/TestDataSource.cs new file mode 100644 index 0000000000..104aee8d8a --- /dev/null +++ b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/TestDataSource.cs @@ -0,0 +1,71 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using System; +using System.Collections.Generic; +using System.Reflection; + +using Microsoft.VisualStudio.TestTools.UnitTesting; + +namespace Microsoft.TestPlatform.AcceptanceTests; + +[AttributeUsage(AttributeTargets.Method)] +public abstract class TestDataSource : Attribute, ITestDataSource where T1 : notnull +{ + private readonly List _data = new(); + + public abstract void CreateData(MethodInfo methodInfo); + + public void AddData(T1 value1) + { + _data.Add(new object[] { value1 }); + } + + public virtual string GetDisplayName(MethodInfo methodInfo, T1 value1) + { + return $"{methodInfo.Name} ({value1})"; + } + + IEnumerable ITestDataSource.GetData(MethodInfo methodInfo) + { + CreateData(methodInfo); + return _data; + } + + string ITestDataSource.GetDisplayName(MethodInfo methodInfo, object[] data) + { + return GetDisplayName(methodInfo, (T1)data[0]); + } +} + +[AttributeUsage(AttributeTargets.Method)] +public abstract class TestDataSource : Attribute, ITestDataSource + where T1 : notnull + where T2 : notnull +{ + private readonly List _data = new(); + + public abstract void CreateData(MethodInfo methodInfo); + + public void AddData(T1 value1, T2 value2) + { + _data.Add(new object[] { value1, value2 }); + } + + public virtual string GetDisplayName(MethodInfo methodInfo, T1 value1, T2 value2) + { + return $"{methodInfo.Name} ({value1}, {value2})"; + } + + IEnumerable ITestDataSource.GetData(MethodInfo methodInfo) + { + CreateData(methodInfo); + return _data; + } + + string ITestDataSource.GetDisplayName(MethodInfo methodInfo, object[] data) + { + return GetDisplayName(methodInfo, (T1)data[0], (T2)data[1]); + } +} + diff --git a/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestBase.cs b/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestBase.cs index c7076109ed..a5d8ca3b5a 100644 --- a/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestBase.cs +++ b/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestBase.cs @@ -128,6 +128,11 @@ public static string PrepareArguments(string[] testAssemblies, string testAdapte if (!string.IsNullOrWhiteSpace(inIsolation)) { + if (inIsolation != "/InIsolation") + { + // TODO: The whole inIsolation should be just a bool, but it is not, and it's changing in other PR. + throw new InvalidOperationException("InIsolation value must be '/InIsolation'"); + } arguments = string.Concat(arguments, " ", inIsolation); } @@ -205,6 +210,32 @@ public void InvokeVsTestForExecution(string testAssembly, Dictionary environmentVariables = null) { var arguments = PrepareArguments(testAssembly, testAdapterPath, runSettings, framework, _testEnvironment.InIsolationValue, resultsDirectory: TempDirectory.Path); + + if (_testEnvironment.DebugVSTestConsole || _testEnvironment.DebugTesthost || _testEnvironment.DebugDataCollector) + { + environmentVariables ??= new Dictionary(); + + if (_testEnvironment.DebugVSTestConsole) + { + environmentVariables.Add("VSTEST_RUNNER_DEBUG_ATTACHVS", "1"); + } + + if (_testEnvironment.DebugTesthost) + { + environmentVariables.Add("VSTEST_HOST_DEBUG_ATTACHVS", "1"); + } + + if (_testEnvironment.DebugDataCollector) + { + environmentVariables.Add("VSTEST_DATACOLLECTOR_DEBUG_ATTACHVS", "1"); + } + + if (_testEnvironment.NoDefaultBreakpoints) + { + environmentVariables.Add("VSTEST_DEBUG_NOBP", "1"); + } + } + InvokeVsTest(arguments, environmentVariables); } @@ -780,7 +811,6 @@ protected string BuildMultipleAssemblyPath(params string[] assetNames) protected string BuildMultipleAssemblyPath(MSTestInfo msTestInfo, params string[] assetNames) { var assetFullPaths = new string[assetNames.Length]; - var slash = Path.DirectorySeparatorChar; for (var i = 0; i < assetNames.Length; i++) { var path = GetAssetFullPath(assetNames[i]); @@ -831,29 +861,3 @@ protected static string GetDownloadedDotnetMuxerFromTools(string architecture) protected static string GetDotnetRunnerPath() => Path.Combine(IntegrationTestEnvironment.TestPlatformRootDirectory, "artifacts", IntegrationTestEnvironment.BuildConfiguration, "netcoreapp2.1", "vstest.console.dll"); } - -public class MSTestInfo -{ - public MSTestInfo(string versionType, string version, string path) - { - VersionType = versionType; - Version = version; - Path = path; - } - - public string VersionType { get; } - public string Version { get; } - public string Path { get; } - - public override string ToString() => $" MSTest = {Version} [{VersionType}]"; - - public string UpdatePath(string path) - { - // Version is not directly used, below, but if it is not populated the path will be incorrect. - // We don't want to throw when creating MSTestInfo because that is happening too early, and has worse error reporting. - if (Version == null) - throw new InvalidOperationException($"Version was not correctly populated from TestPlatform.Dependencies.props, review that there is entry for MSTestFramework{VersionType}Version."); - - return path.Replace($"{System.IO.Path.DirectorySeparatorChar}bin{System.IO.Path.DirectorySeparatorChar}", Path); - } -} diff --git a/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestEnvironment.cs b/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestEnvironment.cs index 51008831c2..66195fba24 100644 --- a/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestEnvironment.cs +++ b/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestEnvironment.cs @@ -193,6 +193,10 @@ public string RunnerFramework // A known AzureDevOps env variable meaning we are running in CI. public static bool IsCI { get; } = Environment.GetEnvironmentVariable("TF_BUILD") == "True"; + public bool DebugVSTestConsole { get; set; } + public bool DebugTesthost { get; set; } + public bool DebugDataCollector { get; set; } + public bool NoDefaultBreakpoints { get; set; } /// /// Gets the full path to a test asset. diff --git a/test/Microsoft.TestPlatform.TestUtilities/MSTestInfo.cs b/test/Microsoft.TestPlatform.TestUtilities/MSTestInfo.cs new file mode 100644 index 0000000000..55453ed196 --- /dev/null +++ b/test/Microsoft.TestPlatform.TestUtilities/MSTestInfo.cs @@ -0,0 +1,34 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using System; + +namespace Microsoft.TestPlatform.TestUtilities; + +public class MSTestInfo +{ + public MSTestInfo(string versionType, string? version, string path) + { + VersionType = versionType; + // Version can be null when we fail to find the respective propertin in TestPlatform.Dependencies.props + // when that happens we throw when we try to update the path. + Version = version; + Path = path; + } + + public string VersionType { get; } + public string? Version { get; } + public string Path { get; } + + public override string ToString() => $" MSTest = {Version} [{VersionType}]"; + + public string UpdatePath(string path) + { + // Version is not directly used, below, but if it is not populated the path will be incorrect. + // We don't want to throw when creating MSTestInfo because that is happening too early, and has worse error reporting. + if (Version == null) + throw new InvalidOperationException($"Version was not correctly populated from TestPlatform.Dependencies.props, review that there is entry for MSTestFramework{VersionType}Version."); + + return path.Replace($"{System.IO.Path.DirectorySeparatorChar}bin{System.IO.Path.DirectorySeparatorChar}", Path); + } +} From a65c00ab7812582b4b3097350fab3a70c7faef6e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Jare=C5=A1?= Date: Thu, 10 Mar 2022 17:21:47 +0100 Subject: [PATCH 045/112] Add vstest console data source, even though it is names translation layer. --- scripts/build.ps1 | 81 +++++++++- scripts/build/TestPlatform.Dependencies.props | 41 +++-- scripts/verify-nupkgs.ps1 | 4 +- .../AcceptanceTestBase.cs | 5 +- .../MSTestCompatibilityDataSource.cs | 6 +- ...TranslationLayerCompatibilityDataSource.cs | 140 ++++++++++++++++++ .../TranslationLayerTests/DiscoverTests.cs | 12 +- .../IntegrationTestBase.cs | 35 +++-- .../MSTestInfo.cs | 1 + .../VSTestConsoleInfo.cs | 23 +++ 10 files changed, 306 insertions(+), 42 deletions(-) create mode 100644 test/Microsoft.TestPlatform.AcceptanceTests/Extension/TranslationLayerCompatibilityDataSource.cs create mode 100644 test/Microsoft.TestPlatform.TestUtilities/VSTestConsoleInfo.cs diff --git a/scripts/build.ps1 b/scripts/build.ps1 index 81475960c4..0b39b742d6 100644 --- a/scripts/build.ps1 +++ b/scripts/build.ps1 @@ -169,19 +169,23 @@ function Invoke-TestAssetsBuild try { Write-Log ".. .. Build: Source: $TPB_TestAssets_Solution -- add NuGet source" Invoke-Exe -IgnoreExitCode 1 $nugetExe -Arguments "sources add -Name ""locally-built-testplatform-packages"" -Source $env:TP_TESTARTIFACTS\packages\ -ConfigFile ""$nugetConfig""" - Invoke-Exe $dotnetExe -Arguments "build $TPB_TestAssets_Solution --configuration $TPB_Configuration -v:minimal -p:CIBuild=$TPB_CIBuild -p:LocalizedBuild=$TPB_LocalizedBuild -bl:""$env:TP_OUT_DIR\log\$Configuration\TestAssets.binlog""" + #Invoke-Exe $dotnetExe -Arguments "build $TPB_TestAssets_Solution --configuration $TPB_Configuration -v:minimal -p:CIBuild=$TPB_CIBuild -p:LocalizedBuild=$TPB_LocalizedBuild -bl:""$env:TP_OUT_DIR\log\$Configuration\TestAssets.binlog""" + # Compatibility matrix build $dependenciesPath = "$env:TP_ROOT_DIR\scripts\build\TestPlatform.Dependencies.props" $dependenciesXml = [xml](Get-Content -Raw -Encoding UTF8 $dependenciesPath) + + # build with multiple versions of MSTest $projects = @( "$env:TP_ROOT_DIR\test\TestAssets\SimpleTestProject\SimpleTestProject.csproj" "$env:TP_ROOT_DIR\test\TestAssets\SimpleTestProject2\SimpleTestProject2.csproj" ) $versionProperties = @( - "MSTestFrameworkLatestStableVersion" "MSTestFrameworkLatestPreviewVersion" + "MSTestFrameworkLatestStableVersion" + "MSTestFrameworkRecentStableVersion" "MSTestFrameworkMostDownloadedVersion" "MSTestFrameworkPreviousStableVersion" "MSTestFrameworkLegacyStableVersion" @@ -201,6 +205,68 @@ function Invoke-TestAssetsBuild Invoke-Exe $dotnetExe -Arguments "build $project --configuration $TPB_Configuration -v:minimal -p:CIBuild=$TPB_CIBuild -p:LocalizedBuild=$TPB_LocalizedBuild -p:MSTestFrameworkVersion=$mstestVersion -p:MSTestAdapterVersion=$mstestVersion -p:BaseOutputPath=""bin\$dirPropertyName-$dirVersion\\""" } } + + # restore previous versions of TestPlatform (for vstest.console.exe), and TestPlatform.CLI (for vstest.console.dll) + $versionProperties = @( + "NETTestSdkVersion" + "VSTestConsoleLatestPreviewVersion" + "VSTestConsoleLatestStableVersion" + "VSTestConsoleRecentStableVersion" + "VSTestConsoleMostDownloadedVersion" + "VSTestConsolePreviousStableVersion" + "VSTestConsoleLegacyStableVersion" + ) + + foreach ($propertyName in $versionProperties) { + if ("VSTestConsoleLatestVersion" -eq $propertyName) { + # NETTestSdkVersion has the version of the locally built package. + $vsTestConsoleVersion = $dependenciesXml.Project.PropertyGroup."NETTestSdkVersion" + } + else { + $vsTestConsoleVersion = $dependenciesXml.Project.PropertyGroup.$propertyName + } + + # The command line tool does not like the package ranges. + $vsTestConsoleVersion = $vsTestConsoleVersion -replace "(\[|\])" + if (-not $vsTestConsoleVersion) + { + throw "VSTestConsoleVersion for $propertyName is empty." + } + + $packages = @( + "Microsoft.TestPlatform" + "Microsoft.TestPlatform.CLI", + "Microsoft.TestPlatform.TranslationLayer" + "Microsoft.NET.Test.SDK" + ) + + foreach ($package in $packages) { + $packagePath = "$env:TP_ROOT_DIR\packages\$($package.ToLower())" + $cachePath = "$packagePath\$vstestConsoleVersion" + + if ((Test-Path -Path $cachePath) -and (Get-ChildItem $cachePath)) { + "Package $package $vsTestConsoleVersion is already in nuget cache at $cachePath." + continue + } + + Invoke-Exe $nugetExe -Arguments "install $package -Version $vsTestConsoleVersion -OutputDirectory $packagePath -ConfigFile ""$nugetConfig""" + + # Install puts it in packages/microsoft.testplatform/Microsoft.TestPlatform.17.1.0, + # because we use that as our output folder. And it also caches it in packages/microsoft.testplatform/17.1.0 + # unless the package is from local source, then it does not do that. So we need to rename the folder and remove + # the original one. + if (-not (Test-Path -Path $cachePath) -or -not (Get-ChildItem $cachePath)) { + Rename-Item "$packagePath\$package.$vsTestConsoleVersion" $cachePath + # nuget locks the locally copied package it seems. + Start-Sleep -Milliseconds 300 + } + if (Test-Path "$packagePath\$package.$vsTestConsoleVersion") { + Remove-Item -Recurse -Force "$packagePath\$package.$vsTestConsoleVersion" + } + } + } + + # end } finally { Write-Log ".. .. Build: Source: $TPB_TestAssets_Solution -- remove NuGet source" @@ -835,9 +901,14 @@ function Create-NugetPackages Copy-Item (Join-Path $env:TP_PACKAGE_PROJ_DIR "Icon.png") $stagingDir -Force + # Remove all locally built nuget packages before we start creating them + # we are leaving them in the folder after uzipping them for easier review. + if (Test-Path $packageOutputDir) { + Remove-Item $packageOutputDir -Recurse -Force + } if (-not (Test-Path $packageOutputDir)) { - New-Item $packageOutputDir -type directory -Force + New-Item $packageOutputDir -Type directory -Force } $tpNuspecDir = Join-Path $env:TP_PACKAGE_PROJ_DIR "nuspec" @@ -1258,9 +1329,9 @@ Get-Variable | Where-Object -FilterScript { $_.Name.StartsWith("TPB_") } | Forma # } if ($Force -or $Steps -contains "PrepareAcceptanceTests") { - # Publish-PatchedDotnet + #Publish-PatchedDotnet Invoke-TestAssetsBuild - # Publish-Tests + #Publish-Tests } if ($Script:ScriptFailed) { diff --git a/scripts/build/TestPlatform.Dependencies.props b/scripts/build/TestPlatform.Dependencies.props index 6d505c671e..30969e3253 100644 --- a/scripts/build/TestPlatform.Dependencies.props +++ b/scripts/build/TestPlatform.Dependencies.props @@ -13,23 +13,16 @@ from a build parameter would not be available, so I am writing this version from the build.ps1 script to keep it in sync --> 17.2.0-dev - + 2.2.8 2.2.8 1.0.3-preview - - [2.2.8] - [2.2.9-preview-20220210-07] - [2.1.0] - [2.1.0] - [1.4.0] - + 2.3.1 - $(XUnitFrameworkVersion) - $(XUnitFrameworkVersion) + 2.3.1 + 2.3.1 3.10.1 3.11.0 @@ -37,6 +30,30 @@ 4.4.12 + + + [2.2.9-preview-20220210-07] + [2.2.8] + [2.2.7] + [2.1.0] + [2.1.0] + [1.4.0] + + + + + 17.2.0-preview-20220131-20 + [17.1.0] + [17.0.0] + [16.9.4] + [16.11.0] + [15.9.2] + + 5.11.0 5.0.0 diff --git a/scripts/verify-nupkgs.ps1 b/scripts/verify-nupkgs.ps1 index 86f09edb87..432bfbbcb1 100644 --- a/scripts/verify-nupkgs.ps1 +++ b/scripts/verify-nupkgs.ps1 @@ -56,7 +56,9 @@ function Verify-Nuget-Packages($packageDirectory, $version) Write-Error "Number of files are not equal $unzipNugetPackageDir, expected: $($expectedNumOfFiles[$packageKey]) actual: $actualNumOfFiles" } - Remove-Item -Force -Recurse $unzipNugetPackageDir | Out-Null + # Don't remove the directories after you unpacked them + # they are useful for reviewing what is in the package. + # Remove-Item -Force -Recurse $unzipNugetPackageDir | Out-Null } Write-Log "Completed Verify-Nuget-Packages." diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/AcceptanceTestBase.cs b/test/Microsoft.TestPlatform.AcceptanceTests/AcceptanceTestBase.cs index 3a108e5cf7..09e2d9fa4c 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/AcceptanceTestBase.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/AcceptanceTestBase.cs @@ -57,8 +57,9 @@ public class AcceptanceTestBase : IntegrationTestBase /// public const string DEFAULT_RUNNER_NETFX_AND_NET = $"{DEFAULT_RUNNER_NETFX};netcoreapp2.1"; public const string DEFAULT_HOST_NETFX_AND_NET = "net451;netcoreapp2.1"; - public const string LATEST_LEGACY = "Latest;LatestStable;LatestPreview;MostDownloaded;PreviousStable;LegacyStable"; - public const string LATESTSTABLE_LEGACY = "LatestStable;LatestPreview;MostDownloaded;PreviousStable;LegacyStable"; + public const string LATEST_LEGACY = "Latest;LatestPreview;LatestStable;RecentStable;MostDownloaded;PreviousStable;LegacyStable"; + public const string LATESTPREVIEW_LEGACY = "LatestPreview;LatestStable;RecentStable;MostDownloaded;PreviousStable;LegacyStable"; + public const string LATEST = "Latest"; public static string And(string left, string right) { diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/MSTestCompatibilityDataSource.cs b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/MSTestCompatibilityDataSource.cs index d560945109..b491bda22e 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/MSTestCompatibilityDataSource.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/MSTestCompatibilityDataSource.cs @@ -25,7 +25,7 @@ public sealed class MSTestCompatibilityDataSource : TestDataSource /// To run tests with desktop runner(vstest.console.exe), use AcceptanceTestBase.Net452TargetFramework or alike values. - public MSTestCompatibilityDataSource(string runners = AcceptanceTestBase.DEFAULT_RUNNER_NETFX_AND_NET, string targetFrameworks = AcceptanceTestBase.DEFAULT_HOST_NETFX_AND_NET, string msTestVersions = AcceptanceTestBase.LATESTSTABLE_LEGACY) + public MSTestCompatibilityDataSource(string runners = AcceptanceTestBase.DEFAULT_RUNNER_NETFX_AND_NET, string targetFrameworks = AcceptanceTestBase.DEFAULT_HOST_NETFX_AND_NET, string msTestVersions = AcceptanceTestBase.LATESTPREVIEW_LEGACY) { _runnerFrameworks = runners.Split(';'); _targetFrameworks = targetFrameworks.Split(';'); @@ -85,9 +85,6 @@ public string GetDisplayName(MethodInfo methodInfo, object[] data) private MSTestInfo GetMSTestInfo(string msTestVersion) { - // TODO: replacing in the result string is lame, but I am not going to fight 20 GetAssetFullPath method overloads right now - // TODO: this could also be cached of course. - var depsXml = GetDependenciesXml(); // It is okay when node is null, we check that Version has value when we update paths by using MSTestInfo, and throw. @@ -115,4 +112,3 @@ private static XmlDocument GetDependenciesXml() return depsXml; } } - diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/TranslationLayerCompatibilityDataSource.cs b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/TranslationLayerCompatibilityDataSource.cs new file mode 100644 index 0000000000..2d1b637399 --- /dev/null +++ b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/TranslationLayerCompatibilityDataSource.cs @@ -0,0 +1,140 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using System; +using System.Globalization; +using System.IO; +using System.Linq; +using System.Reflection; +using System.Xml; + +using Microsoft.TestPlatform.TestUtilities; + + +namespace Microsoft.TestPlatform.AcceptanceTests; + +public sealed class TranslationLayerCompatibilityDataSource : TestDataSource +{ + private static XmlDocument? _depsXml; + private readonly string[] _runnerFrameworks; + private readonly string[] _targetFrameworks; + private readonly string[] _translationLayerVersions; + private readonly string[] _vstestConsoleVersions; + + /// + /// Initializes a new instance. + /// + /// To run tests with desktop runner(vstest.console.exe), use AcceptanceTestBase.Net452TargetFramework or alike values. + public TranslationLayerCompatibilityDataSource( + string runners = AcceptanceTestBase.DEFAULT_RUNNER_NETFX_AND_NET, + string targetFrameworks = AcceptanceTestBase.DEFAULT_HOST_NETFX_AND_NET, + // string translationLayerVersions = AcceptanceTestBase.LATEST_LEGACY, + string vstestConsoleVersions = AcceptanceTestBase.LATEST_LEGACY) + { + _runnerFrameworks = runners.Split(';'); + _targetFrameworks = targetFrameworks.Split(';'); + // _translationLayerVersions = translationLayerVersions.Split(';'); + _vstestConsoleVersions = vstestConsoleVersions.Split(';'); + + // Do not generate the data rows here, properties (e.g. DebugVSTestConsole) are not populated until after constructor is done. + } + + public bool DebugVSTestConsole { get; set; } + public bool DebugTesthost { get; set; } + public bool DebugDataCollector { get; set; } + public bool NoDefaultBreakpoints { get; set; } = true; + + public override void CreateData(MethodInfo methodInfo) + { + var isWindows = Environment.OSVersion.Platform.ToString().StartsWith("Win"); + // Only run .NET Framework tests on Windows. + Func filter = tfm => isWindows || !tfm.StartsWith("net4"); + + + foreach (var runner in _runnerFrameworks.Where(filter)) + { + foreach (var fmw in _targetFrameworks.Where(filter)) + { + foreach (var vstestConsoleVersion in _vstestConsoleVersions) + { + var runnerInfo = new RunnerInfo(runner, fmw, AcceptanceTestBase.InIsolation, + DebugVSTestConsole, DebugTesthost, DebugDataCollector, NoDefaultBreakpoints); + var vsTestConsoleInfo = GetVSTestConsoleInfo(vstestConsoleVersion, runnerInfo); + + AddData(runnerInfo, vsTestConsoleInfo); + } + } + } + } + + public string GetDisplayName(MethodInfo methodInfo, object[] data) + { + return string.Format(CultureInfo.CurrentCulture, "{0} ({1})", methodInfo.Name, string.Join(",", data)); + } + + private VSTestConsoleInfo GetVSTestConsoleInfo(string vstestConsoleVersion, RunnerInfo runnerInfo) + { + var depsXml = GetDependenciesXml(); + + // When version is Latest, we built it locally, but it gets restored into our nuget cache on build + // same as other versions, we just need to grab the version from a different property. + + var propertyName = vstestConsoleVersion == AcceptanceTestBase.LATEST + ? $"NETTestSdkVersion" + : $"VSTestConsole{vstestConsoleVersion}Version"; + + var packageName = runnerInfo.IsNetFrameworkRunner + ? "microsoft.testplatform" + : "microsoft.testplatform.cli"; + + // It is okay when node is null, we will fail to find the executable later, and throw. + // This way it throws in the body of the test which has better error reporting than throwing in the data source. + // And we can easily find out what is going on because --WRONG-VERSION-- sticks out, and is easy to find in the codebase. + XmlNode? node = depsXml.DocumentElement?.SelectSingleNode($"PropertyGroup/{propertyName}"); + var version = node?.InnerText.Replace("[", "").Replace("]", "") ?? "--WRONG-VERSION--"; + var vstestConsolePath = runnerInfo.IsNetFrameworkRunner + ? Path.Combine(IntegrationTestEnvironment.TestPlatformRootDirectory, "packages", packageName, version, + "tools", "net451", "Common7", "IDE", "Extensions", "TestPlatform", "vstest.console.exe") + : Path.Combine(IntegrationTestEnvironment.TestPlatformRootDirectory, "packages", packageName, version, + "contentFiles", "any", "netcoreapp2.1", "vstest.console.dll"); + + if (version.StartsWith("15.")) + { + vstestConsolePath = vstestConsolePath.Replace("netcoreapp2.1", "netcoreapp2.0"); + } + + return new VSTestConsoleInfo(vstestConsoleVersion, version, vstestConsolePath); + } + + //private MSTestInfo GetTranslationLayerInfo(string translationLayerVersion) + //{ + // // TODO: replacing in the result string is lame, but I am not going to fight 20 GetAssetFullPath method overloads right now + // // TODO: this could also be cached of course. + + // var depsXml = GetDependenciesXml(); + + // // It is okay when node is null, we check that Version has value when we update paths by using MSTestInfo, and throw. + // // This way it throws in the body of the test which has better error reporting than throwing in the data source. + // XmlNode? node = depsXml.DocumentElement?.SelectSingleNode($"PropertyGroup/TranslationLayer{translationLayerVersion}Version"); + // var version = node?.InnerText.Replace("[", "").Replace("]", ""); + // var slash = Path.DirectorySeparatorChar; + // var dllPath = $"{slash}bin{slash}MSTest{translationLayerVersion}-{version}{slash}"; + + // return new TranslationLayerInfo(dllPath, version, versionSpecificBinPath); + //} + + private static XmlDocument GetDependenciesXml() + { + if (_depsXml != null) + return _depsXml; + + var depsXmlPath = Path.Combine(IntegrationTestEnvironment.TestPlatformRootDirectory, "scripts", "build", "TestPlatform.Dependencies.props"); + var fileStream = File.OpenRead(depsXmlPath); + var xmlTextReader = new XmlTextReader(fileStream) { Namespaces = false }; + var depsXml = new XmlDocument(); + depsXml.Load(xmlTextReader); + + _depsXml = depsXml; + return depsXml; + } +} diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/DiscoverTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/DiscoverTests.cs index c891784174..482af33303 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/DiscoverTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/DiscoverTests.cs @@ -40,15 +40,17 @@ public void Cleanup() } [TestMethod] - [NetFullTargetFrameworkDataSource] - [NetCoreTargetFrameworkDataSource] - public void DiscoverTestsUsingDiscoveryEventHandler1(RunnerInfo runnerInfo) + [TranslationLayerCompatibilityDataSource()] + public void DiscoverTestsUsingDiscoveryEventHandler1(RunnerInfo runnerInfo, VSTestConsoleInfo vsTestConsoleInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - Setup(); + // Setup(); + _discoveryEventHandler = new DiscoveryEventHandler(); + _discoveryEventHandler2 = new DiscoveryEventHandler2(); - _vstestConsoleWrapper.DiscoverTests(GetTestAssemblies(), GetDefaultRunSettings(), _discoveryEventHandler); + var vstestConsoleWrapper = GetVsTestConsoleWrapper(TempDirectory, vsTestConsoleInfo); + vstestConsoleWrapper.DiscoverTests(GetTestAssemblies(), GetDefaultRunSettings(), _discoveryEventHandler); // Assert. Assert.AreEqual(6, _discoveryEventHandler.DiscoveredTestCases.Count); diff --git a/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestBase.cs b/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestBase.cs index a5d8ca3b5a..1399274b77 100644 --- a/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestBase.cs +++ b/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestBase.cs @@ -555,7 +555,7 @@ protected virtual string SetVSTestConsoleDLLPathInArgs(string args) /// public IVsTestConsoleWrapper GetVsTestConsoleWrapper() { - return GetVsTestConsoleWrapper(TempDirectory); + return GetVsTestConsoleWrapper(TempDirectory, null); } /// @@ -564,21 +564,24 @@ public IVsTestConsoleWrapper GetVsTestConsoleWrapper() public IVsTestConsoleWrapper GetVsTestConsoleWrapper(out TempDirectory logFileDir) { logFileDir = new TempDirectory(); - return GetVsTestConsoleWrapper(logFileDir); + return GetVsTestConsoleWrapper(logFileDir, null); } /// /// Returns the VsTestConsole Wrapper. /// - /// - public IVsTestConsoleWrapper GetVsTestConsoleWrapper(TempDirectory logFileDir) + public IVsTestConsoleWrapper GetVsTestConsoleWrapper(TempDirectory tempDirectory) { - if (!Directory.Exists(logFileDir.Path)) - { - Directory.CreateDirectory(logFileDir.Path); - } + return GetVsTestConsoleWrapper(tempDirectory, vsTestConsoleInfo: null); + } - // Directory is already unique so there is no need to have a unique file name. + /// + /// Returns the VsTestConsole Wrapper. + /// + /// + public IVsTestConsoleWrapper GetVsTestConsoleWrapper(TempDirectory logFileDir, VSTestConsoleInfo vsTestConsoleInfo) + { + // Temp directory is already unique so there is no need to have a unique file name. var logFilePath = Path.Combine(logFileDir.Path, "log.txt"); if (!File.Exists(logFilePath)) { @@ -587,9 +590,11 @@ public IVsTestConsoleWrapper GetVsTestConsoleWrapper(TempDirectory logFileDir) Console.WriteLine($"Logging diagnostics in {logFilePath}"); - var consoleRunnerPath = IsNetCoreRunner() - ? Path.Combine(_testEnvironment.PublishDirectory, "vstest.console.dll") - : GetConsoleRunnerPath(); + var consoleRunnerPath = vsTestConsoleInfo != null + ? vsTestConsoleInfo.Path + : IsNetCoreRunner() + ? Path.Combine(_testEnvironment.PublishDirectory, "vstest.console.dll") + : GetConsoleRunnerPath(); var executablePath = IsWindows ? @"dotnet\dotnet.exe" : @"dotnet-linux/dotnet"; var dotnetPath = Path.Combine(_testEnvironment.ToolsDirectory, executablePath); @@ -598,6 +603,12 @@ public IVsTestConsoleWrapper GetVsTestConsoleWrapper(TempDirectory logFileDir) throw new FileNotFoundException($"File '{dotnetPath}' was not found."); } + if (!File.Exists(consoleRunnerPath)) + { + throw new FileNotFoundException($"File '{consoleRunnerPath}' was not found."); + } + + Console.WriteLine($"Console runner path: {consoleRunnerPath}"); var vstestConsoleWrapper = new VsTestConsoleWrapper(consoleRunnerPath, dotnetPath, new ConsoleParameters() { LogFilePath = logFilePath }); vstestConsoleWrapper.StartSession(); diff --git a/test/Microsoft.TestPlatform.TestUtilities/MSTestInfo.cs b/test/Microsoft.TestPlatform.TestUtilities/MSTestInfo.cs index 55453ed196..132d06a8c5 100644 --- a/test/Microsoft.TestPlatform.TestUtilities/MSTestInfo.cs +++ b/test/Microsoft.TestPlatform.TestUtilities/MSTestInfo.cs @@ -29,6 +29,7 @@ public string UpdatePath(string path) if (Version == null) throw new InvalidOperationException($"Version was not correctly populated from TestPlatform.Dependencies.props, review that there is entry for MSTestFramework{VersionType}Version."); + // TODO: replacing in the result string is lame, but I am not going to fight 20 GetAssetFullPath method overloads right now return path.Replace($"{System.IO.Path.DirectorySeparatorChar}bin{System.IO.Path.DirectorySeparatorChar}", Path); } } diff --git a/test/Microsoft.TestPlatform.TestUtilities/VSTestConsoleInfo.cs b/test/Microsoft.TestPlatform.TestUtilities/VSTestConsoleInfo.cs new file mode 100644 index 0000000000..d88ad20375 --- /dev/null +++ b/test/Microsoft.TestPlatform.TestUtilities/VSTestConsoleInfo.cs @@ -0,0 +1,23 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +namespace Microsoft.TestPlatform.TestUtilities; + +public class VSTestConsoleInfo +{ + public VSTestConsoleInfo(string versionType, string? version, string path) + { + VersionType = versionType; + // Version can be null when we fail to find the respective propertin in TestPlatform.Dependencies.props + // when that happens we throw when we try to update the path. + Version = version; + Path = path; + } + + public string VersionType { get; } + public string? Version { get; } + public string Path { get; } + + public override string ToString() => $" vstest.console = {Version} [{VersionType}]"; +} + From 0fc39cf263e3147cf8e7fe51282d65b9b297cd1b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Jare=C5=A1?= Date: Fri, 11 Mar 2022 05:54:21 +0100 Subject: [PATCH 046/112] Second test --- .../TranslationLayerTests/DiscoverTests.cs | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/DiscoverTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/DiscoverTests.cs index 482af33303..a8452835a0 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/DiscoverTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/DiscoverTests.cs @@ -57,14 +57,17 @@ public void DiscoverTestsUsingDiscoveryEventHandler1(RunnerInfo runnerInfo, VSTe } [TestMethod] - [NetFullTargetFrameworkDataSource] - [NetCoreTargetFrameworkDataSource] - public void DiscoverTestsUsingDiscoveryEventHandler2AndTelemetryOptedOut(RunnerInfo runnerInfo) + [TranslationLayerCompatibilityDataSource()] + public void DiscoverTestsUsingDiscoveryEventHandler2AndTelemetryOptedOut(RunnerInfo runnerInfo, VSTestConsoleInfo vsTestConsoleInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - Setup(); + // Setup(); - _vstestConsoleWrapper.DiscoverTests( + _discoveryEventHandler = new DiscoveryEventHandler(); + _discoveryEventHandler2 = new DiscoveryEventHandler2(); + + var vstestConsoleWrapper = GetVsTestConsoleWrapper(TempDirectory, vsTestConsoleInfo); + vstestConsoleWrapper.DiscoverTests( GetTestAssemblies(), GetDefaultRunSettings(), new TestPlatformOptions() { CollectMetrics = false }, From aaecf7fddca17ff494ecf39a1c7610caf1067aa5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Jare=C5=A1?= Date: Fri, 11 Mar 2022 07:55:05 +0100 Subject: [PATCH 047/112] Remove unused field --- .../Extension/TranslationLayerCompatibilityDataSource.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/TranslationLayerCompatibilityDataSource.cs b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/TranslationLayerCompatibilityDataSource.cs index 2d1b637399..879fbe3830 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/TranslationLayerCompatibilityDataSource.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/TranslationLayerCompatibilityDataSource.cs @@ -18,7 +18,7 @@ public sealed class TranslationLayerCompatibilityDataSource : TestDataSource From 82cb607959539f87c618f7a260a1d4cab42e5e47 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Jare=C5=A1?= Date: Fri, 11 Mar 2022 14:39:29 +0100 Subject: [PATCH 048/112] Translation layer x vstest.console --- scripts/build.ps1 | 76 ++++----- .../ConsoleParameters.cs | 7 - .../PublicAPI/PublicAPI.Shipped.txt | 4 +- .../PublicAPI/net451/PublicAPI.Shipped.txt | 2 - .../VsTestConsoleProcessManager.cs | 3 +- src/SettingsMigrator/ConsoleParameters.cs | 93 +++++++++++ .../DebugTests.cs | 76 +++++++++ .../{ => Extension}/IsExternalInit.cs | 0 .../MSTestCompatibilityDataSource.cs | 3 +- .../Extension/NetCoreRunner.cs | 2 +- .../NetCoreTargetFrameworkDataSource.cs | 2 +- .../NetFullTargetFrameworkDataSource.cs | 4 +- .../Extension/RunnnerInfo.cs | 9 +- ...TranslationLayerCompatibilityDataSource.cs | 52 +++++- ...rosoft.TestPlatform.AcceptanceTests.csproj | 2 + .../CustomTestHostLauncher.cs | 54 ------- .../CustomTestHostTests.cs | 151 ++++++++++++++++-- .../TranslationLayerTests/DiscoverTests.cs | 2 +- .../EventHandler/RunEventHandler.cs | 2 +- .../TranslationLayerTests/RunTests.cs | 11 +- .../RunTestsWithFilterTests.cs | 23 ++- .../IntegrationTestBase.cs | 38 ++++- 22 files changed, 464 insertions(+), 152 deletions(-) create mode 100644 src/SettingsMigrator/ConsoleParameters.cs create mode 100644 test/Microsoft.TestPlatform.AcceptanceTests/DebugTests.cs rename test/Microsoft.TestPlatform.AcceptanceTests/{ => Extension}/IsExternalInit.cs (100%) delete mode 100644 test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/CustomTestHostLauncher.cs diff --git a/scripts/build.ps1 b/scripts/build.ps1 index 0b39b742d6..16160aab93 100644 --- a/scripts/build.ps1 +++ b/scripts/build.ps1 @@ -169,14 +169,14 @@ function Invoke-TestAssetsBuild try { Write-Log ".. .. Build: Source: $TPB_TestAssets_Solution -- add NuGet source" Invoke-Exe -IgnoreExitCode 1 $nugetExe -Arguments "sources add -Name ""locally-built-testplatform-packages"" -Source $env:TP_TESTARTIFACTS\packages\ -ConfigFile ""$nugetConfig""" - #Invoke-Exe $dotnetExe -Arguments "build $TPB_TestAssets_Solution --configuration $TPB_Configuration -v:minimal -p:CIBuild=$TPB_CIBuild -p:LocalizedBuild=$TPB_LocalizedBuild -bl:""$env:TP_OUT_DIR\log\$Configuration\TestAssets.binlog""" + Invoke-Exe $dotnetExe -Arguments "build $TPB_TestAssets_Solution --configuration $TPB_Configuration -v:minimal -p:CIBuild=$TPB_CIBuild -p:LocalizedBuild=$TPB_LocalizedBuild -bl:""$env:TP_OUT_DIR\log\$Configuration\TestAssets.binlog""" - # Compatibility matrix build + # Compatibility matrix build. $dependenciesPath = "$env:TP_ROOT_DIR\scripts\build\TestPlatform.Dependencies.props" $dependenciesXml = [xml](Get-Content -Raw -Encoding UTF8 $dependenciesPath) - # build with multiple versions of MSTest + # Build with multiple versions of MSTest. $projects = @( "$env:TP_ROOT_DIR\test\TestAssets\SimpleTestProject\SimpleTestProject.csproj" "$env:TP_ROOT_DIR\test\TestAssets\SimpleTestProject2\SimpleTestProject2.csproj" @@ -206,7 +206,7 @@ function Invoke-TestAssetsBuild } } - # restore previous versions of TestPlatform (for vstest.console.exe), and TestPlatform.CLI (for vstest.console.dll) + # Restore previous versions of TestPlatform (for vstest.console.exe), and TestPlatform.CLI (for vstest.console.dll). $versionProperties = @( "NETTestSdkVersion" "VSTestConsoleLatestPreviewVersion" @@ -257,7 +257,7 @@ function Invoke-TestAssetsBuild # the original one. if (-not (Test-Path -Path $cachePath) -or -not (Get-ChildItem $cachePath)) { Rename-Item "$packagePath\$package.$vsTestConsoleVersion" $cachePath - # nuget locks the locally copied package it seems. + # Nuget locks the locally copied package from time to time. Start-Sleep -Milliseconds 300 } if (Test-Path "$packagePath\$package.$vsTestConsoleVersion") { @@ -1296,42 +1296,42 @@ Get-ChildItem env: | Where-Object -FilterScript { $_.Name.StartsWith("TP_") } | Write-Log "Test platform build variables: " Get-Variable | Where-Object -FilterScript { $_.Name.StartsWith("TPB_") } | Format-Table -# if ($Force -or $Steps -contains "InstallDotnet") { -# Install-DotNetCli -# } - -# if ($Force -or $Steps -contains "Restore") { -# Clear-Package -# Restore-Package -# } - -# if ($Force -or $Steps -contains "UpdateLocalization") { -# Update-LocalizedResources -# } - -# if ($Force -or $Steps -contains "Build") { -# Invoke-Build -# } - -# if ($Force -or $Steps -contains "Publish") { -# Publish-Package -# Create-VsixPackage -# Create-NugetPackages -# } - -# if ($Force -or $Steps -contains "Publish" -or $Steps -contains "Manifest") { -# Generate-Manifest -PackageFolder $TPB_PackageOutDir -# if (Test-Path $TPB_SourceBuildPackageOutDir) -# { -# Generate-Manifest -PackageFolder $TPB_SourceBuildPackageOutDir -# } -# Copy-PackageIntoStaticDirectory -# } +if ($Force -or $Steps -contains "InstallDotnet") { + Install-DotNetCli +} + +if ($Force -or $Steps -contains "Restore") { + Clear-Package + Restore-Package +} + +if ($Force -or $Steps -contains "UpdateLocalization") { + Update-LocalizedResources +} + +if ($Force -or $Steps -contains "Build") { + Invoke-Build +} + +if ($Force -or $Steps -contains "Publish") { + Publish-Package + Create-VsixPackage + Create-NugetPackages +} + +if ($Force -or $Steps -contains "Publish" -or $Steps -contains "Manifest") { + Generate-Manifest -PackageFolder $TPB_PackageOutDir + if (Test-Path $TPB_SourceBuildPackageOutDir) + { + Generate-Manifest -PackageFolder $TPB_SourceBuildPackageOutDir + } + Copy-PackageIntoStaticDirectory +} if ($Force -or $Steps -contains "PrepareAcceptanceTests") { - #Publish-PatchedDotnet + Publish-PatchedDotnet Invoke-TestAssetsBuild - #Publish-Tests + Publish-Tests } if ($Script:ScriptFailed) { diff --git a/src/Microsoft.TestPlatform.VsTestConsole.TranslationLayer/ConsoleParameters.cs b/src/Microsoft.TestPlatform.VsTestConsole.TranslationLayer/ConsoleParameters.cs index a1abda01df..7386f07d06 100644 --- a/src/Microsoft.TestPlatform.VsTestConsole.TranslationLayer/ConsoleParameters.cs +++ b/src/Microsoft.TestPlatform.VsTestConsole.TranslationLayer/ConsoleParameters.cs @@ -1,9 +1,7 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT license. See LICENSE file in the project root for full license information. -#if NETFRAMEWORK using System.Collections.Generic; -#endif using System.Diagnostics; using System.IO; @@ -41,16 +39,11 @@ public ConsoleParameters(IFileHelper fileHelper) _fileHelper = fileHelper; } -#if NETFRAMEWORK - /// - /// TODO: Remove the #if when project is targeted to netstandard2.0 /// Environment variables to be set for the process /// public Dictionary EnvironmentVariables { get; set; } -#endif - /// /// Trace level for logs. /// diff --git a/src/Microsoft.TestPlatform.VsTestConsole.TranslationLayer/PublicAPI/PublicAPI.Shipped.txt b/src/Microsoft.TestPlatform.VsTestConsole.TranslationLayer/PublicAPI/PublicAPI.Shipped.txt index cb8161d464..e9c4b6ee04 100644 --- a/src/Microsoft.TestPlatform.VsTestConsole.TranslationLayer/PublicAPI/PublicAPI.Shipped.txt +++ b/src/Microsoft.TestPlatform.VsTestConsole.TranslationLayer/PublicAPI/PublicAPI.Shipped.txt @@ -196,4 +196,6 @@ static Microsoft.VisualStudio.TestPlatform.VsTestConsole.TranslationLayer.Resour static Microsoft.VisualStudio.TestPlatform.VsTestConsole.TranslationLayer.Resources.Resources.InvalidFilePath.get -> string static Microsoft.VisualStudio.TestPlatform.VsTestConsole.TranslationLayer.Resources.Resources.ResourceManager.get -> System.Resources.ResourceManager static Microsoft.VisualStudio.TestPlatform.VsTestConsole.TranslationLayer.Resources.Resources.VsTestProcessExitedAbnormally.get -> string -virtual Microsoft.TestPlatform.VsTestConsole.TranslationLayer.TestSession.Dispose(bool disposing) -> void \ No newline at end of file +virtual Microsoft.TestPlatform.VsTestConsole.TranslationLayer.TestSession.Dispose(bool disposing) -> void +Microsoft.TestPlatform.VsTestConsole.TranslationLayer.ConsoleParameters.EnvironmentVariables.get -> System.Collections.Generic.Dictionary +Microsoft.TestPlatform.VsTestConsole.TranslationLayer.ConsoleParameters.EnvironmentVariables.set -> void diff --git a/src/Microsoft.TestPlatform.VsTestConsole.TranslationLayer/PublicAPI/net451/PublicAPI.Shipped.txt b/src/Microsoft.TestPlatform.VsTestConsole.TranslationLayer/PublicAPI/net451/PublicAPI.Shipped.txt index 2b1c581fe8..e69de29bb2 100644 --- a/src/Microsoft.TestPlatform.VsTestConsole.TranslationLayer/PublicAPI/net451/PublicAPI.Shipped.txt +++ b/src/Microsoft.TestPlatform.VsTestConsole.TranslationLayer/PublicAPI/net451/PublicAPI.Shipped.txt @@ -1,2 +0,0 @@ -Microsoft.TestPlatform.VsTestConsole.TranslationLayer.ConsoleParameters.EnvironmentVariables.get -> System.Collections.Generic.Dictionary -Microsoft.TestPlatform.VsTestConsole.TranslationLayer.ConsoleParameters.EnvironmentVariables.set -> void \ No newline at end of file diff --git a/src/Microsoft.TestPlatform.VsTestConsole.TranslationLayer/VsTestConsoleProcessManager.cs b/src/Microsoft.TestPlatform.VsTestConsole.TranslationLayer/VsTestConsoleProcessManager.cs index 48b804836e..323ff2d95a 100644 --- a/src/Microsoft.TestPlatform.VsTestConsole.TranslationLayer/VsTestConsoleProcessManager.cs +++ b/src/Microsoft.TestPlatform.VsTestConsole.TranslationLayer/VsTestConsoleProcessManager.cs @@ -121,7 +121,6 @@ public void StartProcess(ConsoleParameters consoleParameters) EqtTrace.Verbose("VsTestCommandLineWrapper: Process Start Info {0} {1}", info.FileName, info.Arguments); -#if NETFRAMEWORK if (consoleParameters.EnvironmentVariables != null) { info.EnvironmentVariables.Clear(); @@ -133,7 +132,7 @@ public void StartProcess(ConsoleParameters consoleParameters) } } } -#endif + try { _process = Process.Start(info); diff --git a/src/SettingsMigrator/ConsoleParameters.cs b/src/SettingsMigrator/ConsoleParameters.cs new file mode 100644 index 0000000000..80470abdff --- /dev/null +++ b/src/SettingsMigrator/ConsoleParameters.cs @@ -0,0 +1,93 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +#if NETFRAMEWORK +using System.Collections.Generic; +#endif +using System.Collections.Generic; +using System.Diagnostics; +using System.IO; + +using Microsoft.VisualStudio.TestPlatform.CoreUtilities.Extensions; +using Microsoft.VisualStudio.TestPlatform.ObjectModel; +using Microsoft.VisualStudio.TestPlatform.Utilities.Helpers; +using Microsoft.VisualStudio.TestPlatform.Utilities.Helpers.Interfaces; + +#nullable disable + +namespace Microsoft.TestPlatform.VsTestConsole.TranslationLayer; + +/// +/// Class which defines additional specifiable parameters for vstest.console.exe +/// +public class ConsoleParameters +{ + internal static readonly ConsoleParameters Default = new(); + + private string _logFilePath; + private readonly IFileHelper _fileHelper; + + /// + /// Create instance of + /// + public ConsoleParameters() : this(new FileHelper()) + { } + + /// + /// Create instance of + /// + /// Object of type + public ConsoleParameters(IFileHelper fileHelper) + { + _fileHelper = fileHelper; + } + + /// + /// Environment variables to be set for the process + /// + public Dictionary EnvironmentVariables { get; set; } + + /// + /// Trace level for logs. + /// + public TraceLevel TraceLevel { get; set; } = TraceLevel.Verbose; + + /// + /// Full path for the log file + /// + public string LogFilePath + { + get + { + return _logFilePath; + } + + set + { + ValidateArg.NotNullOrEmpty(value, "LogFilePath"); + var directoryPath = Path.GetDirectoryName(value); + if (!string.IsNullOrEmpty(directoryPath) && !_fileHelper.DirectoryExists(directoryPath)) + { + Directory.CreateDirectory(directoryPath); + } + + // Ensure path is double quoted. if path has white space then it can create problem. + _logFilePath = value.AddDoubleQuote(); + } + } + + /// + /// Port Number for communication + /// vstest.console will need this port number to communicate with this component - translation layer + /// Currently Internal as we are not intentionally exposing this to consumers of translation layer + /// + internal int PortNumber { get; set; } + + /// + /// Parent Process ID of the process whose lifetime should dictate the life time of vstest.console.exe + /// vstest.console will need this process ID to know when the process exits. + /// If parent process dies/crashes without invoking EndSession, vstest.console should exit immediately + /// Currently Internal as we are not intentionally exposing this to consumers of translation layer + /// + internal int ParentProcessId { get; set; } +} diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/DebugTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/DebugTests.cs new file mode 100644 index 0000000000..57898375ff --- /dev/null +++ b/test/Microsoft.TestPlatform.AcceptanceTests/DebugTests.cs @@ -0,0 +1,76 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using System.Collections.Generic; +using System.Linq; + +using Microsoft.TestPlatform.TestUtilities; +using Microsoft.TestPlatform.VsTestConsole.TranslationLayer.Interfaces; +using Microsoft.VisualStudio.TestPlatform.ObjectModel; +using Microsoft.VisualStudio.TestTools.UnitTesting; + +#nullable disable + +namespace Microsoft.TestPlatform.AcceptanceTests.TranslationLayerTests; + +/// +/// The Run Tests using VsTestConsoleWrapper API's +/// +[TestClass] +public class DebugTests : AcceptanceTestBase +{ + private IVsTestConsoleWrapper _vstestConsoleWrapper; + + [TestCleanup] + public void Cleanup() + { + _vstestConsoleWrapper?.EndSession(); + } + + [TestMethod] + [TranslationLayerCompatibilityDataSource("net451", "net451", "LegacyStable")] + public void AttachDebugger(RunnerInfo runnerInfo, VSTestConsoleInfo vsTestConsoleInfo) + { + SetTestEnvironment(_testEnvironment, runnerInfo); + // Setup(); + + var runEventHandler = new AttachDebuggerRunEventHandler(attachDebuggerToProcessResponse: true); + _vstestConsoleWrapper = GetVsTestConsoleWrapper(TempDirectory, vsTestConsoleInfo); + _vstestConsoleWrapper.RunTests(GetTestAssemblies(), GetDefaultRunSettings(), runEventHandler); + + // Assert + Assert.AreEqual(6, runEventHandler.TestResults.Count); + Assert.AreEqual(2, runEventHandler.TestResults.Count(t => t.Outcome == TestOutcome.Passed)); + Assert.AreEqual(2, runEventHandler.TestResults.Count(t => t.Outcome == TestOutcome.Failed)); + Assert.AreEqual(2, runEventHandler.TestResults.Count(t => t.Outcome == TestOutcome.Skipped)); + } + + private IList GetTestAssemblies() + { + var testAssemblies = new List + { + GetAssetFullPath("SimpleTestProject.dll"), + GetAssetFullPath("SimpleTestProject2.dll") + }; + + return testAssemblies; + } + + class AttachDebuggerRunEventHandler : RunEventHandler + { + private readonly bool _attachDebuggerToProcessResult; + + public AttachDebuggerRunEventHandler(bool attachDebuggerToProcessResponse) + { + _attachDebuggerToProcessResult = attachDebuggerToProcessResponse; + } + + public int Pid { get; private set; } + + public override bool AttachDebuggerToProcess(int pid) + { + Pid = pid; + return _attachDebuggerToProcessResult; + } + } +} diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/IsExternalInit.cs b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/IsExternalInit.cs similarity index 100% rename from test/Microsoft.TestPlatform.AcceptanceTests/IsExternalInit.cs rename to test/Microsoft.TestPlatform.AcceptanceTests/Extension/IsExternalInit.cs diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/MSTestCompatibilityDataSource.cs b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/MSTestCompatibilityDataSource.cs index b491bda22e..a032a26649 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/MSTestCompatibilityDataSource.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/MSTestCompatibilityDataSource.cs @@ -2,7 +2,6 @@ // Licensed under the MIT license. See LICENSE file in the project root for full license information. using System; -using System.Collections.Generic; using System.Globalization; using System.IO; using System.Linq; @@ -54,7 +53,7 @@ public override void CreateData(MethodInfo methodInfo) { foreach (var msTestVersion in _msTestVersions) { - var runnerInfo = new RunnerInfo(AcceptanceTestBase.DEFAULT_RUNNER_NETFX, AcceptanceTestBase.DEFAULT_RUNNER_NETFX, inIsolation: null, + var runnerInfo = new RunnerInfo(AcceptanceTestBase.DEFAULT_RUNNER_NETFX, AcceptanceTestBase.DEFAULT_RUNNER_NETFX, InIsolationValue: null, DebugVSTestConsole, DebugTesthost, DebugDataCollector, NoDefaultBreakpoints); var msTestInfo = GetMSTestInfo(msTestVersion); // We run in the .NET Framework runner process, the runner and target framework must agree. diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/NetCoreRunner.cs b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/NetCoreRunner.cs index 86f2b05b33..1a3d855e29 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/NetCoreRunner.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/NetCoreRunner.cs @@ -47,7 +47,7 @@ public IEnumerable GetData(MethodInfo methodInfo) Func filter = tfm => isWindows || !tfm.StartsWith("net4"); foreach (var fmw in _targetFrameworks.Split(';').Where(filter)) { - dataRows.Add(new object[] { new RunnerInfo(IntegrationTestBase.CoreRunnerFramework, fmw, inIsolation: null, DebugVSTestConsole, DebugTesthost, DebugDataCollector, NoDefaultBreakpoints) }); + dataRows.Add(new object[] { new RunnerInfo(IntegrationTestBase.CoreRunnerFramework, fmw, InIsolationValue: null, DebugVSTestConsole, DebugTesthost, DebugDataCollector, NoDefaultBreakpoints) }); } return dataRows; diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/NetCoreTargetFrameworkDataSource.cs b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/NetCoreTargetFrameworkDataSource.cs index 786b477bbc..6788d56074 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/NetCoreTargetFrameworkDataSource.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/NetCoreTargetFrameworkDataSource.cs @@ -55,7 +55,7 @@ public NetCoreTargetFrameworkDataSource( private void AddRunnerDataRow(List dataRows, string runnerFramework, string targetFramework) { - var runnerInfo = new RunnerInfo(runnerFramework, targetFramework, inIsolation: null, DebugVSTestConsole, DebugTesthost, DebugDataCollector, NoDefaultBreakpoints); + var runnerInfo = new RunnerInfo(runnerFramework, targetFramework, InIsolationValue: null, DebugVSTestConsole, DebugTesthost, DebugDataCollector, NoDefaultBreakpoints); dataRows.Add(new object[] { runnerInfo }); } diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/NetFullTargetFrameworkDataSource.cs b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/NetFullTargetFrameworkDataSource.cs index a18eeb8271..c3b0af0e6c 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/NetFullTargetFrameworkDataSource.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/NetFullTargetFrameworkDataSource.cs @@ -53,7 +53,7 @@ public IEnumerable GetData(MethodInfo methodInfo) var isWindows = Environment.OSVersion.Platform.ToString().StartsWith("Win"); if (_useCoreRunner && isWindows) { - dataRows.Add(new object[] { new RunnerInfo(IntegrationTestBase.CoreRunnerFramework, AcceptanceTestBase.DesktopTargetFramework, inIsolation: null, DebugVSTestConsole, DebugTesthost, DebugDataCollector, NoDefaultBreakpoints) }); + dataRows.Add(new object[] { new RunnerInfo(IntegrationTestBase.CoreRunnerFramework, AcceptanceTestBase.DesktopTargetFramework, InIsolationValue: null, DebugVSTestConsole, DebugTesthost, DebugDataCollector, NoDefaultBreakpoints) }); } if (_useDesktopRunner && isWindows) @@ -65,7 +65,7 @@ public IEnumerable GetData(MethodInfo methodInfo) if (_inProcess) { - dataRows.Add(new object[] { new RunnerInfo(IntegrationTestBase.DesktopRunnerFramework, AcceptanceTestBase.DesktopTargetFramework, inIsolation: null, DebugVSTestConsole, DebugTesthost, DebugDataCollector, NoDefaultBreakpoints) }); + dataRows.Add(new object[] { new RunnerInfo(IntegrationTestBase.DesktopRunnerFramework, AcceptanceTestBase.DesktopTargetFramework, InIsolationValue: null, DebugVSTestConsole, DebugTesthost, DebugDataCollector, NoDefaultBreakpoints) }); } } diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/RunnnerInfo.cs b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/RunnnerInfo.cs index 7bbe7ce046..c6f67359da 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/RunnnerInfo.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/RunnnerInfo.cs @@ -11,9 +11,14 @@ namespace Microsoft.TestPlatform.AcceptanceTests; /// /// /// Supported value = /InIsolation. -public record RunnerInfo(string RunnerFramework, string TargetFramework, string InIsolationValue = "", - bool DebugVSTestConsole = false, bool DebugTestHost = false, bool DebugDataCollector = false, bool NoDefaultBreakpoints = true) +public record RunnerInfo(string RunnerFramework, string TargetFramework, string? InIsolationValue = "", + bool DebugVSTestConsole = false, bool DebugTesthost = false, bool DebugDataCollector = false, bool NoDefaultBreakpoints = true) { + /// + /// Is running via .NET "Core" vstest.console? + /// + public bool IsNetRunner => !IsNetFrameworkRunner; + /// /// Is running via .NET Framework vstest.console? /// diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/TranslationLayerCompatibilityDataSource.cs b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/TranslationLayerCompatibilityDataSource.cs index 879fbe3830..fc5149e0d4 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/TranslationLayerCompatibilityDataSource.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/TranslationLayerCompatibilityDataSource.cs @@ -2,6 +2,7 @@ // Licensed under the MIT license. See LICENSE file in the project root for full license information. using System; +using System.Collections.Generic; using System.Globalization; using System.IO; using System.Linq; @@ -10,6 +11,7 @@ using Microsoft.TestPlatform.TestUtilities; +using Semver; namespace Microsoft.TestPlatform.AcceptanceTests; @@ -28,12 +30,13 @@ public sealed class TranslationLayerCompatibilityDataSource : TestDataSource filter = tfm => isWindows || !tfm.StartsWith("net4"); - + // TODO: maybe we should throw if we don't end up generating any data + // because none of the versions match, or some other way to identify tests that will never run because they are very outdated. + // We probably don't have that need right now, because legacy version is 15.x.x, which is very old, and we are still keeping + // compatibility. foreach (var runner in _runnerFrameworks.Where(filter)) { foreach (var fmw in _targetFrameworks.Where(filter)) @@ -61,6 +89,12 @@ public override void CreateData(MethodInfo methodInfo) DebugVSTestConsole, DebugTesthost, DebugDataCollector, NoDefaultBreakpoints); var vsTestConsoleInfo = GetVSTestConsoleInfo(vstestConsoleVersion, runnerInfo); + if (beforeVersion != null && vsTestConsoleInfo.Version > beforeVersion) + continue; + + if (afterVersion != null && vsTestConsoleInfo.Version < afterVersion) + continue; + AddData(runnerInfo, vsTestConsoleInfo); } } @@ -138,3 +172,15 @@ private static XmlDocument GetDependenciesXml() return depsXml; } } + +public readonly record struct Feature(string Version, string Issue); + +public static class Features +{ + public const string ATTACH_DEBUGGER = nameof(ATTACH_DEBUGGER); + + public static Dictionary Table { get; } = new Dictionary + { + [ATTACH_DEBUGGER] = new(Version: "v16.7.0-preview-20200519-01", Issue: "https://github.com/microsoft/vstest/pull/2325") + }; +} diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/Microsoft.TestPlatform.AcceptanceTests.csproj b/test/Microsoft.TestPlatform.AcceptanceTests/Microsoft.TestPlatform.AcceptanceTests.csproj index c5a1f80019..c33244d042 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/Microsoft.TestPlatform.AcceptanceTests.csproj +++ b/test/Microsoft.TestPlatform.AcceptanceTests/Microsoft.TestPlatform.AcceptanceTests.csproj @@ -28,6 +28,7 @@ + all runtime; build; native; contentfiles; analyzers; buildtransitive @@ -35,6 +36,7 @@ + diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/CustomTestHostLauncher.cs b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/CustomTestHostLauncher.cs deleted file mode 100644 index e7fe325916..0000000000 --- a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/CustomTestHostLauncher.cs +++ /dev/null @@ -1,54 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT license. See LICENSE file in the project root for full license information. - -using System.Diagnostics; -using System.Threading; - -using Microsoft.VisualStudio.TestPlatform.ObjectModel; -using Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.Interfaces; - -#nullable disable - -namespace Microsoft.TestPlatform.AcceptanceTests.TranslationLayerTests; - -/// -/// The custom test host launcher. -/// -public class CustomTestHostLauncher : ITestHostLauncher2 -{ - public int ProcessId - { - get; - private set; - } - - /// - public bool IsDebug => true; - - public bool AttachDebuggerToProcess(int pid) => AttachDebuggerToProcess(pid, CancellationToken.None); - - public bool AttachDebuggerToProcess(int pid, CancellationToken cancellationToken) => true; - - /// - public int LaunchTestHost(TestProcessStartInfo defaultTestHostStartInfo) - { - return LaunchTestHost(defaultTestHostStartInfo, CancellationToken.None); - } - - /// - public int LaunchTestHost(TestProcessStartInfo defaultTestHostStartInfo, CancellationToken cancellationToken) - { - var processInfo = new ProcessStartInfo( - defaultTestHostStartInfo.FileName, - defaultTestHostStartInfo.Arguments) - { - WorkingDirectory = defaultTestHostStartInfo.WorkingDirectory - }; - processInfo.UseShellExecute = false; - - var process = new Process { StartInfo = processInfo }; - process.Start(); - - return process != null ? process.Id : -1; - } -} diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/CustomTestHostTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/CustomTestHostTests.cs index 829b2e7eda..646fda4dbf 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/CustomTestHostTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/CustomTestHostTests.cs @@ -2,10 +2,16 @@ // Licensed under the MIT license. See LICENSE file in the project root for full license information. using System.Collections.Generic; +using System.Diagnostics; using System.Linq; +using System.Threading; +using FluentAssertions; + +using Microsoft.TestPlatform.TestUtilities; using Microsoft.TestPlatform.VsTestConsole.TranslationLayer.Interfaces; using Microsoft.VisualStudio.TestPlatform.ObjectModel; +using Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.Interfaces; using Microsoft.VisualStudio.TestTools.UnitTesting; #nullable disable @@ -21,36 +27,99 @@ public class CustomTestHostTests : AcceptanceTestBase private IVsTestConsoleWrapper _vstestConsoleWrapper; private RunEventHandler _runEventHandler; - private void Setup() + [TestCleanup] + public void Cleanup() + { + _vstestConsoleWrapper?.EndSession(); + } + + [TestMethod] + [TranslationLayerCompatibilityDataSource(BeforeFeature = Features.ATTACH_DEBUGGER)] + public void RunTestsWithCustomTestHostLauncherLaunchesTheProcessUsingTheProvidedLauncher(RunnerInfo runnerInfo, VSTestConsoleInfo vsTestConsoleInfo) { - _vstestConsoleWrapper = GetVsTestConsoleWrapper(out _); + SetTestEnvironment(_testEnvironment, runnerInfo); + _vstestConsoleWrapper = GetVsTestConsoleWrapper(TempDirectory, vsTestConsoleInfo); _runEventHandler = new RunEventHandler(); + + var customTestHostLauncher = new TestHostLauncherV1(); + _vstestConsoleWrapper.RunTestsWithCustomTestHost(GetTestAssemblies(), GetDefaultRunSettings(), _runEventHandler, customTestHostLauncher); + + // Assert + customTestHostLauncher.Should().BeAssignableTo(); + customTestHostLauncher.LaunchProcessProcessId.Should().NotBeNull("we should launch some real process and save the pid of it"); + + _runEventHandler.TestResults.Should().HaveCount(6); + _runEventHandler.TestResults.Count(t => t.Outcome == TestOutcome.Passed).Should().Be(2); + _runEventHandler.TestResults.Count(t => t.Outcome == TestOutcome.Failed).Should().Be(2); + _runEventHandler.TestResults.Count(t => t.Outcome == TestOutcome.Skipped).Should().Be(2); } - [TestCleanup] - public void Cleanup() + [TestMethod] + [TranslationLayerCompatibilityDataSource(BeforeFeature = Features.ATTACH_DEBUGGER)] + public void RunTestsWithCustomTestHostLauncherLaunchesTheProcessUsingTheProvidedLauncherWhenITestHostLauncher2IsProvided(RunnerInfo runnerInfo, VSTestConsoleInfo vsTestConsoleInfo) { - _vstestConsoleWrapper?.EndSession(); + SetTestEnvironment(_testEnvironment, runnerInfo); + _vstestConsoleWrapper = GetVsTestConsoleWrapper(TempDirectory, vsTestConsoleInfo); + _runEventHandler = new RunEventHandler(); + + var customTestHostLauncher = new TestHostLauncherV2(); + _vstestConsoleWrapper.RunTestsWithCustomTestHost(GetTestAssemblies(), GetDefaultRunSettings(), _runEventHandler, customTestHostLauncher); + + // Assert + customTestHostLauncher.Should().BeAssignableTo(); + customTestHostLauncher.LaunchProcessProcessId.Should().NotBeNull("we should launch some real process and save the pid of it"); + customTestHostLauncher.AttachDebuggerProcessId.Should().BeNull("we should not be asked to attach to a debugger, that flow is not used when vstest.console does not support it yet, even when it is given ITestHostLauncher2"); + + _runEventHandler.TestResults.Should().HaveCount(6); + _runEventHandler.TestResults.Count(t => t.Outcome == TestOutcome.Passed).Should().Be(2); + _runEventHandler.TestResults.Count(t => t.Outcome == TestOutcome.Failed).Should().Be(2); + _runEventHandler.TestResults.Count(t => t.Outcome == TestOutcome.Skipped).Should().Be(2); } + [TestMethod] + [TranslationLayerCompatibilityDataSource(AfterFeature = Features.ATTACH_DEBUGGER)] + public void RunTestsWithCustomTestHostLauncherAttachesToDebuggerUsingTheProvidedLauncher(RunnerInfo runnerInfo, VSTestConsoleInfo vsTestConsoleInfo) + { + SetTestEnvironment(_testEnvironment, runnerInfo); + _vstestConsoleWrapper = GetVsTestConsoleWrapper(TempDirectory, vsTestConsoleInfo); + _runEventHandler = new RunEventHandler(); + + var customTestHostLauncher = new TestHostLauncherV2(); + _vstestConsoleWrapper.RunTestsWithCustomTestHost(GetTestAssemblies(), GetDefaultRunSettings(), _runEventHandler, customTestHostLauncher); + + // Assert + customTestHostLauncher.Should().BeAssignableTo(); + customTestHostLauncher.AttachDebuggerProcessId.Should().NotBeNull("we should be asked to attach a debugger to some process and save the pid of the process"); + customTestHostLauncher.LaunchProcessProcessId.Should().BeNull("we should not be asked to launch some real process, that flow is not used when vstest.console supports it and is given ITestHostLauncher2"); + + _runEventHandler.TestResults.Should().HaveCount(6); + _runEventHandler.TestResults.Count(t => t.Outcome == TestOutcome.Passed).Should().Be(2); + _runEventHandler.TestResults.Count(t => t.Outcome == TestOutcome.Failed).Should().Be(2); + _runEventHandler.TestResults.Count(t => t.Outcome == TestOutcome.Skipped).Should().Be(2); + } [TestMethod] - [NetFullTargetFrameworkDataSource] - [NetCoreTargetFrameworkDataSource] - public void RunTestsWithCustomTestHostLaunch(RunnerInfo runnerInfo) + [Ignore("This is not working. The compatibility code only checks the protocol version (in handler), which is dictated by testhost. " + + "It sees 6 but does not realize that the provided CustomTesthostLauncher is not supporting the new feature, it ends up calling back to EditoAttachDebugger" + + "in translation layer, and that just silently skips the call.")] + [TranslationLayerCompatibilityDataSource("net451", "net451", "Latest", AfterFeature = Features.ATTACH_DEBUGGER, DebugVSTestConsole = true, DebugTesthost=true)] + public void RunTestsWithCustomTestHostLauncherUsesLaunchWhenGivenAnOutdatedITestHostLauncher(RunnerInfo runnerInfo, VSTestConsoleInfo vsTestConsoleInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - Setup(); + _vstestConsoleWrapper = GetVsTestConsoleWrapper(TempDirectory, vsTestConsoleInfo); + _runEventHandler = new RunEventHandler(); - var customTestHostLauncher = new CustomTestHostLauncher(); + var customTestHostLauncher = new TestHostLauncherV1(); _vstestConsoleWrapper.RunTestsWithCustomTestHost(GetTestAssemblies(), GetDefaultRunSettings(), _runEventHandler, customTestHostLauncher); // Assert - Assert.AreEqual(6, _runEventHandler.TestResults.Count); - Assert.IsTrue(customTestHostLauncher.ProcessId != -1); - Assert.AreEqual(2, _runEventHandler.TestResults.Count(t => t.Outcome == TestOutcome.Passed)); - Assert.AreEqual(2, _runEventHandler.TestResults.Count(t => t.Outcome == TestOutcome.Failed)); - Assert.AreEqual(2, _runEventHandler.TestResults.Count(t => t.Outcome == TestOutcome.Skipped)); + customTestHostLauncher.Should().NotBeAssignableTo(); + customTestHostLauncher.LaunchProcessProcessId.Should().NotBeNull("we should launch some real process and save the pid of it"); + + _runEventHandler.TestResults.Should().HaveCount(6); + _runEventHandler.TestResults.Count(t => t.Outcome == TestOutcome.Passed).Should().Be(2); + _runEventHandler.TestResults.Count(t => t.Outcome == TestOutcome.Failed).Should().Be(2); + _runEventHandler.TestResults.Count(t => t.Outcome == TestOutcome.Skipped).Should().Be(2); } private IList GetTestAssemblies() @@ -63,4 +132,56 @@ private IList GetTestAssemblies() return testAssemblies; } + + /// + /// The custom test host launcher implementing ITestHostLauncher. + /// + public class TestHostLauncherV1 : ITestHostLauncher + { + public int? LaunchProcessProcessId { get; private set; } + + /// + public bool IsDebug => true; + + /// + public int LaunchTestHost(TestProcessStartInfo defaultTestHostStartInfo) + { + return LaunchTestHost(defaultTestHostStartInfo, CancellationToken.None); + } + + /// + public int LaunchTestHost(TestProcessStartInfo defaultTestHostStartInfo, CancellationToken cancellationToken) + { + var processInfo = new ProcessStartInfo( + defaultTestHostStartInfo.FileName, + defaultTestHostStartInfo.Arguments) + { + WorkingDirectory = defaultTestHostStartInfo.WorkingDirectory + }; + processInfo.UseShellExecute = false; + + var process = new Process { StartInfo = processInfo }; + process.Start(); + + LaunchProcessProcessId = process?.Id; + return LaunchProcessProcessId ?? -1; + } + } + + /// + /// The custom test host launcher implementing ITestHostLauncher2. + /// + public class TestHostLauncherV2 : TestHostLauncherV1, ITestHostLauncher2 + { + + public int? AttachDebuggerProcessId { get; private set; } + + public bool AttachDebuggerToProcess(int pid) => AttachDebuggerToProcess(pid, CancellationToken.None); + + public bool AttachDebuggerToProcess(int pid, CancellationToken cancellationToken) + { + AttachDebuggerProcessId = pid; + return true; + } + } } diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/DiscoverTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/DiscoverTests.cs index a8452835a0..3fd97f543e 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/DiscoverTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/DiscoverTests.cs @@ -40,7 +40,7 @@ public void Cleanup() } [TestMethod] - [TranslationLayerCompatibilityDataSource()] + [TranslationLayerCompatibilityDataSource] public void DiscoverTestsUsingDiscoveryEventHandler1(RunnerInfo runnerInfo, VSTestConsoleInfo vsTestConsoleInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/EventHandler/RunEventHandler.cs b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/EventHandler/RunEventHandler.cs index b02ecf0730..848dff8996 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/EventHandler/RunEventHandler.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/EventHandler/RunEventHandler.cs @@ -117,7 +117,7 @@ public int LaunchProcessWithDebuggerAttached(TestProcessStartInfo testProcessSta return -1; } - public bool AttachDebuggerToProcess(int pid) + public virtual bool AttachDebuggerToProcess(int pid) { // No op return true; diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/RunTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/RunTests.cs index f8b15f77c9..4c4197078e 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/RunTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/RunTests.cs @@ -40,14 +40,15 @@ public void Cleanup() } [TestMethod] - [NetFullTargetFrameworkDataSource] - [NetCoreTargetFrameworkDataSource] - public void RunAllTests(RunnerInfo runnerInfo) + [TranslationLayerCompatibilityDataSource] + public void RunAllTests(RunnerInfo runnerInfo, VSTestConsoleInfo vsTestConsoleInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - Setup(); + // Setup(); - _vstestConsoleWrapper.RunTests(GetTestAssemblies(), GetDefaultRunSettings(), _runEventHandler); + var vstestConsoleWrapper = GetVsTestConsoleWrapper(TempDirectory, vsTestConsoleInfo); + _runEventHandler = new RunEventHandler(); + vstestConsoleWrapper.RunTests(GetTestAssemblies(), GetDefaultRunSettings(), _runEventHandler); // Assert Assert.AreEqual(6, _runEventHandler.TestResults.Count); diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/RunTestsWithFilterTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/RunTestsWithFilterTests.cs index 694f23409f..010e916d5b 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/RunTestsWithFilterTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/RunTestsWithFilterTests.cs @@ -4,6 +4,7 @@ using System.Collections.Generic; using System.Linq; +using Microsoft.TestPlatform.TestUtilities; using Microsoft.TestPlatform.VsTestConsole.TranslationLayer.Interfaces; using Microsoft.VisualStudio.TestPlatform.ObjectModel; using Microsoft.VisualStudio.TestPlatform.ObjectModel.Client; @@ -35,19 +36,18 @@ public void Cleanup() } [TestMethod] - [NetFullTargetFrameworkDataSource] - [NetCoreTargetFrameworkDataSource] - public void RunTestsWithTestCaseFilter(RunnerInfo runnerInfo) + [TranslationLayerCompatibilityDataSource] + public void RunTestsWithTestCaseFilter(RunnerInfo runnerInfo, VSTestConsoleInfo vsTestConsoleInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - Setup(); + // Setup(); - var sources = new List - { - GetAssetFullPath("SimpleTestProject.dll") - }; + _runEventHandler = new RunEventHandler(); - _vstestConsoleWrapper.RunTests( + var vstestConsoleWrapper = GetVsTestConsoleWrapper(TempDirectory, vsTestConsoleInfo); + var sources = new List { GetAssetFullPath("SimpleTestProject.dll") }; + + vstestConsoleWrapper.RunTests( sources, GetDefaultRunSettings(), new TestPlatformOptions() { TestCaseFilter = "FullyQualifiedName=SampleUnitTestProject.UnitTest1.PassingTest" }, @@ -66,10 +66,7 @@ public void RunTestsWithFastFilter(RunnerInfo runnerInfo) SetTestEnvironment(_testEnvironment, runnerInfo); Setup(); - var sources = new List - { - GetAssetFullPath("SimpleTestProject.dll") - }; + var sources = new List { GetAssetFullPath("SimpleTestProject.dll") }; _vstestConsoleWrapper.RunTests( sources, diff --git a/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestBase.cs b/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestBase.cs index 1399274b77..1b309be65c 100644 --- a/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestBase.cs +++ b/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestBase.cs @@ -2,6 +2,7 @@ // Licensed under the MIT license. See LICENSE file in the project root for full license information. using System; +using System.Collections; using System.Collections.Generic; using System.Diagnostics; using System.IO; @@ -605,11 +606,44 @@ public IVsTestConsoleWrapper GetVsTestConsoleWrapper(TempDirectory logFileDir, V if (!File.Exists(consoleRunnerPath)) { - throw new FileNotFoundException($"File '{consoleRunnerPath}' was not found."); + throw new FileNotFoundException($"File '{consoleRunnerPath}' was not found.") ; } Console.WriteLine($"Console runner path: {consoleRunnerPath}"); - var vstestConsoleWrapper = new VsTestConsoleWrapper(consoleRunnerPath, dotnetPath, new ConsoleParameters() { LogFilePath = logFilePath }); + + VsTestConsoleWrapper vstestConsoleWrapper; + if (_testEnvironment.DebugVSTestConsole || _testEnvironment.DebugTesthost || _testEnvironment.DebugDataCollector) + { + var environmentVariables = new Dictionary(); + Environment.GetEnvironmentVariables().OfType().ToList().ForEach(e=> environmentVariables.Add(e.Key.ToString(), e.Value.ToString())); + + if (_testEnvironment.DebugVSTestConsole) + { + environmentVariables.Add("VSTEST_RUNNER_DEBUG_ATTACHVS", "1"); + } + + if (_testEnvironment.DebugTesthost) + { + environmentVariables.Add("VSTEST_HOST_DEBUG_ATTACHVS", "1"); + } + + if (_testEnvironment.DebugDataCollector) + { + environmentVariables.Add("VSTEST_DATACOLLECTOR_DEBUG_ATTACHVS", "1"); + } + + if (_testEnvironment.NoDefaultBreakpoints) + { + environmentVariables.Add("VSTEST_DEBUG_NOBP", "1"); + } + + // This clears all variables + vstestConsoleWrapper = new VsTestConsoleWrapper(consoleRunnerPath, dotnetPath, new ConsoleParameters() { LogFilePath = logFilePath, EnvironmentVariables = environmentVariables }); + } + else + { + vstestConsoleWrapper = new VsTestConsoleWrapper(consoleRunnerPath, dotnetPath, new ConsoleParameters() { LogFilePath = logFilePath }); + } vstestConsoleWrapper.StartSession(); return vstestConsoleWrapper; From 07cdf1c87f0a4c247a476b5b5b8f2b36bbad59b2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Jare=C5=A1?= Date: Mon, 14 Mar 2022 13:37:19 +0100 Subject: [PATCH 049/112] Fixup console run --- scripts/build.ps1 | 171 ++++++++++-------- scripts/build/TestPlatform.Dependencies.props | 2 +- .../AcceptanceTestBase.cs | 6 +- .../DebugTests.cs | 76 -------- .../ExecutionTests.cs | 14 ++ .../MSTestCompatibilityDataSource.cs | 3 +- .../Extension/TestDataSource.cs | 31 ++++ .../TesthostCompatibilityDataSource.cs | 139 ++++++++++++++ ...TranslationLayerCompatibilityDataSource.cs | 12 +- .../CodeCoverageTests.cs | 6 +- .../CustomTestHostTests.cs | 16 +- .../DifferentTestFrameworkSimpleTests.cs | 2 +- .../TranslationLayerTests/DiscoverTests.cs | 10 +- .../LiveUnitTestingTests.cs | 2 +- .../TranslationLayerTests/RunSelectedTests.cs | 2 +- .../TranslationLayerTests/RunTests.cs | 25 ++- ...RunTestsWithDifferentConfigurationTests.cs | 4 +- .../RunTestsWithFilterTests.cs | 6 +- .../DllInfo.cs | 39 ++++ .../IntegrationTestBase.cs | 62 +++---- .../IntegrationTestEnvironment.cs | 1 + .../MSTestInfo.cs | 27 +-- .../TesthostInfo.cs | 12 ++ 23 files changed, 412 insertions(+), 256 deletions(-) delete mode 100644 test/Microsoft.TestPlatform.AcceptanceTests/DebugTests.cs create mode 100644 test/Microsoft.TestPlatform.AcceptanceTests/Extension/TesthostCompatibilityDataSource.cs create mode 100644 test/Microsoft.TestPlatform.TestUtilities/DllInfo.cs create mode 100644 test/Microsoft.TestPlatform.TestUtilities/TesthostInfo.cs diff --git a/scripts/build.ps1 b/scripts/build.ps1 index 16160aab93..2a27c513e4 100644 --- a/scripts/build.ps1 +++ b/scripts/build.ps1 @@ -175,40 +175,10 @@ function Invoke-TestAssetsBuild $dependenciesPath = "$env:TP_ROOT_DIR\scripts\build\TestPlatform.Dependencies.props" $dependenciesXml = [xml](Get-Content -Raw -Encoding UTF8 $dependenciesPath) - - # Build with multiple versions of MSTest. - $projects = @( - "$env:TP_ROOT_DIR\test\TestAssets\SimpleTestProject\SimpleTestProject.csproj" - "$env:TP_ROOT_DIR\test\TestAssets\SimpleTestProject2\SimpleTestProject2.csproj" - ) - - $versionProperties = @( - "MSTestFrameworkLatestPreviewVersion" - "MSTestFrameworkLatestStableVersion" - "MSTestFrameworkRecentStableVersion" - "MSTestFrameworkMostDownloadedVersion" - "MSTestFrameworkPreviousStableVersion" - "MSTestFrameworkLegacyStableVersion" - ) - - foreach ($project in $projects) { - foreach ($propertyName in $versionProperties) { - $mstestVersion = $dependenciesXml.Project.PropertyGroup.$propertyName - - if (-not $mstestVersion) - { - throw "MSTestVersion for $propertyName is empty." - } - - $dirVersion = $mstestVersion -replace "\[|\]" - $dirPropertyName = $propertyName -replace "Framework" -replace "Version" - Invoke-Exe $dotnetExe -Arguments "build $project --configuration $TPB_Configuration -v:minimal -p:CIBuild=$TPB_CIBuild -p:LocalizedBuild=$TPB_LocalizedBuild -p:MSTestFrameworkVersion=$mstestVersion -p:MSTestAdapterVersion=$mstestVersion -p:BaseOutputPath=""bin\$dirPropertyName-$dirVersion\\""" - } - } - # Restore previous versions of TestPlatform (for vstest.console.exe), and TestPlatform.CLI (for vstest.console.dll). - $versionProperties = @( - "NETTestSdkVersion" + # These properties are coming from TestPlatform.Dependencies.props. + $vstestConsoleVersionProperties = @( + "VSTestConsoleLatestVersion" "VSTestConsoleLatestPreviewVersion" "VSTestConsoleLatestStableVersion" "VSTestConsoleRecentStableVersion" @@ -217,7 +187,7 @@ function Invoke-TestAssetsBuild "VSTestConsoleLegacyStableVersion" ) - foreach ($propertyName in $versionProperties) { + foreach ($propertyName in $vstestConsoleVersionProperties) { if ("VSTestConsoleLatestVersion" -eq $propertyName) { # NETTestSdkVersion has the version of the locally built package. $vsTestConsoleVersion = $dependenciesXml.Project.PropertyGroup."NETTestSdkVersion" @@ -266,6 +236,59 @@ function Invoke-TestAssetsBuild } } + + # Build with multiple versions of MSTest. + $projects = @( + "$env:TP_ROOT_DIR\test\TestAssets\SimpleTestProject\SimpleTestProject.csproj" + "$env:TP_ROOT_DIR\test\TestAssets\SimpleTestProject2\SimpleTestProject2.csproj" + ) + + $msTestVersionProperties = @( + "MSTestFrameworkLatestPreviewVersion" + "MSTestFrameworkLatestStableVersion" + "MSTestFrameworkRecentStableVersion" + "MSTestFrameworkMostDownloadedVersion" + "MSTestFrameworkPreviousStableVersion" + "MSTestFrameworkLegacyStableVersion" + ) + + foreach ($project in $projects) { + foreach ($propertyName in $msTestVersionProperties) { + $mstestVersion = $dependenciesXml.Project.PropertyGroup.$propertyName + + if (-not $mstestVersion) + { + throw "MSTestVersion for $propertyName is empty." + } + + $dirMSTestVersion = $mstestVersion -replace "\[|\]" + $dirMSTestPropertyName = $propertyName -replace "Framework" -replace "Version" + Invoke-Exe $dotnetExe -Arguments "build $project --configuration $TPB_Configuration -v:minimal -p:CIBuild=$TPB_CIBuild -p:LocalizedBuild=$TPB_LocalizedBuild -p:MSTestFrameworkVersion=$mstestVersion -p:MSTestAdapterVersion=$mstestVersion -p:BaseOutputPath=""bin\$dirMSTestPropertyName-$dirMSTestVersion\\""" + } + } + + foreach ($project in $projects) { + # We use the same version properties for NET.Test.Sdk as for VSTestConsole, for now. + foreach ($propertyName in $vstestConsoleVersionProperties) { + if ("VSTestConsoleLatestVersion" -eq $propertyName) { + # NETTestSdkVersion has the version of the locally built package. + $netTestSdkVersion = $dependenciesXml.Project.PropertyGroup."NETTestSdkVersion" + } + else { + $netTestSdkVersion = $dependenciesXml.Project.PropertyGroup.$propertyName + } + + if (-not $netTestSdkVersion) + { + throw "NetTestSdkVersion for $propertyName is empty." + } + + $dirNetTestSdkVersion = $netTestSdkVersion -replace "\[|\]" + $dirNetTestSdkPropertyName = $propertyName -replace "Framework" -replace "Version" -replace "VSTestConsole", "NETTestSdk" + Invoke-Exe $dotnetExe -Arguments "build $project --configuration $TPB_Configuration -v:minimal -p:CIBuild=$TPB_CIBuild -p:LocalizedBuild=$TPB_LocalizedBuild -p:NETTestSdkVersion=$netTestSdkVersion -p:BaseOutputPath=""bin\$dirNetTestSdkPropertyName-$dirNetTestSdkVersion\\""" + } + } + # end } finally { @@ -1289,49 +1312,49 @@ if ($ProjectNamePatterns.Count -ne 0) } # Execute build -$timer = Start-Timer -Write-Log "Build started: args = '$args'" -Write-Log "Test platform environment variables: " -Get-ChildItem env: | Where-Object -FilterScript { $_.Name.StartsWith("TP_") } | Format-Table -Write-Log "Test platform build variables: " -Get-Variable | Where-Object -FilterScript { $_.Name.StartsWith("TPB_") } | Format-Table - -if ($Force -or $Steps -contains "InstallDotnet") { - Install-DotNetCli -} - -if ($Force -or $Steps -contains "Restore") { - Clear-Package - Restore-Package -} - -if ($Force -or $Steps -contains "UpdateLocalization") { - Update-LocalizedResources -} - -if ($Force -or $Steps -contains "Build") { - Invoke-Build -} - -if ($Force -or $Steps -contains "Publish") { - Publish-Package - Create-VsixPackage - Create-NugetPackages -} - -if ($Force -or $Steps -contains "Publish" -or $Steps -contains "Manifest") { - Generate-Manifest -PackageFolder $TPB_PackageOutDir - if (Test-Path $TPB_SourceBuildPackageOutDir) - { - Generate-Manifest -PackageFolder $TPB_SourceBuildPackageOutDir - } - Copy-PackageIntoStaticDirectory -} +# $timer = Start-Timer +# Write-Log "Build started: args = '$args'" +# Write-Log "Test platform environment variables: " +# Get-ChildItem env: | Where-Object -FilterScript { $_.Name.StartsWith("TP_") } | Format-Table +# Write-Log "Test platform build variables: " +# Get-Variable | Where-Object -FilterScript { $_.Name.StartsWith("TPB_") } | Format-Table + +# if ($Force -or $Steps -contains "InstallDotnet") { +# Install-DotNetCli +# } + +# if ($Force -or $Steps -contains "Restore") { +# Clear-Package +# Restore-Package +# } + +# if ($Force -or $Steps -contains "UpdateLocalization") { +# Update-LocalizedResources +# } + +# if ($Force -or $Steps -contains "Build") { +# Invoke-Build +# } + +# if ($Force -or $Steps -contains "Publish") { +# Publish-Package +# Create-VsixPackage +# Create-NugetPackages +# } + +# if ($Force -or $Steps -contains "Publish" -or $Steps -contains "Manifest") { +# Generate-Manifest -PackageFolder $TPB_PackageOutDir +# if (Test-Path $TPB_SourceBuildPackageOutDir) +# { +# Generate-Manifest -PackageFolder $TPB_SourceBuildPackageOutDir +# } +# Copy-PackageIntoStaticDirectory +# } if ($Force -or $Steps -contains "PrepareAcceptanceTests") { - Publish-PatchedDotnet + #Publish-PatchedDotnet Invoke-TestAssetsBuild - Publish-Tests + #Publish-Tests } if ($Script:ScriptFailed) { diff --git a/scripts/build/TestPlatform.Dependencies.props b/scripts/build/TestPlatform.Dependencies.props index 30969e3253..03fd5958bd 100644 --- a/scripts/build/TestPlatform.Dependencies.props +++ b/scripts/build/TestPlatform.Dependencies.props @@ -46,7 +46,7 @@ See Invoke-TestAssetsBuild in scripts/build.ps1. Exact versions are used to avoid Nuget substituting them by closest match, if we make a typo. These versions need to be "statically" readable because we read this file as xml in our build and tests. --> - 17.2.0-preview-20220131-20 + [17.2.0-preview-20220131-20] [17.1.0] [17.0.0] [16.9.4] diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/AcceptanceTestBase.cs b/test/Microsoft.TestPlatform.AcceptanceTests/AcceptanceTestBase.cs index 09e2d9fa4c..48e1d98556 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/AcceptanceTestBase.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/AcceptanceTestBase.cs @@ -68,8 +68,12 @@ public static string And(string left, string right) protected string FrameworkArgValue => DeriveFrameworkArgValue(_testEnvironment); - protected static void SetTestEnvironment(IntegrationTestEnvironment testEnvironment, RunnerInfo runnerInfo) + protected static void SetTestEnvironment(IntegrationTestEnvironment testEnvironment, RunnerInfo runnerInfo, VSTestConsoleInfo vsTestConsoleInfo = null) { + if (vsTestConsoleInfo != null) + { + testEnvironment.VSTestConsolePath = vsTestConsoleInfo.Path; + } testEnvironment.RunnerFramework = runnerInfo.RunnerFramework; testEnvironment.TargetFramework = runnerInfo.TargetFramework; testEnvironment.InIsolationValue = runnerInfo.InIsolationValue; diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/DebugTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/DebugTests.cs deleted file mode 100644 index 57898375ff..0000000000 --- a/test/Microsoft.TestPlatform.AcceptanceTests/DebugTests.cs +++ /dev/null @@ -1,76 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT license. See LICENSE file in the project root for full license information. - -using System.Collections.Generic; -using System.Linq; - -using Microsoft.TestPlatform.TestUtilities; -using Microsoft.TestPlatform.VsTestConsole.TranslationLayer.Interfaces; -using Microsoft.VisualStudio.TestPlatform.ObjectModel; -using Microsoft.VisualStudio.TestTools.UnitTesting; - -#nullable disable - -namespace Microsoft.TestPlatform.AcceptanceTests.TranslationLayerTests; - -/// -/// The Run Tests using VsTestConsoleWrapper API's -/// -[TestClass] -public class DebugTests : AcceptanceTestBase -{ - private IVsTestConsoleWrapper _vstestConsoleWrapper; - - [TestCleanup] - public void Cleanup() - { - _vstestConsoleWrapper?.EndSession(); - } - - [TestMethod] - [TranslationLayerCompatibilityDataSource("net451", "net451", "LegacyStable")] - public void AttachDebugger(RunnerInfo runnerInfo, VSTestConsoleInfo vsTestConsoleInfo) - { - SetTestEnvironment(_testEnvironment, runnerInfo); - // Setup(); - - var runEventHandler = new AttachDebuggerRunEventHandler(attachDebuggerToProcessResponse: true); - _vstestConsoleWrapper = GetVsTestConsoleWrapper(TempDirectory, vsTestConsoleInfo); - _vstestConsoleWrapper.RunTests(GetTestAssemblies(), GetDefaultRunSettings(), runEventHandler); - - // Assert - Assert.AreEqual(6, runEventHandler.TestResults.Count); - Assert.AreEqual(2, runEventHandler.TestResults.Count(t => t.Outcome == TestOutcome.Passed)); - Assert.AreEqual(2, runEventHandler.TestResults.Count(t => t.Outcome == TestOutcome.Failed)); - Assert.AreEqual(2, runEventHandler.TestResults.Count(t => t.Outcome == TestOutcome.Skipped)); - } - - private IList GetTestAssemblies() - { - var testAssemblies = new List - { - GetAssetFullPath("SimpleTestProject.dll"), - GetAssetFullPath("SimpleTestProject2.dll") - }; - - return testAssemblies; - } - - class AttachDebuggerRunEventHandler : RunEventHandler - { - private readonly bool _attachDebuggerToProcessResult; - - public AttachDebuggerRunEventHandler(bool attachDebuggerToProcessResponse) - { - _attachDebuggerToProcessResult = attachDebuggerToProcessResponse; - } - - public int Pid { get; private set; } - - public override bool AttachDebuggerToProcess(int pid) - { - Pid = pid; - return _attachDebuggerToProcessResult; - } - } -} diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/ExecutionTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/ExecutionTests.cs index 9a7f2b275e..89ad03fdaa 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/ExecutionTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/ExecutionTests.cs @@ -32,6 +32,20 @@ public void RunMultipleTestAssemblies(RunnerInfo runnerInfo, MSTestInfo msTestIn ExitCodeEquals(1); // failing tests } + [TestMethod] + [TesthostCompatibilityDataSource] + public void RunMultipleMSTestAssembliesOnVstestConsoleAndTesthostCombinations(RunnerInfo runnerInfo, VSTestConsoleInfo vsTestConsoleInfo, TesthostInfo testhostInfo) + { + SetTestEnvironment(_testEnvironment, runnerInfo, vsTestConsoleInfo); + + var assemblyPaths = BuildMultipleAssemblyPath(testhostInfo, "SimpleTestProject.dll", "SimpleTestProject2.dll").Trim('\"'); + + InvokeVsTestForExecution(assemblyPaths, testAdapterPath: null, FrameworkArgValue, string.Empty); + + ValidateSummaryStatus(2, 2, 2); + ExitCodeEquals(1); // failing tests + } + // TODO: This one mixes different frameworks, I can make it work, but it is worth it? We are going to test // the two respective versions together (e.g. latest xunit and latest mstest), but does using two different test diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/MSTestCompatibilityDataSource.cs b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/MSTestCompatibilityDataSource.cs index a032a26649..0fe704ad78 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/MSTestCompatibilityDataSource.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/MSTestCompatibilityDataSource.cs @@ -46,7 +46,7 @@ public MSTestCompatibilityDataSource(string runners = AcceptanceTestBase.DEFAULT public override void CreateData(MethodInfo methodInfo) { var isWindows = Environment.OSVersion.Platform.ToString().StartsWith("Win"); - // Only run .NET Framework tests on Windows. + // Run .NET Framework tests only on Windows. Func filter = tfm => isWindows || !tfm.StartsWith("net4"); if (InProcess) @@ -111,3 +111,4 @@ private static XmlDocument GetDependenciesXml() return depsXml; } } + diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/TestDataSource.cs b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/TestDataSource.cs index 104aee8d8a..2aeb5fa0c1 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/TestDataSource.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/TestDataSource.cs @@ -69,3 +69,34 @@ string ITestDataSource.GetDisplayName(MethodInfo methodInfo, object[] data) } } +[AttributeUsage(AttributeTargets.Method)] +public abstract class TestDataSource : Attribute, ITestDataSource + where T1 : notnull + where T2 : notnull + where T3 : notnull +{ + private readonly List _data = new(); + + public abstract void CreateData(MethodInfo methodInfo); + + public void AddData(T1 value1, T2 value2, T3 value3) + { + _data.Add(new object[] { value1, value2, value3 }); + } + + public virtual string GetDisplayName(MethodInfo methodInfo, T1 value1, T2 value2, T3 value3) + { + return $"{methodInfo.Name} ({value1}, {value2}, {value3})"; + } + + IEnumerable ITestDataSource.GetData(MethodInfo methodInfo) + { + CreateData(methodInfo); + return _data; + } + + string ITestDataSource.GetDisplayName(MethodInfo methodInfo, object[] data) + { + return GetDisplayName(methodInfo, (T1)data[0], (T2)data[1], (T3)data[2]); + } +} diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/TesthostCompatibilityDataSource.cs b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/TesthostCompatibilityDataSource.cs new file mode 100644 index 0000000000..fc5d78bf95 --- /dev/null +++ b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/TesthostCompatibilityDataSource.cs @@ -0,0 +1,139 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using System; +using System.Globalization; +using System.IO; +using System.Linq; +using System.Reflection; +using System.Xml; + +using Microsoft.TestPlatform.TestUtilities; + + +namespace Microsoft.TestPlatform.AcceptanceTests; + +public sealed class TesthostCompatibilityDataSource : TestDataSource +{ + private static XmlDocument? s_depsXml; + private readonly string[] _runnerFrameworks; + private readonly string[] _targetFrameworks; + private readonly string[] _testhostVersions; + + /// + /// Initializes a new instance. + /// + /// To run tests with desktop runner(vstest.console.exe), use AcceptanceTestBase.Net452TargetFramework or alike values. + public TesthostCompatibilityDataSource(string runners = AcceptanceTestBase.DEFAULT_RUNNER_NETFX_AND_NET, string targetFrameworks = AcceptanceTestBase.DEFAULT_HOST_NETFX_AND_NET, + string runnerVersions = AcceptanceTestBase.LATESTPREVIEW_LEGACY, string testhostVersions = AcceptanceTestBase.LATESTPREVIEW_LEGACY) + { + _runnerFrameworks = runners.Split(';'); + _targetFrameworks = targetFrameworks.Split(';'); + _testhostVersions = testhostVersions.Split(';'); + + // Do not generate the data rows here, properties (e.g. InProcess) are not populated until after constructor is done. + } + + public bool DebugVSTestConsole { get; set; } + public bool DebugTesthost { get; set; } + public bool DebugDataCollector { get; set; } + public bool NoDefaultBreakpoints { get; set; } = true; + + public override void CreateData(MethodInfo methodInfo) + { + var isWindows = Environment.OSVersion.Platform.ToString().StartsWith("Win"); + // Run .NET Framework tests only on Windows. + Func filter = tfm => isWindows || tfm.StartsWith("net4"); + var onlyLatest = new[] { AcceptanceTestBase.LATEST }; + + // Test different versions of vstest console together with latest testhost. + // There is no point in proving that old testhost does not work with old vstest.console + // when we can patch neither. + foreach (var runner in _runnerFrameworks.Where(filter)) + { + foreach (var fmw in _targetFrameworks.Where(filter)) + { + // For .NET Framework generate only latest console with latest testhost, + // we cannot control the version of testhost because it is shipped with the console. + var testhostVersions = runner.StartsWith("net4") ? onlyLatest : _testhostVersions; + foreach (var testhostVersion in testhostVersions) + { + var runnerInfo = new RunnerInfo(runner, fmw, AcceptanceTestBase.InIsolation, + DebugVSTestConsole, DebugTesthost, DebugDataCollector, NoDefaultBreakpoints); + + var vstestConsoleInfo = TranslationLayerCompatibilityDataSource.GetVSTestConsoleInfo(AcceptanceTestBase.LATEST, runnerInfo); + var testhostInfo = GetTesthostInfo(testhostVersion); + + AddData(runnerInfo, vstestConsoleInfo, testhostInfo); + } + } + } + + // Test different versions of vstest console together with latest vstest.console. + // There is no point in proving that old testhost does not work with old vstest.console + // when we can patch neither. + foreach (var runner in _runnerFrameworks.Where(filter)) + { + foreach (var fmw in _targetFrameworks.Where(filter)) + { + // For .NET Framework generate only latest console with latest testhost, + // we cannot control the version of testhost because it is shipped with the console. + var consoleVersions = runner.StartsWith("net4") ? onlyLatest : _testhostVersions; + foreach (var consoleVersion in consoleVersions) + { + var runnerInfo = new RunnerInfo(runner, fmw, AcceptanceTestBase.InIsolation, + DebugVSTestConsole, DebugTesthost, DebugDataCollector, NoDefaultBreakpoints); + + var vstestConsoleInfo = TranslationLayerCompatibilityDataSource.GetVSTestConsoleInfo(consoleVersion, runnerInfo); + // Generate only for latest testhsot, + var testhostInfo = GetTesthostInfo(AcceptanceTestBase.LATEST); + + AddData(runnerInfo, vstestConsoleInfo, testhostInfo); + } + } + } + } + + public string GetDisplayName(MethodInfo methodInfo, object[] data) + { + return string.Format(CultureInfo.CurrentCulture, "{0} ({1})", methodInfo.Name, string.Join(",", data)); + } + + private TesthostInfo GetTesthostInfo(string testhostVersionType) + { + var depsXml = GetDependenciesXml(); + + // When version is Latest, we built it locally, but it gets restored into our nuget cache on build + // same as other versions, we just need to grab the version from a different property. + + var propertyName = testhostVersionType == AcceptanceTestBase.LATEST + ? $"NETTestSdkVersion" + : $"VSTestConsole{testhostVersionType}Version"; + + // It is okay when node is null, we check that Version has value when we update paths by using TesthostInfo, and throw. + // This way it throws in the body of the test which has better error reporting than throwing in the data source. + // + // We use the VSTestConsole properties to figure out testhost version, for now. + XmlNode? node = depsXml.DocumentElement?.SelectSingleNode($"PropertyGroup/{propertyName}"); + var version = node?.InnerText.Replace("[", "").Replace("]", ""); + var slash = Path.DirectorySeparatorChar; + var versionSpecificBinPath = $"{slash}bin{slash}NETTestSdk{testhostVersionType}-{version}{slash}"; + + return new TesthostInfo(testhostVersionType, version, versionSpecificBinPath); + } + + private static XmlDocument GetDependenciesXml() + { + if (s_depsXml != null) + return s_depsXml; + + var depsXmlPath = Path.Combine(IntegrationTestEnvironment.TestPlatformRootDirectory, "scripts", "build", "TestPlatform.Dependencies.props"); + var fileStream = File.OpenRead(depsXmlPath); + var xmlTextReader = new XmlTextReader(fileStream) { Namespaces = false }; + var depsXml = new XmlDocument(); + depsXml.Load(xmlTextReader); + + s_depsXml = depsXml; + return depsXml; + } +} diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/TranslationLayerCompatibilityDataSource.cs b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/TranslationLayerCompatibilityDataSource.cs index fc5149e0d4..4b7f02dfbb 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/TranslationLayerCompatibilityDataSource.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/TranslationLayerCompatibilityDataSource.cs @@ -17,7 +17,7 @@ namespace Microsoft.TestPlatform.AcceptanceTests; public sealed class TranslationLayerCompatibilityDataSource : TestDataSource { - private static XmlDocument? _depsXml; + private static XmlDocument? s_depsXml; private readonly string[] _runnerFrameworks; private readonly string[] _targetFrameworks; // private readonly string[] _translationLayerVersions; @@ -72,7 +72,7 @@ public override void CreateData(MethodInfo methodInfo) } var isWindows = Environment.OSVersion.Platform.ToString().StartsWith("Win"); - // Only run .NET Framework tests on Windows. + // Run .NET Framework tests only on Windows. Func filter = tfm => isWindows || !tfm.StartsWith("net4"); // TODO: maybe we should throw if we don't end up generating any data @@ -106,7 +106,7 @@ public string GetDisplayName(MethodInfo methodInfo, object[] data) return string.Format(CultureInfo.CurrentCulture, "{0} ({1})", methodInfo.Name, string.Join(",", data)); } - private VSTestConsoleInfo GetVSTestConsoleInfo(string vstestConsoleVersion, RunnerInfo runnerInfo) + internal static VSTestConsoleInfo GetVSTestConsoleInfo(string vstestConsoleVersion, RunnerInfo runnerInfo) { var depsXml = GetDependenciesXml(); @@ -159,8 +159,8 @@ private VSTestConsoleInfo GetVSTestConsoleInfo(string vstestConsoleVersion, Runn private static XmlDocument GetDependenciesXml() { - if (_depsXml != null) - return _depsXml; + if (s_depsXml != null) + return s_depsXml; var depsXmlPath = Path.Combine(IntegrationTestEnvironment.TestPlatformRootDirectory, "scripts", "build", "TestPlatform.Dependencies.props"); var fileStream = File.OpenRead(depsXmlPath); @@ -168,7 +168,7 @@ private static XmlDocument GetDependenciesXml() var depsXml = new XmlDocument(); depsXml.Load(xmlTextReader); - _depsXml = depsXml; + s_depsXml = depsXml; return depsXml; } } diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/CodeCoverageTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/CodeCoverageTests.cs index efa4b69597..c77faab8ab 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/CodeCoverageTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/CodeCoverageTests.cs @@ -11,7 +11,6 @@ using Castle.Core.Internal; -using Microsoft.TestPlatform.TestUtilities; using Microsoft.TestPlatform.VsTestConsole.TranslationLayer.Interfaces; using Microsoft.VisualStudio.TestPlatform.Common.Telemetry; using Microsoft.VisualStudio.TestPlatform.ObjectModel; @@ -28,13 +27,12 @@ namespace Microsoft.TestPlatform.AcceptanceTests.TranslationLayerTests; public class CodeCoverageTests : CodeCoverageAcceptanceTestBase { private IVsTestConsoleWrapper _vstestConsoleWrapper; - private TempDirectory _tempDirectory; private RunEventHandler _runEventHandler; private TestRunAttachmentsProcessingEventHandler _testRunAttachmentsProcessingEventHandler; private void Setup() { - _vstestConsoleWrapper = GetVsTestConsoleWrapper(out _tempDirectory); + _vstestConsoleWrapper = GetVsTestConsoleWrapper(); _runEventHandler = new RunEventHandler(); _testRunAttachmentsProcessingEventHandler = new TestRunAttachmentsProcessingEventHandler(); } @@ -493,7 +491,7 @@ private void AssertCoverageResults(IList attachments) { foreach (var attachment in attachmentSet.Attachments) { - var xmlCoverage = GetXmlCoverage(attachments.First().Attachments.First().Uri.LocalPath, _tempDirectory); + var xmlCoverage = GetXmlCoverage(attachments.First().Attachments.First().Uri.LocalPath, TempDirectory); foreach (var project in GetProjects()) { diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/CustomTestHostTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/CustomTestHostTests.cs index 646fda4dbf..f0524f871e 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/CustomTestHostTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/CustomTestHostTests.cs @@ -37,8 +37,8 @@ public void Cleanup() [TranslationLayerCompatibilityDataSource(BeforeFeature = Features.ATTACH_DEBUGGER)] public void RunTestsWithCustomTestHostLauncherLaunchesTheProcessUsingTheProvidedLauncher(RunnerInfo runnerInfo, VSTestConsoleInfo vsTestConsoleInfo) { - SetTestEnvironment(_testEnvironment, runnerInfo); - _vstestConsoleWrapper = GetVsTestConsoleWrapper(TempDirectory, vsTestConsoleInfo); + SetTestEnvironment(_testEnvironment, runnerInfo, vsTestConsoleInfo); + _vstestConsoleWrapper = GetVsTestConsoleWrapper(); _runEventHandler = new RunEventHandler(); var customTestHostLauncher = new TestHostLauncherV1(); @@ -58,8 +58,8 @@ public void RunTestsWithCustomTestHostLauncherLaunchesTheProcessUsingTheProvided [TranslationLayerCompatibilityDataSource(BeforeFeature = Features.ATTACH_DEBUGGER)] public void RunTestsWithCustomTestHostLauncherLaunchesTheProcessUsingTheProvidedLauncherWhenITestHostLauncher2IsProvided(RunnerInfo runnerInfo, VSTestConsoleInfo vsTestConsoleInfo) { - SetTestEnvironment(_testEnvironment, runnerInfo); - _vstestConsoleWrapper = GetVsTestConsoleWrapper(TempDirectory, vsTestConsoleInfo); + SetTestEnvironment(_testEnvironment, runnerInfo, vsTestConsoleInfo); + _vstestConsoleWrapper = GetVsTestConsoleWrapper(); _runEventHandler = new RunEventHandler(); var customTestHostLauncher = new TestHostLauncherV2(); @@ -80,8 +80,8 @@ public void RunTestsWithCustomTestHostLauncherLaunchesTheProcessUsingTheProvided [TranslationLayerCompatibilityDataSource(AfterFeature = Features.ATTACH_DEBUGGER)] public void RunTestsWithCustomTestHostLauncherAttachesToDebuggerUsingTheProvidedLauncher(RunnerInfo runnerInfo, VSTestConsoleInfo vsTestConsoleInfo) { - SetTestEnvironment(_testEnvironment, runnerInfo); - _vstestConsoleWrapper = GetVsTestConsoleWrapper(TempDirectory, vsTestConsoleInfo); + SetTestEnvironment(_testEnvironment, runnerInfo, vsTestConsoleInfo); + _vstestConsoleWrapper = GetVsTestConsoleWrapper(); _runEventHandler = new RunEventHandler(); var customTestHostLauncher = new TestHostLauncherV2(); @@ -105,8 +105,8 @@ public void RunTestsWithCustomTestHostLauncherAttachesToDebuggerUsingTheProvided [TranslationLayerCompatibilityDataSource("net451", "net451", "Latest", AfterFeature = Features.ATTACH_DEBUGGER, DebugVSTestConsole = true, DebugTesthost=true)] public void RunTestsWithCustomTestHostLauncherUsesLaunchWhenGivenAnOutdatedITestHostLauncher(RunnerInfo runnerInfo, VSTestConsoleInfo vsTestConsoleInfo) { - SetTestEnvironment(_testEnvironment, runnerInfo); - _vstestConsoleWrapper = GetVsTestConsoleWrapper(TempDirectory, vsTestConsoleInfo); + SetTestEnvironment(_testEnvironment, runnerInfo, vsTestConsoleInfo); + _vstestConsoleWrapper = GetVsTestConsoleWrapper(); _runEventHandler = new RunEventHandler(); var customTestHostLauncher = new TestHostLauncherV1(); diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/DifferentTestFrameworkSimpleTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/DifferentTestFrameworkSimpleTests.cs index 92150afe0c..b08763da05 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/DifferentTestFrameworkSimpleTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/DifferentTestFrameworkSimpleTests.cs @@ -26,7 +26,7 @@ public class DifferentTestFrameworkSimpleTests : AcceptanceTestBase private void Setup() { - _vstestConsoleWrapper = GetVsTestConsoleWrapper(out _); + _vstestConsoleWrapper = GetVsTestConsoleWrapper(); _runEventHandler = new RunEventHandler(); } diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/DiscoverTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/DiscoverTests.cs index 3fd97f543e..3a59d0de0f 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/DiscoverTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/DiscoverTests.cs @@ -28,7 +28,7 @@ public class DiscoverTests : AcceptanceTestBase public void Setup() { - _vstestConsoleWrapper = GetVsTestConsoleWrapper(out _); + _vstestConsoleWrapper = GetVsTestConsoleWrapper(); _discoveryEventHandler = new DiscoveryEventHandler(); _discoveryEventHandler2 = new DiscoveryEventHandler2(); } @@ -43,13 +43,13 @@ public void Cleanup() [TranslationLayerCompatibilityDataSource] public void DiscoverTestsUsingDiscoveryEventHandler1(RunnerInfo runnerInfo, VSTestConsoleInfo vsTestConsoleInfo) { - SetTestEnvironment(_testEnvironment, runnerInfo); + SetTestEnvironment(_testEnvironment, runnerInfo, vsTestConsoleInfo); // Setup(); _discoveryEventHandler = new DiscoveryEventHandler(); _discoveryEventHandler2 = new DiscoveryEventHandler2(); - var vstestConsoleWrapper = GetVsTestConsoleWrapper(TempDirectory, vsTestConsoleInfo); + var vstestConsoleWrapper = GetVsTestConsoleWrapper(); vstestConsoleWrapper.DiscoverTests(GetTestAssemblies(), GetDefaultRunSettings(), _discoveryEventHandler); // Assert. @@ -60,13 +60,13 @@ public void DiscoverTestsUsingDiscoveryEventHandler1(RunnerInfo runnerInfo, VSTe [TranslationLayerCompatibilityDataSource()] public void DiscoverTestsUsingDiscoveryEventHandler2AndTelemetryOptedOut(RunnerInfo runnerInfo, VSTestConsoleInfo vsTestConsoleInfo) { - SetTestEnvironment(_testEnvironment, runnerInfo); + SetTestEnvironment(_testEnvironment, runnerInfo, vsTestConsoleInfo); // Setup(); _discoveryEventHandler = new DiscoveryEventHandler(); _discoveryEventHandler2 = new DiscoveryEventHandler2(); - var vstestConsoleWrapper = GetVsTestConsoleWrapper(TempDirectory, vsTestConsoleInfo); + var vstestConsoleWrapper = GetVsTestConsoleWrapper(); vstestConsoleWrapper.DiscoverTests( GetTestAssemblies(), GetDefaultRunSettings(), diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/LiveUnitTestingTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/LiveUnitTestingTests.cs index 5faeaa83d2..0beac2e85c 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/LiveUnitTestingTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/LiveUnitTestingTests.cs @@ -21,7 +21,7 @@ public class LiveUnitTestingTests : AcceptanceTestBase public void Setup() { - _vstestConsoleWrapper = GetVsTestConsoleWrapper(out _); + _vstestConsoleWrapper = GetVsTestConsoleWrapper(); _discoveryEventHandler = new DiscoveryEventHandler(); _runEventHandler = new RunEventHandler(); } diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/RunSelectedTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/RunSelectedTests.cs index 2aeb7a7680..73b07981b6 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/RunSelectedTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/RunSelectedTests.cs @@ -23,7 +23,7 @@ public class RunSelectedTests : AcceptanceTestBase private void Setup() { - _vstestConsoleWrapper = GetVsTestConsoleWrapper(out _); + _vstestConsoleWrapper = GetVsTestConsoleWrapper(); _runEventHandler = new RunEventHandler(); _discoveryEventHandler = new DiscoveryEventHandler(); } diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/RunTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/RunTests.cs index 4c4197078e..d966022eeb 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/RunTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/RunTests.cs @@ -29,7 +29,7 @@ public class RunTests : AcceptanceTestBase private void Setup() { - _vstestConsoleWrapper = GetVsTestConsoleWrapper(out _); + _vstestConsoleWrapper = GetVsTestConsoleWrapper(); _runEventHandler = new RunEventHandler(); } @@ -43,18 +43,17 @@ public void Cleanup() [TranslationLayerCompatibilityDataSource] public void RunAllTests(RunnerInfo runnerInfo, VSTestConsoleInfo vsTestConsoleInfo) { - SetTestEnvironment(_testEnvironment, runnerInfo); - // Setup(); - - var vstestConsoleWrapper = GetVsTestConsoleWrapper(TempDirectory, vsTestConsoleInfo); - _runEventHandler = new RunEventHandler(); - vstestConsoleWrapper.RunTests(GetTestAssemblies(), GetDefaultRunSettings(), _runEventHandler); + SetTestEnvironment(_testEnvironment, runnerInfo, vsTestConsoleInfo); + + var vstestConsoleWrapper = GetVsTestConsoleWrapper(); + var runEventHandler = new RunEventHandler(); + vstestConsoleWrapper.RunTests(GetTestAssemblies(), GetDefaultRunSettings(), runEventHandler); // Assert - Assert.AreEqual(6, _runEventHandler.TestResults.Count); - Assert.AreEqual(2, _runEventHandler.TestResults.Count(t => t.Outcome == TestOutcome.Passed)); - Assert.AreEqual(2, _runEventHandler.TestResults.Count(t => t.Outcome == TestOutcome.Failed)); - Assert.AreEqual(2, _runEventHandler.TestResults.Count(t => t.Outcome == TestOutcome.Skipped)); + Assert.AreEqual(6, runEventHandler.TestResults.Count); + Assert.AreEqual(2, runEventHandler.TestResults.Count(t => t.Outcome == TestOutcome.Passed)); + Assert.AreEqual(2, runEventHandler.TestResults.Count(t => t.Outcome == TestOutcome.Failed)); + Assert.AreEqual(2, runEventHandler.TestResults.Count(t => t.Outcome == TestOutcome.Skipped)); } [TestMethod] @@ -189,12 +188,10 @@ public void RunTestsShouldShowProperWarningOnNoTestsForTestCaseFilter(RunnerInfo private IList GetTestAssemblies() { - var testAssemblies = new List + return new List { GetAssetFullPath("SimpleTestProject.dll"), GetAssetFullPath("SimpleTestProject2.dll") }; - - return testAssemblies; } } diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/RunTestsWithDifferentConfigurationTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/RunTestsWithDifferentConfigurationTests.cs index 0da7554f11..4f6e641fb2 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/RunTestsWithDifferentConfigurationTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/RunTestsWithDifferentConfigurationTests.cs @@ -31,8 +31,8 @@ public class RunTestsWithDifferentConfigurationTests : AcceptanceTestBase private void Setup() { - _vstestConsoleWrapper = GetVsTestConsoleWrapper(out var logsDir); - _logsDir = logsDir; + _vstestConsoleWrapper = GetVsTestConsoleWrapper(); + _logsDir = TempDirectory; _runEventHandler = new RunEventHandler(); } diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/RunTestsWithFilterTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/RunTestsWithFilterTests.cs index 010e916d5b..9edda9281e 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/RunTestsWithFilterTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/RunTestsWithFilterTests.cs @@ -25,7 +25,7 @@ public class RunTestsWithFilterTests : AcceptanceTestBase private void Setup() { - _vstestConsoleWrapper = GetVsTestConsoleWrapper(out _); + _vstestConsoleWrapper = GetVsTestConsoleWrapper(); _runEventHandler = new RunEventHandler(); } @@ -39,12 +39,12 @@ public void Cleanup() [TranslationLayerCompatibilityDataSource] public void RunTestsWithTestCaseFilter(RunnerInfo runnerInfo, VSTestConsoleInfo vsTestConsoleInfo) { - SetTestEnvironment(_testEnvironment, runnerInfo); + SetTestEnvironment(_testEnvironment, runnerInfo, vsTestConsoleInfo); // Setup(); _runEventHandler = new RunEventHandler(); - var vstestConsoleWrapper = GetVsTestConsoleWrapper(TempDirectory, vsTestConsoleInfo); + var vstestConsoleWrapper = GetVsTestConsoleWrapper(); var sources = new List { GetAssetFullPath("SimpleTestProject.dll") }; vstestConsoleWrapper.RunTests( diff --git a/test/Microsoft.TestPlatform.TestUtilities/DllInfo.cs b/test/Microsoft.TestPlatform.TestUtilities/DllInfo.cs new file mode 100644 index 0000000000..e417ef9db7 --- /dev/null +++ b/test/Microsoft.TestPlatform.TestUtilities/DllInfo.cs @@ -0,0 +1,39 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using System; + +namespace Microsoft.TestPlatform.TestUtilities; + +public abstract class DllInfo +{ + protected DllInfo(string name, string propertyName, string versionType, string? version, string path) + { + Name = name; + PropertyName = propertyName; + VersionType = versionType; + // Version can be null when we fail to find the respective propertin in TestPlatform.Dependencies.props + // when that happens we throw when we try to update the path. + Version = version; + Path = path; + } + + public string Name { get; } + public string PropertyName { get; } + public string VersionType { get; } + public string? Version { get; } + public string Path { get; } + + public override string ToString() => $" {Name} = {Version} [{VersionType}]"; + + public string UpdatePath(string path) + { + // Version is not directly used, below, but if it is not populated the path will be incorrect. + // We don't want to throw when creating SourcePathInfo because that is happening too early, and has worse error reporting. + if (Version == null) + throw new InvalidOperationException($"Version was not correctly populated from TestPlatform.Dependencies.props, review that there is entry for {PropertyName}{VersionType}Version."); + + // TODO: replacing in the result string is lame, but I am not going to fight 20 GetAssetFullPath method overloads right now + return path.Replace($"{System.IO.Path.DirectorySeparatorChar}bin{System.IO.Path.DirectorySeparatorChar}", Path); + } +} diff --git a/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestBase.cs b/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestBase.cs index 1b309be65c..121d767536 100644 --- a/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestBase.cs +++ b/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestBase.cs @@ -467,6 +467,17 @@ protected string GetAssetFullPath(string assetName, string targetFramework) return _testEnvironment.GetTestAsset(assetName, targetFramework); } + protected List GetAssetFullPath(DllInfo dllInfo, params string[] assetNames) + { + var assets = new List(); + foreach (var assetName in assetNames) + { + assets.Add(dllInfo.UpdatePath(GetAssetFullPath(assetName))); + } + + return assets; + } + protected string GetProjectFullPath(string projectName) { return _testEnvironment.GetTestProject(projectName); @@ -524,7 +535,14 @@ public virtual string GetConsoleRunnerPath() if (IsDesktopRunner()) { - consoleRunnerPath = Path.Combine(_testEnvironment.PublishDirectory, "vstest.console.exe"); + if (!string.IsNullOrWhiteSpace(_testEnvironment.VSTestConsolePath)) + { + consoleRunnerPath = _testEnvironment.VSTestConsolePath; + } + else + { + consoleRunnerPath = Path.Combine(_testEnvironment.PublishDirectory, "vstest.console.exe"); + } } else if (IsNetCoreRunner()) { @@ -536,6 +554,8 @@ public virtual string GetConsoleRunnerPath() Assert.Fail("Unknown Runner framework - [{0}]", _testEnvironment.RunnerFramework); } + + Assert.IsTrue(File.Exists(consoleRunnerPath), "GetConsoleRunnerPath: Path not found: {0}", consoleRunnerPath); return consoleRunnerPath; } @@ -551,39 +571,15 @@ protected virtual string SetVSTestConsoleDLLPathInArgs(string args) return args; } - /// - /// Returns the VsTestConsole Wrapper. - /// - public IVsTestConsoleWrapper GetVsTestConsoleWrapper() - { - return GetVsTestConsoleWrapper(TempDirectory, null); - } - - /// - /// Returns the VsTestConsole Wrapper. - /// - public IVsTestConsoleWrapper GetVsTestConsoleWrapper(out TempDirectory logFileDir) - { - logFileDir = new TempDirectory(); - return GetVsTestConsoleWrapper(logFileDir, null); - } - - /// - /// Returns the VsTestConsole Wrapper. - /// - public IVsTestConsoleWrapper GetVsTestConsoleWrapper(TempDirectory tempDirectory) - { - return GetVsTestConsoleWrapper(tempDirectory, vsTestConsoleInfo: null); - } /// /// Returns the VsTestConsole Wrapper. /// /// - public IVsTestConsoleWrapper GetVsTestConsoleWrapper(TempDirectory logFileDir, VSTestConsoleInfo vsTestConsoleInfo) + public IVsTestConsoleWrapper GetVsTestConsoleWrapper() { // Temp directory is already unique so there is no need to have a unique file name. - var logFilePath = Path.Combine(logFileDir.Path, "log.txt"); + var logFilePath = Path.Combine(TempDirectory.Path, "log.txt"); if (!File.Exists(logFilePath)) { File.Create(logFilePath).Close(); @@ -591,10 +587,10 @@ public IVsTestConsoleWrapper GetVsTestConsoleWrapper(TempDirectory logFileDir, V Console.WriteLine($"Logging diagnostics in {logFilePath}"); - var consoleRunnerPath = vsTestConsoleInfo != null - ? vsTestConsoleInfo.Path - : IsNetCoreRunner() - ? Path.Combine(_testEnvironment.PublishDirectory, "vstest.console.dll") + var consoleRunnerPath = IsNetCoreRunner() + ? _testEnvironment.VSTestConsolePath != null + ? _testEnvironment.VSTestConsolePath + : Path.Combine(_testEnvironment.PublishDirectory, "vstest.console.dll") : GetConsoleRunnerPath(); var executablePath = IsWindows ? @"dotnet\dotnet.exe" : @"dotnet-linux/dotnet"; var dotnetPath = Path.Combine(_testEnvironment.ToolsDirectory, executablePath); @@ -853,13 +849,13 @@ protected string BuildMultipleAssemblyPath(params string[] assetNames) return string.Join(" ", assetFullPath); } - protected string BuildMultipleAssemblyPath(MSTestInfo msTestInfo, params string[] assetNames) + protected string BuildMultipleAssemblyPath(DllInfo dllInfo, params string[] assetNames) { var assetFullPaths = new string[assetNames.Length]; for (var i = 0; i < assetNames.Length; i++) { var path = GetAssetFullPath(assetNames[i]); - var updatedPath = msTestInfo.UpdatePath(path); + var updatedPath = dllInfo.UpdatePath(path); Assert.IsTrue(File.Exists(updatedPath), "GetTestAsset: Path not found: {0}. Most likely you need to build using build.cmd -s PrepareAcceptanceTests.", updatedPath); assetFullPaths[i] = updatedPath.AddDoubleQuote(); diff --git a/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestEnvironment.cs b/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestEnvironment.cs index 66195fba24..7d4429d58c 100644 --- a/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestEnvironment.cs +++ b/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestEnvironment.cs @@ -197,6 +197,7 @@ public string RunnerFramework public bool DebugTesthost { get; set; } public bool DebugDataCollector { get; set; } public bool NoDefaultBreakpoints { get; set; } + public string VSTestConsolePath { get; set; } /// /// Gets the full path to a test asset. diff --git a/test/Microsoft.TestPlatform.TestUtilities/MSTestInfo.cs b/test/Microsoft.TestPlatform.TestUtilities/MSTestInfo.cs index 132d06a8c5..e4c128774e 100644 --- a/test/Microsoft.TestPlatform.TestUtilities/MSTestInfo.cs +++ b/test/Microsoft.TestPlatform.TestUtilities/MSTestInfo.cs @@ -1,35 +1,12 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT license. See LICENSE file in the project root for full license information. -using System; - namespace Microsoft.TestPlatform.TestUtilities; -public class MSTestInfo +public class MSTestInfo : DllInfo { public MSTestInfo(string versionType, string? version, string path) + : base(name: "MSTest", propertyName: "MSTest", versionType, version, path) { - VersionType = versionType; - // Version can be null when we fail to find the respective propertin in TestPlatform.Dependencies.props - // when that happens we throw when we try to update the path. - Version = version; - Path = path; - } - - public string VersionType { get; } - public string? Version { get; } - public string Path { get; } - - public override string ToString() => $" MSTest = {Version} [{VersionType}]"; - - public string UpdatePath(string path) - { - // Version is not directly used, below, but if it is not populated the path will be incorrect. - // We don't want to throw when creating MSTestInfo because that is happening too early, and has worse error reporting. - if (Version == null) - throw new InvalidOperationException($"Version was not correctly populated from TestPlatform.Dependencies.props, review that there is entry for MSTestFramework{VersionType}Version."); - - // TODO: replacing in the result string is lame, but I am not going to fight 20 GetAssetFullPath method overloads right now - return path.Replace($"{System.IO.Path.DirectorySeparatorChar}bin{System.IO.Path.DirectorySeparatorChar}", Path); } } diff --git a/test/Microsoft.TestPlatform.TestUtilities/TesthostInfo.cs b/test/Microsoft.TestPlatform.TestUtilities/TesthostInfo.cs new file mode 100644 index 0000000000..f29006fd38 --- /dev/null +++ b/test/Microsoft.TestPlatform.TestUtilities/TesthostInfo.cs @@ -0,0 +1,12 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +namespace Microsoft.TestPlatform.TestUtilities; + +public class TesthostInfo : DllInfo +{ + public TesthostInfo(string versionType, string? version, string path) + : base(name: "Testhost", propertyName: "VSTestConsole", versionType, version, path) + { + } +} From 1b0202a79deb8ae735a63bd9c851cc3f71e740a0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Jare=C5=A1?= Date: Mon, 14 Mar 2022 17:38:25 +0100 Subject: [PATCH 050/112] Runs tests from the big data source, but test sdk is not used. --- .../AcceptanceTestBase.cs | 5 +- .../ExecutionTests.cs | 17 ++ .../MSTestCompatibilityDataSource.cs | 2 +- .../Extension/RunnnerInfo.cs | 2 +- .../TestPlatformCompatibilityDataSource.cs | 284 ++++++++++++++++++ .../TesthostCompatibilityDataSource.cs | 2 +- ...TranslationLayerCompatibilityDataSource.cs | 21 +- .../PortableNugetPackageTests.cs | 34 --- .../IntegrationTestBase.cs | 85 +++++- 9 files changed, 379 insertions(+), 73 deletions(-) create mode 100644 test/Microsoft.TestPlatform.AcceptanceTests/Extension/TestPlatformCompatibilityDataSource.cs diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/AcceptanceTestBase.cs b/test/Microsoft.TestPlatform.AcceptanceTests/AcceptanceTestBase.cs index 48e1d98556..4f74da75e7 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/AcceptanceTestBase.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/AcceptanceTestBase.cs @@ -57,9 +57,10 @@ public class AcceptanceTestBase : IntegrationTestBase /// public const string DEFAULT_RUNNER_NETFX_AND_NET = $"{DEFAULT_RUNNER_NETFX};netcoreapp2.1"; public const string DEFAULT_HOST_NETFX_AND_NET = "net451;netcoreapp2.1"; - public const string LATEST_LEGACY = "Latest;LatestPreview;LatestStable;RecentStable;MostDownloaded;PreviousStable;LegacyStable"; - public const string LATESTPREVIEW_LEGACY = "LatestPreview;LatestStable;RecentStable;MostDownloaded;PreviousStable;LegacyStable"; + public const string LATEST_TO_LEGACY = "Latest;LatestPreview;LatestStable;RecentStable;MostDownloaded;PreviousStable;LegacyStable"; + public const string LATESTPREVIEW_TO_LEGACY = "LatestPreview;LatestStable;RecentStable;MostDownloaded;PreviousStable;LegacyStable"; public const string LATEST = "Latest"; + internal const string MSTEST="MSTest"; public static string And(string left, string right) { diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/ExecutionTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/ExecutionTests.cs index 89ad03fdaa..749b8f6eaf 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/ExecutionTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/ExecutionTests.cs @@ -32,6 +32,23 @@ public void RunMultipleTestAssemblies(RunnerInfo runnerInfo, MSTestInfo msTestIn ExitCodeEquals(1); // failing tests } + [TestMethod] + //[TestPlatformCompatibilityDataSource("netcoreapp2.1", "LegacyStable", "netcoreapp2.1", "Latest", "LatestPreview")] + [TestPlatformCompatibilityDataSource("netcoreapp2.1", AcceptanceTestBase.LATEST_TO_LEGACY, "netcoreapp2.1", AcceptanceTestBase.LATEST_TO_LEGACY, AcceptanceTestBase.LATESTPREVIEW_TO_LEGACY)] + //[TestPlatformCompatibilityDataSource()] + + public void RunMultipleTestAssemblies223(RunnerInfo runnerInfo, VSTestConsoleInfo consoleInfo, MSTestInfo msTestInfo) + { + SetTestEnvironment(_testEnvironment, runnerInfo, consoleInfo); + + var assemblyPaths = BuildMultipleAssemblyPath(msTestInfo, "SimpleTestProject.dll", "SimpleTestProject2.dll").Trim('\"'); + + InvokeVsTestForExecution(assemblyPaths, testAdapterPath: null, FrameworkArgValue, string.Empty); + + ValidateSummaryStatus(2, 2, 2); + ExitCodeEquals(1); // failing tests + } + [TestMethod] [TesthostCompatibilityDataSource] public void RunMultipleMSTestAssembliesOnVstestConsoleAndTesthostCombinations(RunnerInfo runnerInfo, VSTestConsoleInfo vsTestConsoleInfo, TesthostInfo testhostInfo) diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/MSTestCompatibilityDataSource.cs b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/MSTestCompatibilityDataSource.cs index 0fe704ad78..c9e40513b0 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/MSTestCompatibilityDataSource.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/MSTestCompatibilityDataSource.cs @@ -24,7 +24,7 @@ public sealed class MSTestCompatibilityDataSource : TestDataSource /// To run tests with desktop runner(vstest.console.exe), use AcceptanceTestBase.Net452TargetFramework or alike values. - public MSTestCompatibilityDataSource(string runners = AcceptanceTestBase.DEFAULT_RUNNER_NETFX_AND_NET, string targetFrameworks = AcceptanceTestBase.DEFAULT_HOST_NETFX_AND_NET, string msTestVersions = AcceptanceTestBase.LATESTPREVIEW_LEGACY) + public MSTestCompatibilityDataSource(string runners = AcceptanceTestBase.DEFAULT_RUNNER_NETFX_AND_NET, string targetFrameworks = AcceptanceTestBase.DEFAULT_HOST_NETFX_AND_NET, string msTestVersions = AcceptanceTestBase.LATESTPREVIEW_TO_LEGACY) { _runnerFrameworks = runners.Split(';'); _targetFrameworks = targetFrameworks.Split(';'); diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/RunnnerInfo.cs b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/RunnnerInfo.cs index c6f67359da..da0d836e89 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/RunnnerInfo.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/RunnnerInfo.cs @@ -11,7 +11,7 @@ namespace Microsoft.TestPlatform.AcceptanceTests; /// /// /// Supported value = /InIsolation. -public record RunnerInfo(string RunnerFramework, string TargetFramework, string? InIsolationValue = "", +public record struct RunnerInfo(string RunnerFramework, string TargetFramework, string? InIsolationValue = "", bool DebugVSTestConsole = false, bool DebugTesthost = false, bool DebugDataCollector = false, bool NoDefaultBreakpoints = true) { /// diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/TestPlatformCompatibilityDataSource.cs b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/TestPlatformCompatibilityDataSource.cs new file mode 100644 index 0000000000..63ae2f14de --- /dev/null +++ b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/TestPlatformCompatibilityDataSource.cs @@ -0,0 +1,284 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using System; +using System.Collections.Generic; +using System.Globalization; +using System.IO; +using System.Linq; +using System.Reflection; +using System.Xml; + +using Microsoft.TestPlatform.TestUtilities; + +using Semver; + +namespace Microsoft.TestPlatform.AcceptanceTests; + +public sealed class TestPlatformCompatibilityDataSource : TestDataSource +{ + private static XmlDocument? s_depsXml; + private readonly string[] _runnerFrameworks; + private readonly string[] _runnerVersions; + private readonly string[] _hostFrameworks; + private readonly string[] _adapterVersions; + private readonly string[] _adapters; + private readonly string[] _hostVersions; + + public TestPlatformCompatibilityDataSource( + string runnerFrameworks = AcceptanceTestBase.DEFAULT_HOST_NETFX_AND_NET, + string runnerVersions = AcceptanceTestBase.LATEST_TO_LEGACY, + string hostFrameworks = AcceptanceTestBase.DEFAULT_HOST_NETFX_AND_NET, + string hostVersions = AcceptanceTestBase.LATEST_TO_LEGACY, + string adapterVersions = AcceptanceTestBase.LATESTPREVIEW_TO_LEGACY, + string adapters = AcceptanceTestBase.MSTEST) + { + // TODO: We actually don't generate values to use different translation layers, because we don't have a good way to do + // that right now. Translation layer is loaded directly into the acceptance test, and so we don't have easy way to substitute it. + // I am keeping this source separate from vstest console compatibility data source, to be able to easily add this feature later. + _runnerFrameworks = runnerFrameworks.Split(';'); + _runnerVersions = runnerVersions.Split(';'); + _hostFrameworks = hostFrameworks.Split(';'); + _hostVersions = hostVersions.Split(';'); + _adapterVersions = adapterVersions.Split(';'); + _adapters = adapters.Split(';'); + + // Do not generate the data rows here, properties (e.g. DebugVSTestConsole) are not populated until after constructor is done. + } + + public bool DebugVSTestConsole { get; set; } + public bool DebugTesthost { get; set; } + public bool DebugDataCollector { get; set; } + public bool NoDefaultBreakpoints { get; set; } = true; + + public string? BeforeFeature { get; set; } + public string? AfterFeature { get; set; } + public string? BeforeAdapterFeature { get; set; } + public string? AfterAdapterFeature { get; set; } + + public override void CreateData(MethodInfo methodInfo) + { + var dataRows = new List>>(); + AddEveryVersionOfRunner(dataRows); + + // with every version of host + AddEveryVersionOfHost(dataRows); + + AddEveryVersionOfAdapter(dataRows); + + AddOlderConfigurations(dataRows); + + var c = dataRows.Count(); + + if (BeforeFeature != null && AfterFeature != null) + { + throw new InvalidOperationException($"You cannot specify {nameof(BeforeFeature)} and {nameof(AfterFeature)} at the same time"); + } + + var minVersion = SemVersion.Parse("0.0.0-alpha.1"); + SemVersion? beforeVersion = null; + SemVersion? afterVersion = null; + if (BeforeFeature != null) + { + var feature = Features.Table[BeforeFeature]; + beforeVersion = SemVersion.Parse(feature.Version.TrimStart('v')); + } + if (AfterFeature != null) + { + var feature = Features.Table[AfterFeature]; + afterVersion = SemVersion.Parse(feature.Version.TrimStart('v')); + } + + var isWindows = Environment.OSVersion.Platform.ToString().StartsWith("Win"); + // Run .NET Framework tests only on Windows. + Func filter = tfm => isWindows || !tfm.StartsWith("net4"); + + // TODO: maybe we should throw if we don't end up generating any data + // because none of the versions match, or some other way to identify tests that will never run because they are very outdated. + // We probably don't have that need right now, because legacy version is 15.x.x, which is very old, and we are still keeping + // compatibility. + + foreach (var dataRow in dataRows) + { + AddData(dataRow.Key, dataRow.Value.Key, dataRow.Value.Value); + } + } + + private void AddOlderConfigurations(List>> dataRows) + { + // Older configurations where the runner, host and adapter version are the same. + // We already added the row where all are newest when adding combination with all runners. + foreach (var runnerVersion in _runnerVersions.Skip(1)) + { + foreach (var runnerFramework in _runnerFrameworks) + { + foreach (var hostFramework in _hostFrameworks) + { + var isNetFramework = hostFramework.StartsWith("net4"); + var hostVersion = runnerVersion; + foreach (var adapter in _adapters) + { + var adapterVersion = runnerVersion; + AddRow(dataRows, runnerVersion, runnerFramework, hostVersion, hostFramework, adapter, adapterVersion); + } + } + } + } + } + + private void AddEveryVersionOfAdapter(List>> dataRows) + { + var runnerVersion = _runnerVersions[0]; + foreach (var runnerFramework in _runnerFrameworks) + { + foreach (var hostFramework in _hostFrameworks) + { + var isNetFramework = hostFramework.StartsWith("net4"); + // .NET Framework testhost ships with the runner, and the version from the + // runner directory is always selected, otherwise select the newest version from _hostFrameworks. + var hostVersion = isNetFramework ? runnerVersion : _hostVersions[0]; + foreach (var adapter in _adapters) + { + // We already used the newest when adding combination with every runner + foreach (var adapterVersion in _adapterVersions.Skip(1)) + { + AddRow(dataRows, runnerVersion, runnerFramework, hostVersion, hostFramework, adapter, adapterVersion); + } + } + } + } + } + + private void AddEveryVersionOfHost(List>> dataRows) + { + var runnerVersion = _runnerVersions[0]; + + foreach (var runnerFramework in _runnerFrameworks) + { + foreach (var hostFramework in _hostFrameworks) + { + var isNetFramework = hostFramework.StartsWith("net4"); + // .NET Framework testhost ships with the runner, and the version from the + // runner directory is always the same as the runner. There are no variations + // so we just need to add host versions for .NET testhosts. We also skip the + // newest version because we already added it when AddEveryVersionOfRunner + var hostVersions = isNetFramework ? Array.Empty() : _hostVersions.Skip(1).ToArray(); + foreach (var hostVersion in hostVersions) + { + foreach (var adapter in _adapters) + { + // use the newest + var adapterVersion = _adapterVersions[0]; + AddRow(dataRows, runnerVersion, runnerFramework, hostVersion, hostFramework, adapter, adapterVersion); + } + } + } + } + } + + private void AddEveryVersionOfRunner(List>> dataRows) + { + foreach (var runnerVersion in _runnerVersions) + { + foreach (var runnerFramework in _runnerFrameworks) + { + foreach (var hostFramework in _hostFrameworks) + { + var isNetFramework = hostFramework.StartsWith("net4"); + // .NET Framework testhost ships with the runner, and the version from the + // runner directory is always selected, otherwise select the newest version from _hostFrameworks. + var hostVersion = isNetFramework ? runnerVersion : _hostVersions[0]; + foreach (var adapter in _adapters) + { + // use the newest + var adapterVersion = _adapterVersions[0]; + AddRow(dataRows, runnerVersion, runnerFramework, hostVersion, hostFramework, adapter, adapterVersion); + } + } + } + } + } + + private void AddRow(List>> dataRows, + string runnerVersion, string runnerFramework, string hostVersion, string hostFramework, string adapter, string adapterVersion) + { + var mstestInfo = GetMSTestInfo(adapterVersion); + RunnerInfo runnerInfo = GetRunnerInfo(runnerFramework, hostFramework, inIsolation: true); + var vstestConsoleInfo = GetVSTestConsoleInfo(runnerVersion, runnerInfo); + dataRows.Add(new KeyValuePair>(runnerInfo, new KeyValuePair(vstestConsoleInfo, mstestInfo))); + } + + private RunnerInfo GetRunnerInfo(string runnerFramework, string hostFramework, bool inIsolation) + { + return new RunnerInfo(runnerFramework, hostFramework, inIsolation ? AcceptanceTestBase.InIsolation : null, + DebugVSTestConsole, DebugTesthost, DebugDataCollector, NoDefaultBreakpoints); + } + + public string GetDisplayName(MethodInfo methodInfo, object[] data) + { + return string.Format(CultureInfo.CurrentCulture, "{0} ({1})", methodInfo.Name, string.Join(",", data)); + } + + private MSTestInfo GetMSTestInfo(string msTestVersion) + { + var depsXml = GetDependenciesXml(); + + // It is okay when node is null, we check that Version has value when we update paths by using MSTestInfo, and throw. + // This way it throws in the body of the test which has better error reporting than throwing in the data source. + XmlNode? node = depsXml.DocumentElement?.SelectSingleNode($"PropertyGroup/MSTestFramework{msTestVersion}Version"); + var version = node?.InnerText.Replace("[", "").Replace("]", ""); + var slash = Path.DirectorySeparatorChar; + var versionSpecificBinPath = $"{slash}bin{slash}MSTest{msTestVersion}-{version}{slash}"; + + return new MSTestInfo(msTestVersion, version, versionSpecificBinPath); + } + + internal static VSTestConsoleInfo GetVSTestConsoleInfo(string vstestConsoleVersion, RunnerInfo runnerInfo) + { + var depsXml = GetDependenciesXml(); + + // When version is Latest, we built it locally, but it gets restored into our nuget cache on build + // same as other versions, we just need to grab the version from a different property. + + var propertyName = vstestConsoleVersion == AcceptanceTestBase.LATEST + ? $"NETTestSdkVersion" + : $"VSTestConsole{vstestConsoleVersion}Version"; + + var packageName = runnerInfo.IsNetFrameworkRunner + ? "microsoft.testplatform" + : "microsoft.testplatform.cli"; + + // It is okay when node is null, we will fail to find the executable later, and throw. + // This way it throws in the body of the test which has better error reporting than throwing in the data source. + // And we can easily find out what is going on because --WRONG-VERSION-- sticks out, and is easy to find in the codebase. + XmlNode? node = depsXml.DocumentElement?.SelectSingleNode($"PropertyGroup/{propertyName}"); + var version = node?.InnerText.Replace("[", "").Replace("]", "") ?? "--WRONG-VERSION--"; + var vstestConsolePath = runnerInfo.IsNetFrameworkRunner + ? Path.Combine(IntegrationTestEnvironment.TestPlatformRootDirectory, "packages", packageName, version, + "tools", "net451", "Common7", "IDE", "Extensions", "TestPlatform", "vstest.console.exe") + : Path.Combine(IntegrationTestEnvironment.TestPlatformRootDirectory, "packages", packageName, version, + "contentFiles", "any", "netcoreapp2.1", "vstest.console.dll"); + + if (version.StartsWith("15.")) + { + vstestConsolePath = vstestConsolePath.Replace("netcoreapp2.1", "netcoreapp2.0"); + } + + return new VSTestConsoleInfo(vstestConsoleVersion, version, vstestConsolePath); + } + + private static XmlDocument GetDependenciesXml() + { + if (s_depsXml != null) + return s_depsXml; + + var depsXmlPath = Path.Combine(IntegrationTestEnvironment.TestPlatformRootDirectory, "scripts", "build", "TestPlatform.Dependencies.props"); + var fileStream = File.OpenRead(depsXmlPath); + var xmlTextReader = new XmlTextReader(fileStream) { Namespaces = false }; + var depsXml = new XmlDocument(); + depsXml.Load(xmlTextReader); + + s_depsXml = depsXml; + return depsXml; + } +} diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/TesthostCompatibilityDataSource.cs b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/TesthostCompatibilityDataSource.cs index fc5d78bf95..8039ecce7c 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/TesthostCompatibilityDataSource.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/TesthostCompatibilityDataSource.cs @@ -25,7 +25,7 @@ public sealed class TesthostCompatibilityDataSource : TestDataSource /// To run tests with desktop runner(vstest.console.exe), use AcceptanceTestBase.Net452TargetFramework or alike values. public TesthostCompatibilityDataSource(string runners = AcceptanceTestBase.DEFAULT_RUNNER_NETFX_AND_NET, string targetFrameworks = AcceptanceTestBase.DEFAULT_HOST_NETFX_AND_NET, - string runnerVersions = AcceptanceTestBase.LATESTPREVIEW_LEGACY, string testhostVersions = AcceptanceTestBase.LATESTPREVIEW_LEGACY) + string runnerVersions = AcceptanceTestBase.LATESTPREVIEW_TO_LEGACY, string testhostVersions = AcceptanceTestBase.LATESTPREVIEW_TO_LEGACY) { _runnerFrameworks = runners.Split(';'); _targetFrameworks = targetFrameworks.Split(';'); diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/TranslationLayerCompatibilityDataSource.cs b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/TranslationLayerCompatibilityDataSource.cs index 4b7f02dfbb..d5e8d2ee77 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/TranslationLayerCompatibilityDataSource.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/TranslationLayerCompatibilityDataSource.cs @@ -30,7 +30,7 @@ public sealed class TranslationLayerCompatibilityDataSource : TestDataSource environmen /// Arguments provided to vstest.console.exe public void InvokeDotnetTest(string arguments) { - var vstestConsolePath = Path.Combine(IntegrationTestEnvironment.TestPlatformRootDirectory, "artifacts", IntegrationTestEnvironment.BuildConfiguration, "netcoreapp2.1", "vstest.console.dll"); + var vstestConsolePath = GetDotnetRunnerPath(); var env = "VSTEST_CONSOLE_PATH"; var originalVstestConsolePath = Environment.GetEnvironmentVariable(env); @@ -275,6 +276,13 @@ public void ExecuteNotSupportedRunnerFrameworkTests(string runnerFramework, stri /// Skipped test count public void ValidateSummaryStatus(int passedTestsCount, int failedTestsCount, int skippedTestsCount) { + // TODO: Switch on the actual version of vstest console when we have that set on test environment. + if (_testEnvironment.VSTestConsolePath.Contains($"{Path.DirectorySeparatorChar}15.")) + { + ValidateSummaryStatusv15(passedTestsCount, failedTestsCount, skippedTestsCount); + return; + } + var totalTestCount = passedTestsCount + failedTestsCount + skippedTestsCount; if (totalTestCount == 0) { @@ -323,6 +331,57 @@ public void ValidateSummaryStatus(int passedTestsCount, int failedTestsCount, in } } + /// + /// Validate if the overall test count and results are matching. + /// + /// Passed test count + /// Failed test count + /// Skipped test count + public void ValidateSummaryStatusv15(int passedTestsCount, int failedTestsCount, int skippedTestsCount) + { + // example: Total tests: 6. Passed: 2. Failed: 2. Skipped: 2. + var totalTestCount = passedTestsCount + failedTestsCount + skippedTestsCount; + if (totalTestCount == 0) + { + // No test should be found/run + StringAssert.DoesNotMatch( + _standardTestOutput, + new Regex("Total tests\\:"), + "Excepted: There should not be test summary{2}Actual: {0}{2}Standard Error: {1}{2}Arguments: {3}{2}", + _standardTestOutput, + _standardTestError, + Environment.NewLine, + _arguments); + } + else + { + var summaryStatus = $"Total tests: {totalTestCount}."; + if (passedTestsCount != 0) + { + summaryStatus += $" Passed: {passedTestsCount}."; + } + + if (failedTestsCount != 0) + { + summaryStatus += $" Failed: {failedTestsCount}."; + } + + if (skippedTestsCount != 0) + { + summaryStatus += $" Skipped: {skippedTestsCount}."; + } + + Assert.IsTrue( + _standardTestOutput.Contains(summaryStatus), + "The Test summary does not match.{3}Expected summary: {1}{3}Test Output: {0}{3}Standard Error: {2}{3}Arguments: {4}{3}", + _standardTestOutput, + summaryStatus, + _standardTestError, + Environment.NewLine, + _arguments); + } + } + public void StdErrorContains(string substring) { Assert.IsTrue(_standardTestError.Contains(substring), "StdErrorOutput - [{0}] did not contain expected string '{1}'", _standardTestError, substring); @@ -417,7 +476,7 @@ public void ValidateDiscoveredTests(params string[] discoveredTestsList) || _standardTestOutput.Contains(GetTestMethodName(test)); Assert.IsTrue(flag, $"Test {test} does not appear in discovered tests list." + $"{Environment.NewLine}Std Output: {_standardTestOutput}" + - $"{Environment.NewLine}Std Error: { _standardTestError}"); + $"{Environment.NewLine}Std Error: {_standardTestError}"); } } @@ -433,7 +492,7 @@ public void ValidateTestsNotDiscovered(params string[] testsList) || _standardTestOutput.Contains(GetTestMethodName(test)); Assert.IsFalse(flag, $"Test {test} should not appear in discovered tests list." + $"{Environment.NewLine}Std Output: {_standardTestOutput}" + - $"{Environment.NewLine}Std Error: { _standardTestError}"); + $"{Environment.NewLine}Std Error: {_standardTestError}"); } } @@ -448,7 +507,7 @@ public void ValidateFullyQualifiedDiscoveredTests(string filePath, params string || fileOutput.Contains(GetTestMethodName(test)); Assert.IsTrue(flag, $"Test {test} does not appear in discovered tests list." + $"{Environment.NewLine}Std Output: {_standardTestOutput}" + - $"{Environment.NewLine}Std Error: { _standardTestError}"); + $"{Environment.NewLine}Std Error: {_standardTestError}"); } } @@ -472,7 +531,7 @@ protected List GetAssetFullPath(DllInfo dllInfo, params string[] assetNa var assets = new List(); foreach (var assetName in assetNames) { - assets.Add(dllInfo.UpdatePath(GetAssetFullPath(assetName))); + assets.Add(dllInfo.UpdatePath(GetAssetFullPath(assetName))); } return assets; @@ -524,7 +583,7 @@ protected bool IsNetCoreRunner() } /// - /// Gets the path to vstest.console.exe. + /// Gets the path to vstest.console.exe or dotnet.exe. /// /// /// Full path to test runner @@ -554,15 +613,13 @@ public virtual string GetConsoleRunnerPath() Assert.Fail("Unknown Runner framework - [{0}]", _testEnvironment.RunnerFramework); } - - Assert.IsTrue(File.Exists(consoleRunnerPath), "GetConsoleRunnerPath: Path not found: {0}", consoleRunnerPath); return consoleRunnerPath; } protected virtual string SetVSTestConsoleDLLPathInArgs(string args) { - var vstestConsoleDll = Path.Combine(_testEnvironment.PublishDirectory, "vstest.console.dll"); + var vstestConsoleDll = GetDotnetRunnerPath(); vstestConsoleDll = vstestConsoleDll.AddDoubleQuote(); args = string.Concat( vstestConsoleDll, @@ -588,9 +645,7 @@ public IVsTestConsoleWrapper GetVsTestConsoleWrapper() Console.WriteLine($"Logging diagnostics in {logFilePath}"); var consoleRunnerPath = IsNetCoreRunner() - ? _testEnvironment.VSTestConsolePath != null - ? _testEnvironment.VSTestConsolePath - : Path.Combine(_testEnvironment.PublishDirectory, "vstest.console.dll") + ? GetDotnetRunnerPath() : GetConsoleRunnerPath(); var executablePath = IsWindows ? @"dotnet\dotnet.exe" : @"dotnet-linux/dotnet"; var dotnetPath = Path.Combine(_testEnvironment.ToolsDirectory, executablePath); @@ -602,7 +657,7 @@ public IVsTestConsoleWrapper GetVsTestConsoleWrapper() if (!File.Exists(consoleRunnerPath)) { - throw new FileNotFoundException($"File '{consoleRunnerPath}' was not found.") ; + throw new FileNotFoundException($"File '{consoleRunnerPath}' was not found."); } Console.WriteLine($"Console runner path: {consoleRunnerPath}"); @@ -611,7 +666,7 @@ public IVsTestConsoleWrapper GetVsTestConsoleWrapper() if (_testEnvironment.DebugVSTestConsole || _testEnvironment.DebugTesthost || _testEnvironment.DebugDataCollector) { var environmentVariables = new Dictionary(); - Environment.GetEnvironmentVariables().OfType().ToList().ForEach(e=> environmentVariables.Add(e.Key.ToString(), e.Value.ToString())); + Environment.GetEnvironmentVariables().OfType().ToList().ForEach(e => environmentVariables.Add(e.Key.ToString(), e.Value.ToString())); if (_testEnvironment.DebugVSTestConsole) { @@ -900,5 +955,5 @@ protected static string GetDownloadedDotnetMuxerFromTools(string architecture) return path; } - protected static string GetDotnetRunnerPath() => Path.Combine(IntegrationTestEnvironment.TestPlatformRootDirectory, "artifacts", IntegrationTestEnvironment.BuildConfiguration, "netcoreapp2.1", "vstest.console.dll"); + protected string GetDotnetRunnerPath() => _testEnvironment.VSTestConsolePath ?? Path.Combine(_testEnvironment.PublishDirectory, "vstest.console.dll"); } From d5565250a072a1b86fdb9ec2e6a228cbc370a0ff Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Jare=C5=A1?= Date: Mon, 14 Mar 2022 18:04:42 +0100 Subject: [PATCH 051/112] Compose path to testhost + adapter. --- .../ExecutionTests.cs | 10 +++--- .../Extension/TestDataSource.cs | 32 +++++++++++++++++++ .../TestPlatformCompatibilityDataSource.cs | 21 ++++++------ .../TesthostCompatibilityDataSource.cs | 2 +- .../IntegrationTestBase.cs | 15 +++++++++ 5 files changed, 64 insertions(+), 16 deletions(-) diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/ExecutionTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/ExecutionTests.cs index 749b8f6eaf..57be416dd5 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/ExecutionTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/ExecutionTests.cs @@ -20,7 +20,7 @@ public class ExecutionTests : AcceptanceTestBase [TestMethod] [MSTestCompatibilityDataSource(InProcess = true)] - public void RunMultipleTestAssemblies(RunnerInfo runnerInfo, MSTestInfo msTestInfo) + public void RunMultipleTestAssemblies(RunnerInfo runnerInfo, TesthostInfo testhostInfo, MSTestInfo msTestInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); @@ -33,15 +33,15 @@ public void RunMultipleTestAssemblies(RunnerInfo runnerInfo, MSTestInfo msTestIn } [TestMethod] - //[TestPlatformCompatibilityDataSource("netcoreapp2.1", "LegacyStable", "netcoreapp2.1", "Latest", "LatestPreview")] - [TestPlatformCompatibilityDataSource("netcoreapp2.1", AcceptanceTestBase.LATEST_TO_LEGACY, "netcoreapp2.1", AcceptanceTestBase.LATEST_TO_LEGACY, AcceptanceTestBase.LATESTPREVIEW_TO_LEGACY)] + [TestPlatformCompatibilityDataSource("netcoreapp2.1", "LegacyStable", "netcoreapp2.1", "Latest", "LatestPreview")] + // [TestPlatformCompatibilityDataSource("netcoreapp2.1", LATEST_TO_LEGACY, "netcoreapp2.1", LATEST_TO_LEGACY, LATESTPREVIEW_TO_LEGACY)] //[TestPlatformCompatibilityDataSource()] - public void RunMultipleTestAssemblies223(RunnerInfo runnerInfo, VSTestConsoleInfo consoleInfo, MSTestInfo msTestInfo) + public void RunMultipleTestAssemblies223(RunnerInfo runnerInfo, VSTestConsoleInfo consoleInfo, TesthostInfo testhostInfo, MSTestInfo msTestInfo) { SetTestEnvironment(_testEnvironment, runnerInfo, consoleInfo); - var assemblyPaths = BuildMultipleAssemblyPath(msTestInfo, "SimpleTestProject.dll", "SimpleTestProject2.dll").Trim('\"'); + var assemblyPaths = BuildMultipleAssemblyPath(testhostInfo, msTestInfo, "SimpleTestProject.dll", "SimpleTestProject2.dll").Trim('\"'); InvokeVsTestForExecution(assemblyPaths, testAdapterPath: null, FrameworkArgValue, string.Empty); diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/TestDataSource.cs b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/TestDataSource.cs index 2aeb5fa0c1..abfead224e 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/TestDataSource.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/TestDataSource.cs @@ -100,3 +100,35 @@ string ITestDataSource.GetDisplayName(MethodInfo methodInfo, object[] data) return GetDisplayName(methodInfo, (T1)data[0], (T2)data[1], (T3)data[2]); } } + +[AttributeUsage(AttributeTargets.Method)] +public abstract class TestDataSource : Attribute, ITestDataSource + where T1 : notnull + where T2 : notnull + where T3 : notnull +{ + private readonly List _data = new(); + + public abstract void CreateData(MethodInfo methodInfo); + + public void AddData(T1 value1, T2 value2, T3 value3, T4 value4) + { + _data.Add(new object[] { value1, value2, value3, value4 }); + } + + public virtual string GetDisplayName(MethodInfo methodInfo, T1 value1, T2 value2, T3 value3, T4 value4) + { + return $"{methodInfo.Name} ({value1}, {value2}, {value3}, {value4})"; + } + + IEnumerable ITestDataSource.GetData(MethodInfo methodInfo) + { + CreateData(methodInfo); + return _data; + } + + string ITestDataSource.GetDisplayName(MethodInfo methodInfo, object[] data) + { + return GetDisplayName(methodInfo, (T1)data[0], (T2)data[1], (T3)data[2], (T4)data[3]); + } +} diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/TestPlatformCompatibilityDataSource.cs b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/TestPlatformCompatibilityDataSource.cs index 63ae2f14de..63d0a7105e 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/TestPlatformCompatibilityDataSource.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/TestPlatformCompatibilityDataSource.cs @@ -15,7 +15,7 @@ namespace Microsoft.TestPlatform.AcceptanceTests; -public sealed class TestPlatformCompatibilityDataSource : TestDataSource +public sealed class TestPlatformCompatibilityDataSource : TestDataSource { private static XmlDocument? s_depsXml; private readonly string[] _runnerFrameworks; @@ -58,7 +58,7 @@ public TestPlatformCompatibilityDataSource( public override void CreateData(MethodInfo methodInfo) { - var dataRows = new List>>(); + var dataRows = new List<(RunnerInfo runnerInfo, VSTestConsoleInfo vstestConsoleInfo, TesthostInfo testhostInfo, MSTestInfo mstestInfo)>(); AddEveryVersionOfRunner(dataRows); // with every version of host @@ -100,11 +100,11 @@ public override void CreateData(MethodInfo methodInfo) foreach (var dataRow in dataRows) { - AddData(dataRow.Key, dataRow.Value.Key, dataRow.Value.Value); + AddData(dataRow.runnerInfo, dataRow.vstestConsoleInfo, dataRow.testhostInfo, dataRow.mstestInfo); } } - private void AddOlderConfigurations(List>> dataRows) + private void AddOlderConfigurations(List<(RunnerInfo, VSTestConsoleInfo, TesthostInfo, MSTestInfo)> dataRows) { // Older configurations where the runner, host and adapter version are the same. // We already added the row where all are newest when adding combination with all runners. @@ -126,7 +126,7 @@ private void AddOlderConfigurations(List>> dataRows) + private void AddEveryVersionOfAdapter(List<(RunnerInfo, VSTestConsoleInfo, TesthostInfo, MSTestInfo)> dataRows) { var runnerVersion = _runnerVersions[0]; foreach (var runnerFramework in _runnerFrameworks) @@ -149,7 +149,7 @@ private void AddEveryVersionOfAdapter(List>> dataRows) + private void AddEveryVersionOfHost(List<(RunnerInfo, VSTestConsoleInfo, TesthostInfo, MSTestInfo)> dataRows) { var runnerVersion = _runnerVersions[0]; @@ -176,7 +176,7 @@ private void AddEveryVersionOfHost(List>> dataRows) + private void AddEveryVersionOfRunner(List<(RunnerInfo, VSTestConsoleInfo, TesthostInfo, MSTestInfo)> dataRows) { foreach (var runnerVersion in _runnerVersions) { @@ -199,13 +199,14 @@ private void AddEveryVersionOfRunner(List>> dataRows, + private void AddRow(List<(RunnerInfo, VSTestConsoleInfo, TesthostInfo, MSTestInfo)> dataRows, string runnerVersion, string runnerFramework, string hostVersion, string hostFramework, string adapter, string adapterVersion) { - var mstestInfo = GetMSTestInfo(adapterVersion); RunnerInfo runnerInfo = GetRunnerInfo(runnerFramework, hostFramework, inIsolation: true); var vstestConsoleInfo = GetVSTestConsoleInfo(runnerVersion, runnerInfo); - dataRows.Add(new KeyValuePair>(runnerInfo, new KeyValuePair(vstestConsoleInfo, mstestInfo))); + var testhostInfo = TesthostCompatibilityDataSource.GetTesthostInfo(hostVersion); + var mstestInfo = GetMSTestInfo(adapterVersion); + dataRows.Add(new(runnerInfo, vstestConsoleInfo, testhostInfo, mstestInfo)); } private RunnerInfo GetRunnerInfo(string runnerFramework, string hostFramework, bool inIsolation) diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/TesthostCompatibilityDataSource.cs b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/TesthostCompatibilityDataSource.cs index 8039ecce7c..6892b4a210 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/TesthostCompatibilityDataSource.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/TesthostCompatibilityDataSource.cs @@ -99,7 +99,7 @@ public string GetDisplayName(MethodInfo methodInfo, object[] data) return string.Format(CultureInfo.CurrentCulture, "{0} ({1})", methodInfo.Name, string.Join(",", data)); } - private TesthostInfo GetTesthostInfo(string testhostVersionType) + internal static TesthostInfo GetTesthostInfo(string testhostVersionType) { var depsXml = GetDependenciesXml(); diff --git a/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestBase.cs b/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestBase.cs index 7836d28ed5..1e84f3b094 100644 --- a/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestBase.cs +++ b/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestBase.cs @@ -919,6 +919,21 @@ protected string BuildMultipleAssemblyPath(DllInfo dllInfo, params string[] asse return string.Join(" ", assetFullPaths); } + protected string BuildMultipleAssemblyPath(TesthostInfo testhostInfo, DllInfo adapterInfo, params string[] assetNames) + { + var assetFullPaths = new string[assetNames.Length]; + for (var i = 0; i < assetNames.Length; i++) + { + var path = GetAssetFullPath(assetNames[i]); + var updatedPath = testhostInfo.UpdatePath(adapterInfo.UpdatePath(path)); + Assert.IsTrue(File.Exists(updatedPath), "GetTestAsset: Path not found: {0}. Most likely you need to build using build.cmd -s PrepareAcceptanceTests.", updatedPath); + + assetFullPaths[i] = updatedPath.AddDoubleQuote(); + } + + return string.Join(" ", assetFullPaths); + } + protected static string GetDiagArg(string rootDir) => " --diag:" + Path.Combine(rootDir, "log.txt"); From d243b59ca398f4a326b07562e76c7123ce71ec7b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Jare=C5=A1?= Date: Tue, 15 Mar 2022 18:21:58 +0100 Subject: [PATCH 052/112] One big unified data source. Still no version filtering. --- scripts/build.ps1 | 374 ++++++++---------- .../ExecutionTests.cs | 7 +- .../MSTestCompatibilityDataSource.cs | 8 +- .../TestPlatformCompatibilityDataSource.cs | 65 ++- .../IntegrationTestBase.cs | 1 - .../MSTestProject2/MSTestProject2.csproj | Bin 0 -> 2194 bytes test/TestAssets/MSTestProject2/UnitTest1.cs | 53 +++ test/TestAssets/TestAssets.sln | 16 +- 8 files changed, 298 insertions(+), 226 deletions(-) create mode 100644 test/TestAssets/MSTestProject2/MSTestProject2.csproj create mode 100644 test/TestAssets/MSTestProject2/UnitTest1.cs diff --git a/scripts/build.ps1 b/scripts/build.ps1 index 2a27c513e4..78b06fd19e 100644 --- a/scripts/build.ps1 +++ b/scripts/build.ps1 @@ -3,47 +3,47 @@ [CmdletBinding()] Param( - [Parameter(Mandatory=$false)] + [Parameter(Mandatory = $false)] [ValidateSet("Debug", "Release")] [Alias("c")] [System.String] $Configuration = "Debug", - [Parameter(Mandatory=$false)] + [Parameter(Mandatory = $false)] [Alias("r")] [System.String] $TargetRuntime = "win7-x64", # Versioning scheme = Major(15).Minor(RTW, Updates).SubUpdates(preview4, preview5, RC etc) # E.g. VS 2017 Update 1 Preview will have version 15.1.1 - [Parameter(Mandatory=$false)] + [Parameter(Mandatory = $false)] [Alias("v")] [System.String] $Version, # Will set this later by reading TestPlatform.Settings.targets file. - [Parameter(Mandatory=$false)] + [Parameter(Mandatory = $false)] [Alias("vs")] [System.String] $VersionSuffix = "dev", - [Parameter(Mandatory=$false)] + [Parameter(Mandatory = $false)] [Alias("bn")] [System.String] $BuildNumber = "20991231-99", - [Parameter(Mandatory=$false)] + [Parameter(Mandatory = $false)] [Alias("ff")] [System.Boolean] $FailFast = $true, - [Parameter(Mandatory=$false)] + [Parameter(Mandatory = $false)] [Alias("noloc")] [Switch] $DisableLocalizedBuild, - [Parameter(Mandatory=$false)] + [Parameter(Mandatory = $false)] [Alias("ci")] [Switch] $CIBuild, - [Parameter(Mandatory=$false)] + [Parameter(Mandatory = $false)] [Alias("pt")] [Switch] $PublishTestArtifacts, # Build specific projects - [Parameter(Mandatory=$false)] + [Parameter(Mandatory = $false)] [Alias("p")] [System.String[]] $ProjectNamePatterns = @(), @@ -61,11 +61,10 @@ $ErrorView = 'Normal' # Set Version from scripts/build/TestPlatform.Settings.targets, when we are running locally and not providing the version as the parameter # or when the build is done directly in VS -if([string]::IsNullOrWhiteSpace($Version)) -{ +if ([string]::IsNullOrWhiteSpace($Version)) { $Version = ([xml](Get-Content $env:TP_ROOT_DIR\scripts\build\TestPlatform.Settings.targets)).Project.PropertyGroup[0].TPVersionPrefix | - ForEach-Object { $_.Trim() } | - Select-Object -First 1 + ForEach-Object { $_.Trim() } | + Select-Object -First 1 Write-Verbose "Version was not provided using version '$Version' from TestPlatform.Settings.targets" } @@ -128,21 +127,18 @@ if ($env:PATH -notlike "*$attachVsPath") { # so nuget restores it correctly. $vsSdkBuildToolsVersion = ([xml](Get-Content $env:TP_ROOT_DIR\scripts\build\TestPlatform.Dependencies.props)).Project.PropertyGroup.VSSdkBuildToolsVersion $vsixUtilDir = "$env:TP_ROOT_DIR\packages\microsoft.vssdk.buildtools" -if ((Test-Path $vsixUtilDir) -and -not (Test-Path "$vsixUtilDir\$vsSdkBuildToolsVersion\tools\vssdk\bin\VsixUtil.exe")) -{ +if ((Test-Path $vsixUtilDir) -and -not (Test-Path "$vsixUtilDir\$vsSdkBuildToolsVersion\tools\vssdk\bin\VsixUtil.exe")) { Remove-Item -Recurse -Force $vsixUtilDir } # Procdump gets regularly eaten by antivirus or something. Remove the package dir if it gets broken # so nuget restores it correctly. $procdumpDir = "$env:TP_ROOT_DIR\packages\procdump" -if ((Test-Path $procdumpDir) -and 2 -ne @(Get-Item "$procdumpDir\0.0.1\bin").Length) -{ +if ((Test-Path $procdumpDir) -and 2 -ne @(Get-Item "$procdumpDir\0.0.1\bin").Length) { Remove-Item -Recurse -Force $procdumpDir } -function Invoke-Build -{ +function Invoke-Build { $timer = Start-Timer Write-Log "Invoke-Build: Start build." $dotnetExe = Get-DotNetPath @@ -157,8 +153,7 @@ function Invoke-Build Write-Log "Invoke-Build: Complete. {$(Get-ElapsedTime($timer))}" } -function Invoke-TestAssetsBuild -{ +function Invoke-TestAssetsBuild { $timer = Start-Timer Write-Log "Invoke-TestAssetsBuild: Start test assets build." $dotnetExe = Get-DotNetPath @@ -198,8 +193,7 @@ function Invoke-TestAssetsBuild # The command line tool does not like the package ranges. $vsTestConsoleVersion = $vsTestConsoleVersion -replace "(\[|\])" - if (-not $vsTestConsoleVersion) - { + if (-not $vsTestConsoleVersion) { throw "VSTestConsoleVersion for $propertyName is empty." } @@ -240,7 +234,9 @@ function Invoke-TestAssetsBuild # Build with multiple versions of MSTest. $projects = @( "$env:TP_ROOT_DIR\test\TestAssets\SimpleTestProject\SimpleTestProject.csproj" - "$env:TP_ROOT_DIR\test\TestAssets\SimpleTestProject2\SimpleTestProject2.csproj" + "$env:TP_ROOT_DIR\test\TestAssets\MSTestProject2\MSTestProject2.csproj" + # Don't use this one, it does not use the variables for mstest and test sdk. + # "$env:TP_ROOT_DIR\test\TestAssets\SimpleTestProject2\SimpleTestProject2.csproj" ) $msTestVersionProperties = @( @@ -252,40 +248,35 @@ function Invoke-TestAssetsBuild "MSTestFrameworkLegacyStableVersion" ) - foreach ($project in $projects) { - foreach ($propertyName in $msTestVersionProperties) { - $mstestVersion = $dependenciesXml.Project.PropertyGroup.$propertyName - - if (-not $mstestVersion) - { - throw "MSTestVersion for $propertyName is empty." - } - - $dirMSTestVersion = $mstestVersion -replace "\[|\]" - $dirMSTestPropertyName = $propertyName -replace "Framework" -replace "Version" - Invoke-Exe $dotnetExe -Arguments "build $project --configuration $TPB_Configuration -v:minimal -p:CIBuild=$TPB_CIBuild -p:LocalizedBuild=$TPB_LocalizedBuild -p:MSTestFrameworkVersion=$mstestVersion -p:MSTestAdapterVersion=$mstestVersion -p:BaseOutputPath=""bin\$dirMSTestPropertyName-$dirMSTestVersion\\""" - } - } - foreach ($project in $projects) { # We use the same version properties for NET.Test.Sdk as for VSTestConsole, for now. - foreach ($propertyName in $vstestConsoleVersionProperties) { - if ("VSTestConsoleLatestVersion" -eq $propertyName) { + foreach ($sdkPropertyName in $vstestConsoleVersionProperties) { + if ("VSTestConsoleLatestVersion" -eq $sdkPropertyName) { # NETTestSdkVersion has the version of the locally built package. $netTestSdkVersion = $dependenciesXml.Project.PropertyGroup."NETTestSdkVersion" } else { - $netTestSdkVersion = $dependenciesXml.Project.PropertyGroup.$propertyName + $netTestSdkVersion = $dependenciesXml.Project.PropertyGroup.$sdkPropertyName } - - if (-not $netTestSdkVersion) - { - throw "NetTestSdkVersion for $propertyName is empty." + + if (-not $netTestSdkVersion) { + throw "NetTestSdkVersion for $sdkPropertyName is empty." } + + $dirNetTestSdkVersion = $netTestSdkVersion -replace "\[|\]" + $dirNetTestSdkPropertyName = $sdkPropertyName -replace "Framework" -replace "Version" -replace "VSTestConsole", "NETTestSdk" + + foreach ($propertyName in $msTestVersionProperties) { + $mstestVersion = $dependenciesXml.Project.PropertyGroup.$propertyName - $dirNetTestSdkVersion = $netTestSdkVersion -replace "\[|\]" - $dirNetTestSdkPropertyName = $propertyName -replace "Framework" -replace "Version" -replace "VSTestConsole", "NETTestSdk" - Invoke-Exe $dotnetExe -Arguments "build $project --configuration $TPB_Configuration -v:minimal -p:CIBuild=$TPB_CIBuild -p:LocalizedBuild=$TPB_LocalizedBuild -p:NETTestSdkVersion=$netTestSdkVersion -p:BaseOutputPath=""bin\$dirNetTestSdkPropertyName-$dirNetTestSdkVersion\\""" + if (-not $mstestVersion) { + throw "MSTestVersion for $propertyName is empty." + } + + $dirMSTestVersion = $mstestVersion -replace "\[|\]" + $dirMSTestPropertyName = $propertyName -replace "Framework" -replace "Version" + Invoke-Exe $dotnetExe -Arguments "build $project --configuration $TPB_Configuration -v:minimal -p:CIBuild=$TPB_CIBuild -p:LocalizedBuild=$TPB_LocalizedBuild -p:MSTestFrameworkVersion=$mstestVersion -p:MSTestAdapterVersion=$mstestVersion -p:NETTestSdkVersion=$netTestSdkVersion -p:BaseOutputPath=""bin\$dirNetTestSdkPropertyName-$dirNetTestSdkVersion\$dirMSTestPropertyName-$dirMSTestVersion\\"" -bl:""$env:TP_OUT_DIR\log\$Configuration\perm.binlog""" + } } } @@ -328,8 +319,7 @@ function Publish-PatchedDotnet { Copy-Item $buildArtifactsPath $dotnetTestArtifactsSdkPath -Force } -function Publish-Package -{ +function Publish-Package { $timer = Start-Timer Write-Log "Publish-Package: Started." $fullCLRPackage451Dir = Get-FullCLRPackageDirectory @@ -475,73 +465,73 @@ function Publish-Package # into the output folder (for some reason), and we overwrite it with actual uap10.0 version below Copy-Bulk -root (Join-Path $env:TP_ROOT_DIR "src\Microsoft.TestPlatform.ObjectModel\bin\$TPB_Configuration") ` - -files @{ - $TPB_TargetFramework45 = $fullCLRPackage45Dir # net45 - $TPB_TargetFramework451 = $fullCLRPackage451Dir # net451 - $TPB_TargetFrameworkCore10 = $coreCLR10PackageDir # netcoreapp1.0 - $TPB_TargetFrameworkCore20 = $coreCLR20PackageDir # netcoreapp2.1 - $TPB_TargetFrameworkNS10 = $netstandard10PackageDir # netstandard1_0 - $TPB_TargetFrameworkNS13 = $netstandard13PackageDir # netstandard1_3 - $TPB_TargetFrameworkNS20 = $netstandard20PackageDir # netstandard2_0 - $TPB_TargetFrameworkUap100 = $uap100PackageDir # uap10.0 - } + -files @{ + $TPB_TargetFramework45 = $fullCLRPackage45Dir # net45 + $TPB_TargetFramework451 = $fullCLRPackage451Dir # net451 + $TPB_TargetFrameworkCore10 = $coreCLR10PackageDir # netcoreapp1.0 + $TPB_TargetFrameworkCore20 = $coreCLR20PackageDir # netcoreapp2.1 + $TPB_TargetFrameworkNS10 = $netstandard10PackageDir # netstandard1_0 + $TPB_TargetFrameworkNS13 = $netstandard13PackageDir # netstandard1_3 + $TPB_TargetFrameworkNS20 = $netstandard20PackageDir # netstandard2_0 + $TPB_TargetFrameworkUap100 = $uap100PackageDir # uap10.0 + } Copy-Bulk -root (Join-Path $env:TP_ROOT_DIR "src\Microsoft.TestPlatform.ObjectModel\bin\$TPB_Configuration") ` - -files @{ - $TPB_TargetFrameworkUap100 = $testhostUapPackageDir # uap10.0 - testhost - } + -files @{ + $TPB_TargetFrameworkUap100 = $testhostUapPackageDir # uap10.0 - testhost + } ################################################################################ # Publish Microsoft.TestPlatform.PlatformAbstractions Copy-Bulk -root (Join-Path $env:TP_ROOT_DIR "src\Microsoft.TestPlatform.PlatformAbstractions\bin\$TPB_Configuration") ` - -files @{ - $TPB_TargetFramework45 = $fullCLRPackage45Dir # net45 - $TPB_TargetFramework451 = $fullCLRPackage451Dir # net451 - $TPB_TargetFrameworkCore20 = $coreCLR20PackageDir # netcoreapp2.1 - $TPB_TargetFrameworkNS10 = $netstandard10PackageDir # netstandard1_0 - $TPB_TargetFrameworkNS13 = $netstandard13PackageDir # netstandard1_3 - $TPB_TargetFrameworkNS20 = $netstandard20PackageDir # netstandard2_0 - $TPB_TargetFrameworkUap100 = $uap100PackageDir # uap10.0 - } + -files @{ + $TPB_TargetFramework45 = $fullCLRPackage45Dir # net45 + $TPB_TargetFramework451 = $fullCLRPackage451Dir # net451 + $TPB_TargetFrameworkCore20 = $coreCLR20PackageDir # netcoreapp2.1 + $TPB_TargetFrameworkNS10 = $netstandard10PackageDir # netstandard1_0 + $TPB_TargetFrameworkNS13 = $netstandard13PackageDir # netstandard1_3 + $TPB_TargetFrameworkNS20 = $netstandard20PackageDir # netstandard2_0 + $TPB_TargetFrameworkUap100 = $uap100PackageDir # uap10.0 + } Copy-Bulk -root (Join-Path $env:TP_ROOT_DIR "src\Microsoft.TestPlatform.PlatformAbstractions\bin\$TPB_Configuration") ` - -files @{ - $TPB_TargetFrameworkUap100 = $testhostUapPackageDir # uap10.0 - testhost - } + -files @{ + $TPB_TargetFrameworkUap100 = $testhostUapPackageDir # uap10.0 - testhost + } ################################################################################ # Publish Microsoft.TestPlatform.CoreUtilities Copy-Bulk -root (Join-Path $env:TP_ROOT_DIR "src\Microsoft.TestPlatform.CoreUtilities\bin\$TPB_Configuration") ` - -files @{ - $TPB_TargetFramework45 = $fullCLRPackage45Dir # net45 - $TPB_TargetFramework451 = $fullCLRPackage451Dir # net451 - $TPB_TargetFrameworkNS10 = $netstandard10PackageDir # netstandard1_0 - $TPB_TargetFrameworkNS13 = $netstandard13PackageDir # netstandard1_3 - $TPB_TargetFrameworkNS20 = $netstandard20PackageDir # netstandard2_0 - $TPB_TargetFrameworkUap100 = $uap100PackageDir # uap10.0 - } + -files @{ + $TPB_TargetFramework45 = $fullCLRPackage45Dir # net45 + $TPB_TargetFramework451 = $fullCLRPackage451Dir # net451 + $TPB_TargetFrameworkNS10 = $netstandard10PackageDir # netstandard1_0 + $TPB_TargetFrameworkNS13 = $netstandard13PackageDir # netstandard1_3 + $TPB_TargetFrameworkNS20 = $netstandard20PackageDir # netstandard2_0 + $TPB_TargetFrameworkUap100 = $uap100PackageDir # uap10.0 + } Copy-Bulk -root (Join-Path $env:TP_ROOT_DIR "src\Microsoft.TestPlatform.CoreUtilities\bin\$TPB_Configuration") ` - -files @{ - $TPB_TargetFrameworkUap100 = $testhostUapPackageDir # uap10.0 - testhost - } + -files @{ + $TPB_TargetFrameworkUap100 = $testhostUapPackageDir # uap10.0 - testhost + } ################################################################################ # Publish Microsoft.TestPlatform.AdapterUtilities Copy-Bulk -root (Join-Path $env:TP_ROOT_DIR "src\Microsoft.TestPlatform.AdapterUtilities\bin\$TPB_Configuration") ` - -files @{ - # "net20" = $net20PackageDir # net20 - "net45/any" = $net45PackageDir # $net4 - $TPB_TargetFrameworkNS10 = $netstandard10PackageDir # netstandard1_0 - $TPB_TargetFrameworkNS20 = $netstandard20PackageDir # netstandard2_0 - $TPB_TargetFrameworkUap100 = $uap100PackageDir # uap10.0 - } + -files @{ + # "net20" = $net20PackageDir # net20 + "net45/any" = $net45PackageDir # $net4 + $TPB_TargetFrameworkNS10 = $netstandard10PackageDir # netstandard1_0 + $TPB_TargetFrameworkNS20 = $netstandard20PackageDir # netstandard2_0 + $TPB_TargetFrameworkUap100 = $uap100PackageDir # uap10.0 + } ################################################################################ # Publish Microsoft.TestPlatform.CrossPlatEngine Copy-Bulk -root (Join-Path $env:TP_ROOT_DIR "src\Microsoft.TestPlatform.CrossPlatEngine\bin\$TPB_Configuration") ` - -files @{ - $TPB_TargetFrameworkNS13 = $netstandard13PackageDir # netstandard1_3 - } + -files @{ + $TPB_TargetFrameworkNS13 = $netstandard13PackageDir # netstandard1_3 + } ################################################################################ # Publish msdia @@ -572,7 +562,7 @@ function Publish-Package "Microsoft.VisualStudio.TestPlatform.Extensions.Html.TestLogger.dll", "Microsoft.VisualStudio.TestPlatform.Extensions.Html.TestLogger.pdb" ) - foreach($file in $loggers) { + foreach ($file in $loggers) { Write-Verbose "Move-Item $fullCLRPackage451Dir\$file $fullCLRExtensionsDir -Force" Move-Item $fullCLRPackage451Dir\$file $fullCLRExtensionsDir -Force @@ -581,7 +571,7 @@ function Publish-Package } # Move logger resource dlls - if($TPB_LocalizedBuild) { + if ($TPB_LocalizedBuild) { Move-Loc-Files $fullCLRPackage451Dir $fullCLRExtensionsDir "Microsoft.VisualStudio.TestPlatform.Extensions.Trx.TestLogger.resources.dll" Move-Loc-Files $coreCLR20PackageDir $coreCLRExtensionsDir "Microsoft.VisualStudio.TestPlatform.Extensions.Trx.TestLogger.resources.dll" Move-Loc-Files $fullCLRPackage451Dir $fullCLRExtensionsDir "Microsoft.VisualStudio.TestPlatform.Extensions.Html.TestLogger.resources.dll" @@ -622,7 +612,7 @@ function Publish-Package # Copy-Item $blameDataCollectorNetStandard\procdump $coreCLRExtensionsDir\procdump -Force # Copy blame data collector resource dlls - if($TPB_LocalizedBuild) { + if ($TPB_LocalizedBuild) { Copy-Loc-Files $blameDataCollectorNetFull $fullCLRExtensionsDir "Microsoft.TestPlatform.Extensions.BlameDataCollector.resources.dll" Copy-Loc-Files $blameDataCollectorNetStandard $coreCLRExtensionsDir "Microsoft.TestPlatform.Extensions.BlameDataCollector.resources.dll" } @@ -636,7 +626,7 @@ function Publish-Package Copy-Item $eventLogDataCollectorNetFull\Microsoft.TestPlatform.Extensions.EventLogCollector.pdb $coreCLRExtensionsDir -Force # Copy EventLogCollector resource dlls - if($TPB_LocalizedBuild) { + if ($TPB_LocalizedBuild) { Copy-Loc-Files $eventLogDataCollectorNetFull $fullCLRExtensionsDir "Microsoft.TestPlatform.Extensions.EventLogCollector.resources.dll" Copy-Loc-Files $eventLogDataCollectorNetFull $coreCLRExtensionsDir "Microsoft.TestPlatform.Extensions.EventLogCollector.resources.dll" } @@ -645,13 +635,13 @@ function Publish-Package $codeCoverageExternalsVersion = ([xml](Get-Content $env:TP_ROOT_DIR\eng\Versions.props)).Project.PropertyGroup.MicrosoftInternalCodeCoverageVersion $codeCoverageIOPackagesDir = Join-Path $env:TP_PACKAGES_DIR "microsoft.visualstudio.coverage.io\$codeCoverageExternalsVersion\lib\$TPB_TargetFrameworkStandard" Copy-Item $codeCoverageIOPackagesDir\Microsoft.VisualStudio.Coverage.IO.dll $coreCLR20PackageDir -Force - if($TPB_LocalizedBuild) { + if ($TPB_LocalizedBuild) { Copy-Loc-Files $codeCoverageIOPackagesDir $coreCLR20PackageDir "Microsoft.VisualStudio.Coverage.IO.resources.dll" } # If there are some dependencies for the TestHostRuntimeProvider assemblies, those need to be moved too. $runtimeproviders = @("Microsoft.TestPlatform.TestHostRuntimeProvider.dll", "Microsoft.TestPlatform.TestHostRuntimeProvider.pdb") - foreach($file in $runtimeproviders) { + foreach ($file in $runtimeproviders) { Write-Verbose "Move-Item $fullCLRPackage451Dir\$file $fullCLRExtensionsDir -Force" Move-Item $fullCLRPackage451Dir\$file $fullCLRExtensionsDir -Force @@ -690,8 +680,8 @@ function Publish-Package Copy-Item $testhostCore20PackageDir\testhost.pdb $coreCLR20PackageDir -Force Get-Item "$testhostCore20PackageDir\*" | - Where-Object { $_.Name -notin ("x64", "x86", "win7-x64", "win7-x86", "testhost.deps.json", "testhost.runtimeconfig.json")} | - Copy-Item -Recurse -Destination $fullCLRTestHostDir -Force + Where-Object { $_.Name -notin ("x64", "x86", "win7-x64", "win7-x86", "testhost.deps.json", "testhost.runtimeconfig.json") } | + Copy-Item -Recurse -Destination $fullCLRTestHostDir -Force Copy-Item $standaloneTesthost $fullCLRTestHostDir -Force # For libraries that are externally published, copy the output into artifacts. These will be signed and packaged independently. @@ -737,10 +727,8 @@ function Publish-Package Write-Log "Publish-Package: Complete. {$(Get-ElapsedTime($timer))}" } -function Publish-Tests -{ - if($TPB_PublishTests) - { +function Publish-Tests { + if ($TPB_PublishTests) { Write-Log "Publish-Tests: Started." # Adding only Perf project for now @@ -764,21 +752,18 @@ function Publish-Tests } } -function Publish-PackageInternal($packagename, $framework, $output) -{ +function Publish-PackageInternal($packagename, $framework, $output) { $dotnetExe = Get-DotNetPath Invoke-Exe $dotnetExe -Arguments "publish $packagename --configuration $TPB_Configuration --framework $framework --output $output -v:minimal -p:Version=$TPB_Version -p:CIBuild=$TPB_CIBuild -p:LocalizedBuild=$TPB_LocalizedBuild" } -function Publish-PackageWithRuntimeInternal($packagename, $framework, $runtime, $selfcontained, $output) -{ +function Publish-PackageWithRuntimeInternal($packagename, $framework, $runtime, $selfcontained, $output) { $dotnetExe = Get-DotNetPath Invoke-Exe $dotnetExe -Arguments "publish $packagename --configuration $TPB_Configuration --framework $framework --runtime $runtime --self-contained $selfcontained --output $output -v:minimal -p:Version=$TPB_Version -p:CIBuild=$TPB_CIBuild -p:LocalizedBuild=$TPB_LocalizedBuild" } -function Copy-Loc-Files($sourceDir, $destinationDir, $dllName) -{ - foreach($lang in $language) { +function Copy-Loc-Files($sourceDir, $destinationDir, $dllName) { + foreach ($lang in $language) { $dllToCopy = Join-Path $sourceDir\$lang $dllName $destinationFolder = Join-Path $destinationDir $lang if (-not (Test-Path $destinationFolder)) { @@ -788,9 +773,8 @@ function Copy-Loc-Files($sourceDir, $destinationDir, $dllName) } } -function Move-Loc-Files($sourceDir, $destinationDir, $dllName) -{ - foreach($lang in $language) { +function Move-Loc-Files($sourceDir, $destinationDir, $dllName) { + foreach ($lang in $language) { $dllToCopy = Join-Path $sourceDir\$lang $dllName $destinationFolder = Join-Path $destinationDir $lang if (-not (Test-Path $destinationFolder)) { @@ -800,8 +784,7 @@ function Move-Loc-Files($sourceDir, $destinationDir, $dllName) } } -function Create-VsixPackage -{ +function Create-VsixPackage { Write-Log "Create-VsixPackage: Started." $timer = Start-Timer @@ -820,7 +803,7 @@ function Create-VsixPackage # Copy Microsoft.VisualStudio.TraceDataCollector to Extensions $traceDataCollectorPackageDirectory = Join-Path $env:TP_PACKAGES_DIR "Microsoft.VisualStudio.TraceDataCollector\$codeCoverageExternalsVersion\lib\$TPB_TargetFramework472" Copy-Item $traceDataCollectorPackageDirectory\Microsoft.VisualStudio.TraceDataCollector.dll $extensionsPackageDir -Force - if($TPB_LocalizedBuild) { + if ($TPB_LocalizedBuild) { Copy-Loc-Files $traceDataCollectorPackageDirectory $extensionsPackageDir "Microsoft.VisualStudio.TraceDataCollector.resources.dll" } @@ -841,7 +824,7 @@ function Create-VsixPackage # Copy Microsoft.VisualStudio.IO to root $codeCoverageIOPackageDirectory = Join-Path $env:TP_PACKAGES_DIR "Microsoft.VisualStudio.Coverage.IO\$codeCoverageExternalsVersion\lib\$TPB_TargetFramework451" Copy-Item $codeCoverageIOPackageDirectory\Microsoft.VisualStudio.Coverage.IO.dll $packageDir -Force - if($TPB_LocalizedBuild) { + if ($TPB_LocalizedBuild) { Copy-Loc-Files $codeCoverageIOPackageDirectory $packageDir "Microsoft.VisualStudio.Coverage.IO.resources.dll" } @@ -894,8 +877,7 @@ function Create-VsixPackage $msbuildPath = Locate-MSBuildPath # Create vsix only when msbuild is installed. - if(![string]::IsNullOrEmpty($msbuildPath)) - { + if (![string]::IsNullOrEmpty($msbuildPath)) { # Copy the vsix project to artifacts directory to modify manifest New-Item $vsixProjectDir -Type Directory -Force Copy-Item -Recurse $vsixSourceDir\* $vsixProjectDir -Force @@ -906,16 +888,14 @@ function Create-VsixPackage # Build vsix project to get TestPlatform.vsix Invoke-Exe $msbuildPath -Arguments """$vsixProjectDir\TestPlatform.csproj"" -p:Configuration=$Configuration" } - else - { + else { throw ".. Create-VsixPackage: Cannot generate vsix as msbuild.exe not found at '$msbuildPath'." } Write-Log "Create-VsixPackage: Complete. {$(Get-ElapsedTime($timer))}" } -function Create-NugetPackages -{ +function Create-NugetPackages { $timer = Start-Timer Write-Log "Create-NugetPackages: Started." @@ -1049,8 +1029,7 @@ function Create-NugetPackages Write-Log "Create-NugetPackages: Complete. {$(Get-ElapsedTime($timer))}" } -function Copy-CodeCoverage-Package-Artifacts -{ +function Copy-CodeCoverage-Package-Artifacts { # Copy TraceDataCollector to Microsoft.CodeCoverage folder. $codeCoverageExternalsVersion = ([xml](Get-Content $env:TP_ROOT_DIR\eng\Versions.props)).Project.PropertyGroup.MicrosoftInternalCodeCoverageVersion $traceDataCollectorPackagesDir = Join-Path $env:TP_PACKAGES_DIR "microsoft.visualstudio.tracedatacollector\$codeCoverageExternalsVersion\lib\$TPB_TargetFrameworkNS20" @@ -1085,13 +1064,12 @@ function Copy-CodeCoverage-Package-Artifacts Copy-Item $codeCoverageImMacosPackagesDir\x64 $microsoftCodeCoveragePackageDir\InstrumentationEngine\macos\ -Force -Recurse # Copy TraceDataCollector resource dlls - if($TPB_LocalizedBuild) { + if ($TPB_LocalizedBuild) { Copy-Loc-Files $traceDataCollectorPackagesDir $microsoftCodeCoveragePackageDir "Microsoft.VisualStudio.TraceDataCollector.resources.dll" } } -function Copy-PackageItems($packageName) -{ +function Copy-PackageItems($packageName) { # Packages published separately are copied into their own artifacts directory # E.g. src\Microsoft.TestPlatform.ObjectModel\bin\Debug\net451\* is copied # to artifacts\Debug\Microsoft.TestPlatform.ObjectModel\net451 @@ -1106,8 +1084,7 @@ function Copy-PackageItems($packageName) Copy-Item -Path $binariesDirectory -Destination $publishDirectory -Recurse -Force } -function Update-LocalizedResources -{ +function Update-LocalizedResources { $timer = Start-Timer $dotnetExe = Get-DotNetPath @@ -1125,8 +1102,7 @@ function Update-LocalizedResources # # Helper functions # -function Get-DotNetPath -{ +function Get-DotNetPath { $dotnetPath = Join-Path $env:TP_TOOLS_DIR "dotnet\dotnet.exe" if (-not (Test-Path $dotnetPath)) { Write-Error "Dotnet.exe not found at $dotnetPath. Did the dotnet cli installation succeed?" @@ -1135,33 +1111,27 @@ function Get-DotNetPath return $dotnetPath } -function Get-FullCLRPackageDirectory -{ +function Get-FullCLRPackageDirectory { return $(Join-Path $env:TP_OUT_DIR "$TPB_Configuration\$TPB_TargetFramework451\$TPB_TargetRuntime") } -function Get-FullCLRPackageDirectory45 -{ +function Get-FullCLRPackageDirectory45 { return $(Join-Path $env:TP_OUT_DIR "$TPB_Configuration\$TPB_TargetFramework45\$TPB_TargetRuntime") } -function Get-CoreCLR20PackageDirectory -{ +function Get-CoreCLR20PackageDirectory { return $(Join-Path $env:TP_OUT_DIR "$TPB_Configuration\$TPB_TargetFrameworkCore20") } -function Get-CoreCLR10PackageDirectory -{ +function Get-CoreCLR10PackageDirectory { return $(Join-Path $env:TP_OUT_DIR "$TPB_Configuration\$TPB_TargetFrameworkCore10") } -function Get-CoreCLR20TestHostPackageDirectory -{ +function Get-CoreCLR20TestHostPackageDirectory { return $(Join-Path $env:TP_OUT_DIR "$TPB_Configuration\$TPB_TargetFrameworkCore20\TestHost") } -function Locate-MSBuildPath -{ +function Locate-MSBuildPath { $vsInstallPath = Locate-VsInstallPath $msbuildPath = Get-ChildItem (Join-Path -path $vsInstallPath -childPath "MSBuild\*\Bin\MSBuild.exe") @@ -1176,66 +1146,61 @@ function Locate-MSBuildPath return $msBuild.FullName } -function Locate-VsInstallPath -{ - $vswhere = Join-Path -path $env:TP_PACKAGES_DIR -ChildPath "vswhere\$env:VSWHERE_VERSION\tools\vswhere.exe" - if (!(Test-Path -path $vswhere)) { - throw "Unable to locate vswhere in path '$vswhere'." - } - - Write-Verbose "Using '$vswhere' to locate VS installation path." - - $requiredPackageIds = @("Microsoft.Component.MSBuild", "Microsoft.Net.Component.4.6.TargetingPack", "Microsoft.VisualStudio.Component.VSSDK") - Write-Verbose "VSInstallation requirements : $requiredPackageIds" - - Try - { - if ($TPB_CIBuild) { - $vsInstallPath = Invoke-Exe $vswhere -CaptureOutput -Arguments "-version (15.0 -products * -requires $requiredPackageIds -property installationPath" - } - else { - # Allow using pre release versions of VS for dev builds - $vsInstallPath = Invoke-Exe $vswhere -CaptureOutput -Arguments "-version (15.0 -prerelease -products * -requires $requiredPackageIds -property installationPath" - } - } - Catch [System.Management.Automation.MethodInvocationException] - { - throw "Failed to find VS installation with requirements: $requiredPackageIds" - } - - if ($null -eq $vsInstallPath -or 0 -eq @($vsInstallPath).Count) { +function Locate-VsInstallPath { + $vswhere = Join-Path -path $env:TP_PACKAGES_DIR -ChildPath "vswhere\$env:VSWHERE_VERSION\tools\vswhere.exe" + if (!(Test-Path -path $vswhere)) { + throw "Unable to locate vswhere in path '$vswhere'." + } + + Write-Verbose "Using '$vswhere' to locate VS installation path." + + $requiredPackageIds = @("Microsoft.Component.MSBuild", "Microsoft.Net.Component.4.6.TargetingPack", "Microsoft.VisualStudio.Component.VSSDK") + Write-Verbose "VSInstallation requirements : $requiredPackageIds" + + Try { + if ($TPB_CIBuild) { + $vsInstallPath = Invoke-Exe $vswhere -CaptureOutput -Arguments "-version (15.0 -products * -requires $requiredPackageIds -property installationPath" + } + else { + # Allow using pre release versions of VS for dev builds + $vsInstallPath = Invoke-Exe $vswhere -CaptureOutput -Arguments "-version (15.0 -prerelease -products * -requires $requiredPackageIds -property installationPath" + } + } + Catch [System.Management.Automation.MethodInvocationException] { throw "Failed to find VS installation with requirements: $requiredPackageIds" - } - else { + } + + if ($null -eq $vsInstallPath -or 0 -eq @($vsInstallPath).Count) { + throw "Failed to find VS installation with requirements: $requiredPackageIds" + } + else { Write-Verbose "Found VS installation with requirements '$($requiredPackageIds -join "','")' : '$($vsInstallPath -join "','")'." - } + } - $vsPath = $vsInstallPath | Select-Object -First 1 - Write-Verbose "VSInstallPath is : $vsPath" - return $vsPath + $vsPath = $vsInstallPath | Select-Object -First 1 + Write-Verbose "VSInstallPath is : $vsPath" + return $vsPath } -function Update-VsixVersion($vsixProjectDir) -{ +function Update-VsixVersion($vsixProjectDir) { Write-Log "Update-VsixVersion: Started." $vsixVersion = $Version # Build number comes in the form 20170111-01(yyyymmdd-buildNoOfThatDay) # So Version of the vsix will be 15.1.0.2017011101 $vsixVersionSuffix = $BuildNumber.Split("-"); - if($vsixVersionSuffix.Length -ige 2) { + if ($vsixVersionSuffix.Length -ige 2) { $vsixVersion = "$vsixVersion.$($vsixVersionSuffix[0])$($vsixVersionSuffix[1])" } - $manifestContentWithVersion = Get-Content "$vsixProjectDir\source.extension.vsixmanifest" -raw | ForEach-Object {$_.ToString().Replace("`$version`$", "$vsixVersion") } + $manifestContentWithVersion = Get-Content "$vsixProjectDir\source.extension.vsixmanifest" -raw | ForEach-Object { $_.ToString().Replace("`$version`$", "$vsixVersion") } Set-Content -path "$vsixProjectDir\source.extension.vsixmanifest" -value $manifestContentWithVersion Write-Log "Update-VsixVersion: Completed." } -function Generate-Manifest ($PackageFolder) -{ - $packagesFolderName = [System.IO.Path]::GetFileName($PackageFolder) +function Generate-Manifest ($PackageFolder) { + $packagesFolderName = [System.IO.Path]::GetFileName($PackageFolder) Write-Log "Generate-Manifest ($packagesFolderName): Started." $generateManifestPath = Join-Path $env:TP_ROOT_DIR "scripts\build\GenerateManifest.proj" @@ -1246,8 +1211,7 @@ function Generate-Manifest ($PackageFolder) Write-Log "Generate-Manifest ($packagesFolderName): Completed." } -function Build-SpecificProjects -{ +function Build-SpecificProjects { Write-Log "Build-SpecificProjects: Started for pattern: $ProjectNamePatterns" # FrameworksAndOutDirs format ("", ""). $FrameworksAndOutDirs = ( @@ -1263,18 +1227,18 @@ function Build-SpecificProjects # Get projects to build. Get-ChildItem -Recurse -Path $env:TP_ROOT_DIR -Include *.csproj | ForEach-Object { foreach ($ProjectNamePattern in $ProjectNamePatterns) { - if($_.FullName -match $ProjectNamePattern) { - $ProjectsToBuild += ,"$_" + if ($_.FullName -match $ProjectNamePattern) { + $ProjectsToBuild += , "$_" } } } - if( $null -eq $ProjectsToBuild){ + if ( $null -eq $ProjectsToBuild) { Write-Error "No csproj name match for given pattern: $ProjectNamePatterns" } # Build Projects. - foreach($ProjectToBuild in $ProjectsToBuild) { + foreach ($ProjectToBuild in $ProjectsToBuild) { Write-Log "Building Project $ProjectToBuild" # Restore and Build $output = Invoke-Exe $dotnetPath -Arguments "restore $ProjectToBuild" @@ -1289,13 +1253,13 @@ function Build-SpecificProjects # Copy artifacts $ProjectDir = [System.IO.Path]::GetDirectoryName($ProjectToBuild) - foreach($FrameworkAndOutDir in $FrameworksAndOutDirs) { + foreach ($FrameworkAndOutDir in $FrameworksAndOutDirs) { $fromDir = $([System.IO.Path]::Combine($ProjectDir, "bin", $TPB_Configuration, $FrameworkAndOutDir[0])) $toDir = $([System.IO.Path]::Combine($env:TP_OUT_DIR, $TPB_Configuration, $FrameworkAndOutDir[1])) - if ( Test-Path $fromDir){ + if ( Test-Path $fromDir) { Write-Log "Copying artifacts from $fromDir to $toDir" Get-ChildItem $fromDir | ForEach-Object { - if(-not ($_.PSIsContainer)) { + if (-not ($_.PSIsContainer)) { Copy-Item $_.FullName $toDir } } @@ -1304,8 +1268,7 @@ function Build-SpecificProjects } } -if ($ProjectNamePatterns.Count -ne 0) -{ +if ($ProjectNamePatterns.Count -ne 0) { # Build Specific projects. Build-SpecificProjects Exit @@ -1352,7 +1315,7 @@ if ($ProjectNamePatterns.Count -ne 0) # } if ($Force -or $Steps -contains "PrepareAcceptanceTests") { - #Publish-PatchedDotnet + #Publish-PatchedDotnet Invoke-TestAssetsBuild #Publish-Tests } @@ -1360,7 +1323,8 @@ if ($Force -or $Steps -contains "PrepareAcceptanceTests") { if ($Script:ScriptFailed) { Write-Log "Build failed. {$(Get-ElapsedTime($timer))}" -Level "Error" Exit 1 -} else { +} +else { Write-Log "Build succeeded. {$(Get-ElapsedTime($timer))}" Exit 0 } diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/ExecutionTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/ExecutionTests.cs index 57be416dd5..e10173e02a 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/ExecutionTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/ExecutionTests.cs @@ -17,7 +17,6 @@ namespace Microsoft.TestPlatform.AcceptanceTests; public class ExecutionTests : AcceptanceTestBase { // TODO: It looks like the first 3 tests would be useful to multiply by all 3 test frameworks, should we make the test even more generic, or duplicate them? - [TestMethod] [MSTestCompatibilityDataSource(InProcess = true)] public void RunMultipleTestAssemblies(RunnerInfo runnerInfo, TesthostInfo testhostInfo, MSTestInfo msTestInfo) @@ -33,15 +32,15 @@ public void RunMultipleTestAssemblies(RunnerInfo runnerInfo, TesthostInfo testho } [TestMethod] - [TestPlatformCompatibilityDataSource("netcoreapp2.1", "LegacyStable", "netcoreapp2.1", "Latest", "LatestPreview")] + //[TestPlatformCompatibilityDataSource("netcoreapp2.1", "LegacyStable", "netcoreapp2.1", "Latest", "LatestPreview")] // [TestPlatformCompatibilityDataSource("netcoreapp2.1", LATEST_TO_LEGACY, "netcoreapp2.1", LATEST_TO_LEGACY, LATESTPREVIEW_TO_LEGACY)] - //[TestPlatformCompatibilityDataSource()] + [TestPlatformCompatibilityDataSource( WithInProcess = true)] public void RunMultipleTestAssemblies223(RunnerInfo runnerInfo, VSTestConsoleInfo consoleInfo, TesthostInfo testhostInfo, MSTestInfo msTestInfo) { SetTestEnvironment(_testEnvironment, runnerInfo, consoleInfo); - var assemblyPaths = BuildMultipleAssemblyPath(testhostInfo, msTestInfo, "SimpleTestProject.dll", "SimpleTestProject2.dll").Trim('\"'); + var assemblyPaths = BuildMultipleAssemblyPath(testhostInfo, msTestInfo, "SimpleTestProject.dll", "MSTestProject2.dll").Trim('\"'); InvokeVsTestForExecution(assemblyPaths, testAdapterPath: null, FrameworkArgValue, string.Empty); diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/MSTestCompatibilityDataSource.cs b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/MSTestCompatibilityDataSource.cs index c9e40513b0..388217cea8 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/MSTestCompatibilityDataSource.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/MSTestCompatibilityDataSource.cs @@ -15,7 +15,7 @@ namespace Microsoft.TestPlatform.AcceptanceTests; public sealed class MSTestCompatibilityDataSource : TestDataSource { - private static XmlDocument? _depsXml; + private static XmlDocument? s_depsXml; private readonly string[] _runnerFrameworks; private readonly string[] _targetFrameworks; private readonly string[] _msTestVersions; @@ -98,8 +98,8 @@ private MSTestInfo GetMSTestInfo(string msTestVersion) private static XmlDocument GetDependenciesXml() { - if (_depsXml != null) - return _depsXml; + if (s_depsXml != null) + return s_depsXml; var depsXmlPath = Path.Combine(IntegrationTestEnvironment.TestPlatformRootDirectory, "scripts", "build", "TestPlatform.Dependencies.props"); var fileStream = File.OpenRead(depsXmlPath); @@ -107,7 +107,7 @@ private static XmlDocument GetDependenciesXml() var depsXml = new XmlDocument(); depsXml.Load(xmlTextReader); - _depsXml = depsXml; + s_depsXml = depsXml; return depsXml; } } diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/TestPlatformCompatibilityDataSource.cs b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/TestPlatformCompatibilityDataSource.cs index 63d0a7105e..262b3f1b85 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/TestPlatformCompatibilityDataSource.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/TestPlatformCompatibilityDataSource.cs @@ -51,6 +51,20 @@ public TestPlatformCompatibilityDataSource( public bool DebugDataCollector { get; set; } public bool NoDefaultBreakpoints { get; set; } = true; + /// + /// Add run for in-process using the selected .NET Framework runners, and and all selected adapters. + /// + public bool WithInProcess { get; set; } + + public bool WithEveryVersionOfRunner { get; set; } = true; + + public bool WithEveryVersionOfHost { get; set; } = true; + + public bool WithEveryVersionOfAdapter { get; set; } = true; + + public bool WithOlderConfigurations { get; set; } = true; + + public string? BeforeFeature { get; set; } public string? AfterFeature { get; set; } public string? BeforeAdapterFeature { get; set; } @@ -59,14 +73,21 @@ public TestPlatformCompatibilityDataSource( public override void CreateData(MethodInfo methodInfo) { var dataRows = new List<(RunnerInfo runnerInfo, VSTestConsoleInfo vstestConsoleInfo, TesthostInfo testhostInfo, MSTestInfo mstestInfo)>(); - AddEveryVersionOfRunner(dataRows); - // with every version of host - AddEveryVersionOfHost(dataRows); + if (WithEveryVersionOfRunner) + AddEveryVersionOfRunner(dataRows); - AddEveryVersionOfAdapter(dataRows); + if (WithEveryVersionOfHost) + AddEveryVersionOfHost(dataRows); - AddOlderConfigurations(dataRows); + if (WithEveryVersionOfAdapter) + AddEveryVersionOfAdapter(dataRows); + + if (WithOlderConfigurations) + AddOlderConfigurations(dataRows); + + if (WithInProcess) + AddInProcess(dataRows); var c = dataRows.Count(); @@ -104,6 +125,28 @@ public override void CreateData(MethodInfo methodInfo) } } + private void AddInProcess(List<(RunnerInfo runnerInfo, VSTestConsoleInfo vstestConsoleInfo, TesthostInfo testhostInfo, MSTestInfo mstestInfo)> dataRows) + { + foreach (var runnerFramework in _runnerFrameworks) + { + if (!runnerFramework.StartsWith("net4")) + { + continue; + } + + foreach (var runnerVersion in _runnerVersions) + { + foreach (var adapter in _adapters) + { + foreach (var adapterVersion in _adapterVersions) + { + AddRow(dataRows, runnerVersion, runnerFramework, runnerVersion, runnerFramework, adapter, adapterVersion, inIsolation: false); + } + } + } + } + } + private void AddOlderConfigurations(List<(RunnerInfo, VSTestConsoleInfo, TesthostInfo, MSTestInfo)> dataRows) { // Older configurations where the runner, host and adapter version are the same. @@ -119,7 +162,7 @@ private void AddOlderConfigurations(List<(RunnerInfo, VSTestConsoleInfo, Testhos foreach (var adapter in _adapters) { var adapterVersion = runnerVersion; - AddRow(dataRows, runnerVersion, runnerFramework, hostVersion, hostFramework, adapter, adapterVersion); + AddRow(dataRows, runnerVersion, runnerFramework, hostVersion, hostFramework, adapter, adapterVersion, inIsolation: true); } } } @@ -142,7 +185,7 @@ private void AddEveryVersionOfAdapter(List<(RunnerInfo, VSTestConsoleInfo, Testh // We already used the newest when adding combination with every runner foreach (var adapterVersion in _adapterVersions.Skip(1)) { - AddRow(dataRows, runnerVersion, runnerFramework, hostVersion, hostFramework, adapter, adapterVersion); + AddRow(dataRows, runnerVersion, runnerFramework, hostVersion, hostFramework, adapter, adapterVersion, inIsolation: true); } } } @@ -169,7 +212,7 @@ private void AddEveryVersionOfHost(List<(RunnerInfo, VSTestConsoleInfo, Testhost { // use the newest var adapterVersion = _adapterVersions[0]; - AddRow(dataRows, runnerVersion, runnerFramework, hostVersion, hostFramework, adapter, adapterVersion); + AddRow(dataRows, runnerVersion, runnerFramework, hostVersion, hostFramework, adapter, adapterVersion, inIsolation: true); } } } @@ -192,7 +235,7 @@ private void AddEveryVersionOfRunner(List<(RunnerInfo, VSTestConsoleInfo, Testho { // use the newest var adapterVersion = _adapterVersions[0]; - AddRow(dataRows, runnerVersion, runnerFramework, hostVersion, hostFramework, adapter, adapterVersion); + AddRow(dataRows, runnerVersion, runnerFramework, hostVersion, hostFramework, adapter, adapterVersion, inIsolation: true); } } } @@ -200,9 +243,9 @@ private void AddEveryVersionOfRunner(List<(RunnerInfo, VSTestConsoleInfo, Testho } private void AddRow(List<(RunnerInfo, VSTestConsoleInfo, TesthostInfo, MSTestInfo)> dataRows, - string runnerVersion, string runnerFramework, string hostVersion, string hostFramework, string adapter, string adapterVersion) + string runnerVersion, string runnerFramework, string hostVersion, string hostFramework, string adapter, string adapterVersion, bool inIsolation) { - RunnerInfo runnerInfo = GetRunnerInfo(runnerFramework, hostFramework, inIsolation: true); + RunnerInfo runnerInfo = GetRunnerInfo(runnerFramework, hostFramework, inIsolation); var vstestConsoleInfo = GetVSTestConsoleInfo(runnerVersion, runnerInfo); var testhostInfo = TesthostCompatibilityDataSource.GetTesthostInfo(hostVersion); var mstestInfo = GetMSTestInfo(adapterVersion); diff --git a/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestBase.cs b/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestBase.cs index 1e84f3b094..f979ad4906 100644 --- a/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestBase.cs +++ b/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestBase.cs @@ -7,7 +7,6 @@ using System.Diagnostics; using System.IO; using System.Linq; -using System.Runtime.CompilerServices; using System.Text; using System.Text.RegularExpressions; using System.Xml; diff --git a/test/TestAssets/MSTestProject2/MSTestProject2.csproj b/test/TestAssets/MSTestProject2/MSTestProject2.csproj new file mode 100644 index 0000000000000000000000000000000000000000..18ce8041acd97ea3ce45df23fd308ba411dedd89 GIT binary patch literal 2194 zcmd6pPjAye5XIjaiSJ-4q|!qTEq^Y!sY)m)hoXv9oN_U))0U>GWhYR+JoGoaacw7Q zoPv;06ld+3dHd$=%s9V)McUVmt~64h?@E=c(1orw)}A)BqfjsS4X})r@cb0(SZB^U z)kH&WCcz_6pQuY0aYS_DtOXuJtf4;XP$&F)uZKwP9J zpl`G4!7w8tC!*lK%le9s@eGJgc{~o#6POxtg5l|_6TsQH~v83*q)y1+LU%DLsB&)8@!A_{m zLl}%&ROwKyF + /// The unit test 1. + /// + [TestClass] + public class UnitTest1 + { + /// + /// The passing test. + /// + [Priority(2)] + [TestMethod] + public void PassingTest() + { + Assert.AreEqual(2, 2); + } + + /// + /// The failing test. + /// + [TestCategory("CategoryA")] + [Priority(3)] + [TestMethod] + public void FailingTest() + { +#if NETFRAMEWORK + // current App domain should be write to file to test DisableAppDomain acceptance test. + var appDomainFilePath = Environment.GetEnvironmentVariable("TEST_ASSET_APPDOMAIN_TEST_PATH") ?? Path.Combine(Path.GetTempPath(), "appdomain_test.txt"); + File.WriteAllText(appDomainFilePath, "AppDomain FriendlyName: " + AppDomain.CurrentDomain.FriendlyName); +#endif + Assert.AreEqual(2, 3); + } + + /// + /// The skipping test. + /// + [Ignore] + [TestMethod] + public void SkippingTest() + { + } + } +} diff --git a/test/TestAssets/TestAssets.sln b/test/TestAssets/TestAssets.sln index fb41d58e19..c5d6cba585 100644 --- a/test/TestAssets/TestAssets.sln +++ b/test/TestAssets/TestAssets.sln @@ -94,7 +94,9 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "AttachmentProcessorDataColl EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "ArchitectureSwitch", "ArchitectureSwitch\ArchitectureSwitch.csproj", "{452352E1-71CA-436E-8165-F284EE36C924}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "SimpleTestProjectMessedUpTargetFramework", "SimpleTestProjectMessedUpTargetFramework\SimpleTestProjectMessedUpTargetFramework.csproj", "{08C44607-EB80-4EE5-927D-08C34AA277AF}" +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "SimpleTestProjectMessedUpTargetFramework", "SimpleTestProjectMessedUpTargetFramework\SimpleTestProjectMessedUpTargetFramework.csproj", "{08C44607-EB80-4EE5-927D-08C34AA277AF}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "MSTestProject2", "MSTestProject2\MSTestProject2.csproj", "{10AA955C-B412-41A8-899F-8609AAE19F61}" EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution @@ -634,6 +636,18 @@ Global {08C44607-EB80-4EE5-927D-08C34AA277AF}.Release|x64.Build.0 = Release|Any CPU {08C44607-EB80-4EE5-927D-08C34AA277AF}.Release|x86.ActiveCfg = Release|Any CPU {08C44607-EB80-4EE5-927D-08C34AA277AF}.Release|x86.Build.0 = Release|Any CPU + {10AA955C-B412-41A8-899F-8609AAE19F61}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {10AA955C-B412-41A8-899F-8609AAE19F61}.Debug|Any CPU.Build.0 = Debug|Any CPU + {10AA955C-B412-41A8-899F-8609AAE19F61}.Debug|x64.ActiveCfg = Debug|Any CPU + {10AA955C-B412-41A8-899F-8609AAE19F61}.Debug|x64.Build.0 = Debug|Any CPU + {10AA955C-B412-41A8-899F-8609AAE19F61}.Debug|x86.ActiveCfg = Debug|Any CPU + {10AA955C-B412-41A8-899F-8609AAE19F61}.Debug|x86.Build.0 = Debug|Any CPU + {10AA955C-B412-41A8-899F-8609AAE19F61}.Release|Any CPU.ActiveCfg = Release|Any CPU + {10AA955C-B412-41A8-899F-8609AAE19F61}.Release|Any CPU.Build.0 = Release|Any CPU + {10AA955C-B412-41A8-899F-8609AAE19F61}.Release|x64.ActiveCfg = Release|Any CPU + {10AA955C-B412-41A8-899F-8609AAE19F61}.Release|x64.Build.0 = Release|Any CPU + {10AA955C-B412-41A8-899F-8609AAE19F61}.Release|x86.ActiveCfg = Release|Any CPU + {10AA955C-B412-41A8-899F-8609AAE19F61}.Release|x86.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE From 1d22cd99f8ec2f442a7b0d87c366e802ee5ac58c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Jare=C5=A1?= Date: Wed, 16 Mar 2022 13:04:33 +0100 Subject: [PATCH 053/112] Remove quote path trimming. --- build.cmd | 1 + scripts/build.ps1 | 13 ++-- sln.cmd | 14 ++++ .../BlameDataCollectorTests.cs | 6 +- .../DataCollectionTests.cs | 8 +-- .../DebugAssertTests.cs | 4 +- .../DiscoveryTests.cs | 2 +- .../DotnetTestTests.cs | 4 +- .../ExecutionTests.cs | 40 ++++++----- .../ExecutionThreadApartmentStateTests.cs | 8 +-- .../FilePatternParserTests.cs | 2 +- .../MultitargetingTestHostTests.cs | 4 +- .../PortableNugetPackageTests.cs | 2 +- .../RunsettingsTests.cs | 2 +- .../SelfContainedAppTests.cs | 4 +- .../TestPlatformNugetPackageTests.cs | 4 +- .../Hosting/DefaultTestHostManagerTests.cs | 4 +- .../IntegrationTestBase.cs | 64 ++++++++++-------- .../IntegrationTestEnvironment.cs | 6 +- .../MSTestProject1/MSTestProject1.csproj | 28 ++++++++ test/TestAssets/MSTestProject1/UnitTest1.cs | 28 ++++++++ .../MSTestProject2/MSTestProject2.csproj | Bin 2194 -> 1063 bytes test/TestAssets/MSTestProject2/UnitTest1.cs | 59 +++++----------- test/TestAssets/TestAssets.sln | 22 +++++- 24 files changed, 203 insertions(+), 126 deletions(-) create mode 100644 sln.cmd create mode 100644 test/TestAssets/MSTestProject1/MSTestProject1.csproj create mode 100644 test/TestAssets/MSTestProject1/UnitTest1.cs diff --git a/build.cmd b/build.cmd index f2a9a38318..51620a6421 100644 --- a/build.cmd +++ b/build.cmd @@ -3,4 +3,5 @@ REM Copyright (c) Microsoft. All rights reserved. powershell -ExecutionPolicy Bypass -NoProfile -NoLogo -Command "%~dp0scripts\build.ps1 %*; exit $LastExitCode;" + if %errorlevel% neq 0 exit /b %errorlevel% diff --git a/scripts/build.ps1 b/scripts/build.ps1 index 78b06fd19e..23c5872736 100644 --- a/scripts/build.ps1 +++ b/scripts/build.ps1 @@ -231,9 +231,10 @@ function Invoke-TestAssetsBuild { } - # Build with multiple versions of MSTest. + # Build with multiple versions of MSTest. The projects are directly in the root. + # The folder structure in VS is not echoed in the TestAssets directory. $projects = @( - "$env:TP_ROOT_DIR\test\TestAssets\SimpleTestProject\SimpleTestProject.csproj" + "$env:TP_ROOT_DIR\test\TestAssets\MSTestProject1\MSTestProject1.csproj" "$env:TP_ROOT_DIR\test\TestAssets\MSTestProject2\MSTestProject2.csproj" # Don't use this one, it does not use the variables for mstest and test sdk. # "$env:TP_ROOT_DIR\test\TestAssets\SimpleTestProject2\SimpleTestProject2.csproj" @@ -275,7 +276,7 @@ function Invoke-TestAssetsBuild { $dirMSTestVersion = $mstestVersion -replace "\[|\]" $dirMSTestPropertyName = $propertyName -replace "Framework" -replace "Version" - Invoke-Exe $dotnetExe -Arguments "build $project --configuration $TPB_Configuration -v:minimal -p:CIBuild=$TPB_CIBuild -p:LocalizedBuild=$TPB_LocalizedBuild -p:MSTestFrameworkVersion=$mstestVersion -p:MSTestAdapterVersion=$mstestVersion -p:NETTestSdkVersion=$netTestSdkVersion -p:BaseOutputPath=""bin\$dirNetTestSdkPropertyName-$dirNetTestSdkVersion\$dirMSTestPropertyName-$dirMSTestVersion\\"" -bl:""$env:TP_OUT_DIR\log\$Configuration\perm.binlog""" + Invoke-Exe $dotnetExe -Arguments "build $project --configuration $TPB_Configuration --no-restore -v:minimal -p:CIBuild=$TPB_CIBuild -p:LocalizedBuild=$TPB_LocalizedBuild -p:MSTestFrameworkVersion=$mstestVersion -p:MSTestAdapterVersion=$mstestVersion -p:NETTestSdkVersion=$netTestSdkVersion -p:BaseOutputPath=""bin\$dirNetTestSdkPropertyName-$dirNetTestSdkVersion\$dirMSTestPropertyName-$dirMSTestVersion\\"" -bl:""$env:TP_OUT_DIR\log\$Configuration\perm.binlog""" } } } @@ -1282,9 +1283,9 @@ if ($ProjectNamePatterns.Count -ne 0) { # Write-Log "Test platform build variables: " # Get-Variable | Where-Object -FilterScript { $_.Name.StartsWith("TPB_") } | Format-Table -# if ($Force -or $Steps -contains "InstallDotnet") { -# Install-DotNetCli -# } +if ($Force -or $Steps -contains "InstallDotnet") { + Install-DotNetCli +} # if ($Force -or $Steps -contains "Restore") { # Clear-Package diff --git a/sln.cmd b/sln.cmd new file mode 100644 index 0000000000..8ba8ab0bb9 --- /dev/null +++ b/sln.cmd @@ -0,0 +1,14 @@ +@echo off + +REM Copyright (c) Microsoft. All rights reserved. + +REM set DOTNET_ROOT to point at the locally installed dotnet, +REM to avoid problems with .NET Core 2.1 that we run our tests and tools against, +REM but that is regularly corrupted. + +set DOTNET_ROOT=%~dp0tools\dotnet +set DOTNET_ROOT(x86)=%~dp0tools\dotnet_x86 + +start %~dp0TestPlatform.sln + +if %errorlevel% neq 0 exit /b %errorlevel% diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/BlameDataCollectorTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/BlameDataCollectorTests.cs index 671a91618d..e9e6af25ee 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/BlameDataCollectorTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/BlameDataCollectorTests.cs @@ -64,7 +64,7 @@ public void BlameDataCollectorShouldOutputDumpFile(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - var assemblyPaths = BuildMultipleAssemblyPath("SimpleTestProject3.dll").Trim('\"'); + var assemblyPaths = BuildMultipleAssemblyPath("SimpleTestProject3.dll"); var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, string.Empty, runnerInfo.InIsolationValue); arguments = string.Concat(arguments, $" /Blame:CollectDump"); arguments = string.Concat(arguments, $" /ResultsDirectory:{TempDirectory.Path}"); @@ -89,7 +89,7 @@ public void BlameDataCollectorShouldNotOutputDumpFileWhenNoCrashOccurs(RunnerInf { SetTestEnvironment(_testEnvironment, runnerInfo); - var assemblyPaths = BuildMultipleAssemblyPath("SimpleTestProject.dll").Trim('\"'); + var assemblyPaths = BuildMultipleAssemblyPath("SimpleTestProject.dll"); var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, string.Empty, runnerInfo.InIsolationValue); arguments = string.Concat(arguments, $" /Blame:CollectDump"); arguments = string.Concat(arguments, $" /ResultsDirectory:{TempDirectory.Path}"); @@ -114,7 +114,7 @@ public void BlameDataCollectorShouldOutputDumpFileWhenNoCrashOccursButCollectAlw { SetTestEnvironment(_testEnvironment, runnerInfo); - var assemblyPaths = BuildMultipleAssemblyPath("SimpleTestProject.dll").Trim('\"'); + var assemblyPaths = BuildMultipleAssemblyPath("SimpleTestProject.dll"); var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, string.Empty, runnerInfo.InIsolationValue); arguments = string.Concat(arguments, $" /Blame:CollectDump;CollectAlways=True"); arguments = string.Concat(arguments, $" /ResultsDirectory:{TempDirectory.Path}"); diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/DataCollectionTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/DataCollectionTests.cs index b7907cbdd6..2486a605ff 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/DataCollectionTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/DataCollectionTests.cs @@ -28,7 +28,7 @@ public void ExecuteTestsWithDataCollection(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - var assemblyPaths = BuildMultipleAssemblyPath("SimpleTestProject2.dll").Trim('\"'); + var assemblyPaths = BuildMultipleAssemblyPath("SimpleTestProject2.dll"); string runSettings = GetRunsettingsFilePath(TempDirectory.Path); string diagFileName = Path.Combine(TempDirectory.Path, "diaglog.txt"); var extensionsPath = Path.Combine( @@ -58,7 +58,7 @@ public void ExecuteTestsWithDataCollectionUsingCollectArgument(RunnerInfo runner { SetTestEnvironment(_testEnvironment, runnerInfo); - var assemblyPaths = BuildMultipleAssemblyPath("SimpleTestProject2.dll").Trim('\"'); + var assemblyPaths = BuildMultipleAssemblyPath("SimpleTestProject2.dll"); string diagFileName = Path.Combine(TempDirectory.Path, "diaglog.txt"); var extensionsPath = Path.Combine( _testEnvironment.TestAssetsPath, @@ -113,8 +113,8 @@ public void DataCollectorAttachmentProcessor(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - var assemblyPath = BuildMultipleAssemblyPath("SimpleTestProject.dll").Trim('\"'); - var secondAssemblyPath = BuildMultipleAssemblyPath("SimpleTestProject2.dll").Trim('\"'); + var assemblyPath = BuildMultipleAssemblyPath("SimpleTestProject.dll"); + var secondAssemblyPath = BuildMultipleAssemblyPath("SimpleTestProject2.dll"); string runSettings = GetRunsettingsFilePath(TempDirectory.Path); string diagFileName = Path.Combine(TempDirectory.Path, "diaglog.txt"); var extensionsPath = Path.Combine( diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/DebugAssertTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/DebugAssertTests.cs index fba250c5a2..a03280f37c 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/DebugAssertTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/DebugAssertTests.cs @@ -20,13 +20,13 @@ public void RunningTestWithAFailingDebugAssertDoesNotCrashTheHostingProcess(Runn // is to not crash the process when we are running in debug, and debugger is attached SetTestEnvironment(_testEnvironment, runnerInfo); - var assemblyPath = BuildMultipleAssemblyPath("CrashingOnDebugAssertTestProject.dll").Trim('\"'); + var assemblyPath = BuildMultipleAssemblyPath("CrashingOnDebugAssertTestProject.dll"); var arguments = PrepareArguments(assemblyPath, null, null, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: TempDirectory.Path); InvokeVsTest(arguments); // this will have failed tests when our trace listener works and crash the testhost process when it does not // because crashing processes is what a failed Debug.Assert does by default, unless you have a debugger attached - ValidateSummaryStatus(passedTestsCount: 4, failedTestsCount: 4, 0); + ValidateSummaryStatus(passed: 4, failed: 4, 0); StringAssert.Contains(StdOut, "threw exception: Microsoft.VisualStudio.TestPlatform.TestHost.DebugAssertException:"); } } diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/DiscoveryTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/DiscoveryTests.cs index 2f6087aa17..fa9bb8627b 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/DiscoveryTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/DiscoveryTests.cs @@ -38,7 +38,7 @@ public void DiscoverAllTests(RunnerInfo runnerInfo) public void MultipleSourcesDiscoverAllTests(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - var assemblyPaths = BuildMultipleAssemblyPath("SimpleTestProject.dll", "SimpleTestProject2.dll").Trim('\"'); + var assemblyPaths = BuildMultipleAssemblyPath("SimpleTestProject.dll", "SimpleTestProject2.dll"); var listOfTests = new[] { "SampleUnitTestProject.UnitTest1.PassingTest", "SampleUnitTestProject.UnitTest1.FailingTest", diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/DotnetTestTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/DotnetTestTests.cs index 4fb0d68eca..3e50b7c219 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/DotnetTestTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/DotnetTestTests.cs @@ -38,7 +38,7 @@ public void RunDotnetTestWithDll(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - var assemblyPath = BuildMultipleAssemblyPath("SimpleTestProject.dll").Trim('\"'); + var assemblyPath = BuildMultipleAssemblyPath("SimpleTestProject.dll"); InvokeDotnetTest($@"{assemblyPath} --logger:""Console;Verbosity=normal"""); // ensure our dev version is used @@ -70,7 +70,7 @@ public void PassInlineSettingsToDll(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - var assemblyPath = BuildMultipleAssemblyPath("ParametrizedTestProject.dll").Trim('\"'); + var assemblyPath = BuildMultipleAssemblyPath("ParametrizedTestProject.dll"); InvokeDotnetTest($@"{assemblyPath} --logger:""Console;Verbosity=normal"" -- TestRunParameters.Parameter(name=\""weburl\"", value=\""http://localhost//def\"")"); ValidateSummaryStatus(1, 0, 0); diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/ExecutionTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/ExecutionTests.cs index e10173e02a..14daacdf20 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/ExecutionTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/ExecutionTests.cs @@ -19,11 +19,11 @@ public class ExecutionTests : AcceptanceTestBase // TODO: It looks like the first 3 tests would be useful to multiply by all 3 test frameworks, should we make the test even more generic, or duplicate them? [TestMethod] [MSTestCompatibilityDataSource(InProcess = true)] - public void RunMultipleTestAssemblies(RunnerInfo runnerInfo, TesthostInfo testhostInfo, MSTestInfo msTestInfo) + public void RunMultipleTestAssemblies(RunnerInfo runnerInfo, MSTestInfo msTestInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - var assemblyPaths = BuildMultipleAssemblyPath(msTestInfo, "SimpleTestProject.dll", "SimpleTestProject2.dll").Trim('\"'); + var assemblyPaths = BuildMultipleAssemblyPath(msTestInfo, "SimpleTestProject.dll", "SimpleTestProject2.dll"); InvokeVsTestForExecution(assemblyPaths, testAdapterPath: null, FrameworkArgValue, string.Empty); @@ -32,19 +32,17 @@ public void RunMultipleTestAssemblies(RunnerInfo runnerInfo, TesthostInfo testho } [TestMethod] - //[TestPlatformCompatibilityDataSource("netcoreapp2.1", "LegacyStable", "netcoreapp2.1", "Latest", "LatestPreview")] - // [TestPlatformCompatibilityDataSource("netcoreapp2.1", LATEST_TO_LEGACY, "netcoreapp2.1", LATEST_TO_LEGACY, LATESTPREVIEW_TO_LEGACY)] - [TestPlatformCompatibilityDataSource( WithInProcess = true)] + [TestPlatformCompatibilityDataSource(WithInProcess = true)] - public void RunMultipleTestAssemblies223(RunnerInfo runnerInfo, VSTestConsoleInfo consoleInfo, TesthostInfo testhostInfo, MSTestInfo msTestInfo) + public void RunTestsFromMultipleMSTestAssemblies(RunnerInfo runnerInfo, VSTestConsoleInfo consoleInfo, TesthostInfo testhostInfo, MSTestInfo msTestInfo) { SetTestEnvironment(_testEnvironment, runnerInfo, consoleInfo); - var assemblyPaths = BuildMultipleAssemblyPath(testhostInfo, msTestInfo, "SimpleTestProject.dll", "MSTestProject2.dll").Trim('\"'); + var assemblyPaths = BuildMultipleAssemblyPath(testhostInfo, msTestInfo, "MSTestProject1.dll", "MSTestProject2.dll"); InvokeVsTestForExecution(assemblyPaths, testAdapterPath: null, FrameworkArgValue, string.Empty); - ValidateSummaryStatus(2, 2, 2); + ValidateSummaryStatus(passed: 2, failed: 2, skipped: 2); ExitCodeEquals(1); // failing tests } @@ -54,7 +52,7 @@ public void RunMultipleMSTestAssembliesOnVstestConsoleAndTesthostCombinations(Ru { SetTestEnvironment(_testEnvironment, runnerInfo, vsTestConsoleInfo); - var assemblyPaths = BuildMultipleAssemblyPath(testhostInfo, "SimpleTestProject.dll", "SimpleTestProject2.dll").Trim('\"'); + var assemblyPaths = BuildMultipleAssemblyPath(testhostInfo, "SimpleTestProject.dll", "SimpleTestProject2.dll"); InvokeVsTestForExecution(assemblyPaths, testAdapterPath: null, FrameworkArgValue, string.Empty); @@ -73,7 +71,7 @@ public void RunMultipleTestAssembliesWithoutTestAdapterPath(RunnerInfo runnerInf { SetTestEnvironment(_testEnvironment, runnerInfo); - var assemblyPaths = BuildMultipleAssemblyPath("SimpleTestProject.dll").Trim('\"'); + var assemblyPaths = BuildMultipleAssemblyPath("SimpleTestProject.dll"); var xunitAssemblyPath = _testEnvironment.TargetFramework.Equals("net451") ? _testEnvironment.GetTestAsset("XUTestProject.dll", "net46") : _testEnvironment.GetTestAsset("XUTestProject.dll"); @@ -95,7 +93,7 @@ public void RunMultipleTestAssembliesInParallel(RunnerInfo runnerInfo, MSTestInf SetTestEnvironment(_testEnvironment, runnerInfo); using var tempDir = new TempDirectory(); - var assemblyPaths = BuildMultipleAssemblyPath(msTestInfo,"SimpleTestProject.dll", "SimpleTestProject2.dll").Trim('\"'); + var assemblyPaths = BuildMultipleAssemblyPath(msTestInfo,"SimpleTestProject.dll", "SimpleTestProject2.dll"); var arguments = PrepareArguments(assemblyPaths, testAdapterPath: null, runSettings: null, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: tempDir.Path); arguments = string.Concat(arguments, " /Parallel"); arguments = string.Concat(arguments, " /Platform:x86"); @@ -126,7 +124,7 @@ public void TestSessionTimeOutTests(RunnerInfo runnerInfo) using var tempDir = new TempDirectory(); var assemblyPaths = - BuildMultipleAssemblyPath("SimpleTestProject3.dll").Trim('\"'); + BuildMultipleAssemblyPath("SimpleTestProject3.dll"); var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: tempDir.Path); arguments = string.Concat(arguments, " /TestCaseFilter:TestSessionTimeoutTest"); @@ -147,7 +145,7 @@ public void TestPlatformShouldBeCompatibleWithOldTestHost(RunnerInfo runnerInfo) SetTestEnvironment(_testEnvironment, runnerInfo); using var tempDir = new TempDirectory(); - var assemblyPaths = BuildMultipleAssemblyPath("SampleProjectWithOldTestHost.dll").Trim('\"'); + var assemblyPaths = BuildMultipleAssemblyPath("SampleProjectWithOldTestHost.dll"); var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: tempDir.Path); InvokeVsTest(arguments); @@ -164,7 +162,7 @@ public void WorkingDirectoryIsSourceDirectory(RunnerInfo runnerInfo) SetTestEnvironment(_testEnvironment, runnerInfo); using var tempDir = new TempDirectory(); - var assemblyPaths = BuildMultipleAssemblyPath("SimpleTestProject3.dll").Trim('\"'); + var assemblyPaths = BuildMultipleAssemblyPath("SimpleTestProject3.dll"); var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: tempDir.Path); arguments = string.Concat(arguments, " /tests:WorkingDirectoryTest"); @@ -192,7 +190,7 @@ public void StackOverflowExceptionShouldBeLoggedToConsoleAndDiagLogFile(RunnerIn var diagLogFilePath = Path.Combine(tempDir.Path, $"std_error_log_{Guid.NewGuid()}.txt"); File.Delete(diagLogFilePath); - var assemblyPaths = BuildMultipleAssemblyPath("SimpleTestProject3.dll").Trim('\"'); + var assemblyPaths = BuildMultipleAssemblyPath("SimpleTestProject3.dll"); var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: tempDir.Path); arguments = string.Concat(arguments, " /testcasefilter:ExitWithStackoverFlow"); arguments = string.Concat(arguments, $" /diag:{diagLogFilePath}"); @@ -223,7 +221,7 @@ public void UnhandleExceptionExceptionShouldBeLoggedToDiagLogFile(RunnerInfo run File.Delete(diagLogFilePath); var assemblyPaths = - BuildMultipleAssemblyPath("SimpleTestProject3.dll").Trim('\"'); + BuildMultipleAssemblyPath("SimpleTestProject3.dll"); var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: tempDir.Path); arguments = string.Concat(arguments, " /testcasefilter:ExitwithUnhandleException"); arguments = string.Concat(arguments, $" /diag:{diagLogFilePath}"); @@ -245,7 +243,7 @@ public void IncompatibleSourcesWarningShouldBeDisplayedInTheConsole(RunnerInfo r var expectedWarningContains = @"Following DLL(s) do not match current settings, which are .NETFramework,Version=v4.5.1 framework and X86 platform. SimpleTestProject3.dll is built for Framework .NETFramework,Version=v4.5.1 and Platform X64"; var assemblyPaths = - BuildMultipleAssemblyPath("SimpleTestProject3.dll", "SimpleTestProjectx86.dll").Trim('\"'); + BuildMultipleAssemblyPath("SimpleTestProject3.dll", "SimpleTestProjectx86.dll"); var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: tempDir.Path); arguments = string.Concat(arguments, " /testcasefilter:PassingTestx86"); @@ -317,7 +315,7 @@ public void ExitCodeShouldReturnOneWhenTreatNoTestsAsErrorParameterSetToTrueAndN SetTestEnvironment(_testEnvironment, runnerInfo); using var tempDir = new TempDirectory(); - var assemblyPaths = BuildMultipleAssemblyPath("SimpleTestProject2.dll").Trim('\"'); + var assemblyPaths = BuildMultipleAssemblyPath("SimpleTestProject2.dll"); var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: tempDir.Path); @@ -338,7 +336,7 @@ public void ExitCodeShouldReturnZeroWhenTreatNoTestsAsErrorParameterSetToFalseAn SetTestEnvironment(_testEnvironment, runnerInfo); using var tempDir = new TempDirectory(); - var assemblyPaths = BuildMultipleAssemblyPath("SimpleTestProject2.dll").Trim('\"'); + var assemblyPaths = BuildMultipleAssemblyPath("SimpleTestProject2.dll"); var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: tempDir.Path); // Setting /TestCaseFilter to the test name, which does not exists in the assembly, so we will have 0 tests executed @@ -358,7 +356,7 @@ public void ExitCodeShouldNotDependOnTreatNoTestsAsErrorTrueValueWhenThereAreAny SetTestEnvironment(_testEnvironment, runnerInfo); using var tempDir = new TempDirectory(); - var assemblyPaths = BuildMultipleAssemblyPath("SimpleTestProject2.dll").Trim('\"'); + var assemblyPaths = BuildMultipleAssemblyPath("SimpleTestProject2.dll"); var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: tempDir.Path); @@ -377,7 +375,7 @@ public void ExitCodeShouldNotDependOnFailTreatNoTestsAsErrorFalseValueWhenThereA SetTestEnvironment(_testEnvironment, runnerInfo); using var tempDir = new TempDirectory(); - var assemblyPaths = BuildMultipleAssemblyPath("SimpleTestProject2.dll").Trim('\"'); + var assemblyPaths = BuildMultipleAssemblyPath("SimpleTestProject2.dll"); var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: tempDir.Path); arguments = string.Concat(arguments, " -- RunConfiguration.TreatNoTestsAsError=false"); diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/ExecutionThreadApartmentStateTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/ExecutionThreadApartmentStateTests.cs index 763e0d5016..7548f73d3f 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/ExecutionThreadApartmentStateTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/ExecutionThreadApartmentStateTests.cs @@ -18,7 +18,7 @@ public void UITestShouldPassIfApartmentStateIsSTA(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - var assemblyPaths = BuildMultipleAssemblyPath("SimpleTestProject3.dll").Trim('\"'); + var assemblyPaths = BuildMultipleAssemblyPath("SimpleTestProject3.dll"); var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: TempDirectory.Path); arguments = string.Concat(arguments, " /testcasefilter:UITestMethod"); InvokeVsTest(arguments); @@ -32,7 +32,7 @@ public void WarningShouldBeShownWhenValueIsSTAForNetCore(RunnerInfo runnerInfo) SetTestEnvironment(_testEnvironment, runnerInfo); var assemblyPaths = - BuildMultipleAssemblyPath("SimpleTestProject2.dll").Trim('\"'); + BuildMultipleAssemblyPath("SimpleTestProject2.dll"); var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: TempDirectory.Path); arguments = string.Concat(arguments, " /testcasefilter:PassingTest2 -- RunConfiguration.ExecutionThreadApartmentState=STA"); InvokeVsTest(arguments); @@ -47,7 +47,7 @@ public void UITestShouldFailWhenDefaultApartmentStateIsMTA(RunnerInfo runnerInfo SetTestEnvironment(_testEnvironment, runnerInfo); var assemblyPaths = - BuildMultipleAssemblyPath("SimpleTestProject3.dll").Trim('\"'); + BuildMultipleAssemblyPath("SimpleTestProject3.dll"); var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: TempDirectory.Path); arguments = string.Concat(arguments, " /testcasefilter:UITestMethod -- RunConfiguration.ExecutionThreadApartmentState=MTA"); InvokeVsTest(arguments); @@ -62,7 +62,7 @@ public void CancelTestExectionShouldWorkWhenApartmentStateIsSTA(RunnerInfo runne SetTestEnvironment(_testEnvironment, runnerInfo); var assemblyPaths = - BuildMultipleAssemblyPath("SimpleTestProject3.dll").Trim('\"'); + BuildMultipleAssemblyPath("SimpleTestProject3.dll"); var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: TempDirectory.Path); arguments = string.Concat(arguments, " /tests:UITestWithSleep1,UITestMethod -- RunConfiguration.ExecutionThreadApartmentState=STA RunConfiguration.TestSessionTimeout=2000"); InvokeVsTest(arguments); diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/FilePatternParserTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/FilePatternParserTests.cs index 4ab4715e72..3d8f36218b 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/FilePatternParserTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/FilePatternParserTests.cs @@ -92,7 +92,7 @@ public void WildCardPatternShouldCorrectlyWorkOnMultipleFiles(RunnerInfo runnerI { SetTestEnvironment(_testEnvironment, runnerInfo); - var testAssembly = BuildMultipleAssemblyPath("SimpleTestProject.dll", "SimpleTestProject2.dll").Trim('\"'); + var testAssembly = BuildMultipleAssemblyPath("SimpleTestProject.dll", "SimpleTestProject2.dll"); testAssembly = testAssembly.Replace("SimpleTestProject.dll", "*TestProj*.dll"); testAssembly = testAssembly.Replace("SimpleTestProject2.dll", "*TestProj*.dll"); diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/MultitargetingTestHostTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/MultitargetingTestHostTests.cs index 7b1dff7eb9..70a132de3a 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/MultitargetingTestHostTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/MultitargetingTestHostTests.cs @@ -23,7 +23,7 @@ public void TestRunInATesthostThatTargetsTheirChosenNETFramework(RunnerInfo runn { SetTestEnvironment(_testEnvironment, runnerInfo); - var assemblyPath = BuildMultipleAssemblyPath("MultitargetedNetFrameworkProject.dll").Trim('\"'); + var assemblyPath = BuildMultipleAssemblyPath("MultitargetedNetFrameworkProject.dll"); var arguments = PrepareArguments(assemblyPath, null, null, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: TempDirectory.Path); // Tell the test project which target framework we are expecting it to run as. @@ -35,6 +35,6 @@ public void TestRunInATesthostThatTargetsTheirChosenNETFramework(RunnerInfo runn InvokeVsTest(arguments, env); - ValidateSummaryStatus(passedTestsCount: 1, failedTestsCount: 0, 0); + ValidateSummaryStatus(passed: 1, failed: 0, 0); } } diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/PortableNugetPackageTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/PortableNugetPackageTests.cs index 5510ef2aed..d5f227fccf 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/PortableNugetPackageTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/PortableNugetPackageTests.cs @@ -43,7 +43,7 @@ public void RunMultipleTestAssemblies(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - var assemblyPaths = BuildMultipleAssemblyPath("SimpleTestProject.dll", "SimpleTestProject2.dll").Trim('\"'); + var assemblyPaths = BuildMultipleAssemblyPath("SimpleTestProject.dll", "SimpleTestProject2.dll"); InvokeVsTestForExecution(assemblyPaths, GetTestAdapterPath(), FrameworkArgValue, string.Empty); diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/RunsettingsTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/RunsettingsTests.cs index 2a0809cbaa..3d5dbcbea7 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/RunsettingsTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/RunsettingsTests.cs @@ -537,7 +537,7 @@ private void RunTestWithRunSettings(Dictionary runConfigurationD string runSettingsArgs, string additionalArgs, IEnumerable testhostProcessNames, int expectedNumOfProcessCreated) { - var assemblyPaths = BuildMultipleAssemblyPath("SimpleTestProject.dll", "SimpleTestProject2.dll").Trim('\"'); + var assemblyPaths = BuildMultipleAssemblyPath("SimpleTestProject.dll", "SimpleTestProject2.dll"); var runsettingsPath = string.Empty; diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/SelfContainedAppTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/SelfContainedAppTests.cs index c7dc6b7804..ed5e10fc8b 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/SelfContainedAppTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/SelfContainedAppTests.cs @@ -27,10 +27,10 @@ public void RunningApplicationThatIsBuiltAsSelfContainedWillNotFailToFindHostpol SetTestEnvironment(_testEnvironment, runnerInfo); // the app is published to win10-x64 because of the runtime identifier in the project - var assemblyPath = BuildMultipleAssemblyPath($@"win10-x64{Path.DirectorySeparatorChar}SelfContainedAppTestProject.dll").Trim('\"'); + var assemblyPath = BuildMultipleAssemblyPath($@"win10-x64{Path.DirectorySeparatorChar}SelfContainedAppTestProject.dll"); var arguments = PrepareArguments(assemblyPath, null, null, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: TempDirectory.Path); InvokeVsTest(arguments); - ValidateSummaryStatus(passedTestsCount: 1, 0, 0); + ValidateSummaryStatus(passed: 1, 0, 0); } } diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/TestPlatformNugetPackageTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/TestPlatformNugetPackageTests.cs index 6125803ad0..d3304c795d 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/TestPlatformNugetPackageTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/TestPlatformNugetPackageTests.cs @@ -64,7 +64,7 @@ public void RunMultipleTestAssembliesWithCodeCoverage(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - var assemblyPaths = BuildMultipleAssemblyPath("SimpleTestProject.dll", "SimpleTestProject2.dll").Trim('\"'); + var assemblyPaths = BuildMultipleAssemblyPath("SimpleTestProject.dll", "SimpleTestProject2.dll"); var arguments = CreateCodeCoverageArguments(runnerInfo, assemblyPaths, out var trxFilePath); InvokeVsTest(arguments); @@ -85,7 +85,7 @@ public override string GetConsoleRunnerPath() consoleRunnerPath = Path.Combine(s_nugetPackageFolder, "tools", "net451", "Common7", "IDE", "Extensions", "TestPlatform", "vstest.console.exe"); } - Assert.IsTrue(File.Exists(consoleRunnerPath), "GetConsoleRunnerPath: Path not found: {0}", consoleRunnerPath); + Assert.IsTrue(File.Exists(consoleRunnerPath), "GetConsoleRunnerPath: Path not found: \"{0}\"", consoleRunnerPath); return consoleRunnerPath; } diff --git a/test/Microsoft.TestPlatform.TestHostProvider.UnitTests/Hosting/DefaultTestHostManagerTests.cs b/test/Microsoft.TestPlatform.TestHostProvider.UnitTests/Hosting/DefaultTestHostManagerTests.cs index 7fd3915a62..b18464992d 100644 --- a/test/Microsoft.TestPlatform.TestHostProvider.UnitTests/Hosting/DefaultTestHostManagerTests.cs +++ b/test/Microsoft.TestPlatform.TestHostProvider.UnitTests/Hosting/DefaultTestHostManagerTests.cs @@ -142,7 +142,7 @@ public void GetTestHostProcessStartInfoShouldIncludeTestSourcePathInArgumentsIfN { _testHostManager.Initialize(_mockMessageLogger.Object, $" {Architecture.X86} {Framework.DefaultFramework} {true} "); var connectionInfo = new TestRunnerConnectionInfo { Port = 123, ConnectionInfo = new TestHostConnectionInfo { Endpoint = "127.0.0.0:123", Role = ConnectionRole.Client, Transport = Transport.Sockets }, RunnerProcessId = 101 }; - var source = "C:\temp\a.dll"; + var source = @"C:\temp\a.dll"; var info = _testHostManager.GetTestHostProcessStartInfo( new List() { source }, @@ -158,7 +158,7 @@ public void GetTestHostProcessStartInfoShouldUseMonoAsHostOnNonWindowsIfNotStart _mockProcessHelper.Setup(p => p.GetCurrentProcessFileName()).Returns("/usr/bin/dotnet"); _mockEnvironment.Setup(e => e.OperatingSystem).Returns(PlatformOperatingSystem.Unix); _mockDotnetHostHelper.Setup(d => d.GetMonoPath()).Returns("/usr/bin/mono"); - var source = "C:\temp\a.dll"; + var source = @"C:\temp\a.dll"; var info = _testHostManager.GetTestHostProcessStartInfo( new List() { source }, diff --git a/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestBase.cs b/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestBase.cs index f979ad4906..6155bfa57a 100644 --- a/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestBase.cs +++ b/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestBase.cs @@ -102,7 +102,17 @@ public static string PrepareArguments(string[] testAssemblies, string testAdapte var arguments = ""; foreach (var path in testAssemblies) { - arguments += path.AddDoubleQuote() + " "; + // The incoming testAssembly path is either a single dll path in quotes or without quotes. + // Or multiple assembly paths in a single string each double quoted and joined by space. + // We trim, and add quotes here to get either: + // C:\1.dll -> "C:\1.dll" + // "C:\1.dll" -> "C:\1.dll" + // "C:\1.dll" "C:\2.dll" -> "C:\1.dll" "C:\2.dll" + // + // For unquoted multi path string C:\1.dll C:\2.dll, we will get "C:\1.dll C:\2.dll" + // which is wrong and will fail later, but it's the test's fault for doing it wrong + // rather than providing an array of strings that this overload takes. + arguments += path.Trim('\"').AddDoubleQuote() + " "; } arguments = arguments.Trim(); @@ -270,19 +280,19 @@ public void ExecuteNotSupportedRunnerFrameworkTests(string runnerFramework, stri /// /// Validate if the overall test count and results are matching. /// - /// Passed test count - /// Failed test count - /// Skipped test count - public void ValidateSummaryStatus(int passedTestsCount, int failedTestsCount, int skippedTestsCount) + /// Passed test count + /// Failed test count + /// Skipped test count + public void ValidateSummaryStatus(int passed, int failed, int skipped) { // TODO: Switch on the actual version of vstest console when we have that set on test environment. if (_testEnvironment.VSTestConsolePath.Contains($"{Path.DirectorySeparatorChar}15.")) { - ValidateSummaryStatusv15(passedTestsCount, failedTestsCount, skippedTestsCount); + ValidateSummaryStatusv15(passed, failed, skipped); return; } - var totalTestCount = passedTestsCount + failedTestsCount + skippedTestsCount; + var totalTestCount = passed + failed + skipped; if (totalTestCount == 0) { // No test should be found/run @@ -304,19 +314,19 @@ public void ValidateSummaryStatus(int passedTestsCount, int failedTestsCount, in else { var summaryStatus = string.Format(TotalTestsMessage, totalTestCount); - if (passedTestsCount != 0) + if (passed != 0) { - summaryStatus += string.Format(PassedTestsMessage, passedTestsCount); + summaryStatus += string.Format(PassedTestsMessage, passed); } - if (failedTestsCount != 0) + if (failed != 0) { - summaryStatus += string.Format(FailedTestsMessage, failedTestsCount); + summaryStatus += string.Format(FailedTestsMessage, failed); } - if (skippedTestsCount != 0) + if (skipped != 0) { - summaryStatus += string.Format(SkippedTestsMessage, skippedTestsCount); + summaryStatus += string.Format(SkippedTestsMessage, skipped); } Assert.IsTrue( @@ -333,13 +343,13 @@ public void ValidateSummaryStatus(int passedTestsCount, int failedTestsCount, in /// /// Validate if the overall test count and results are matching. /// - /// Passed test count - /// Failed test count - /// Skipped test count - public void ValidateSummaryStatusv15(int passedTestsCount, int failedTestsCount, int skippedTestsCount) + /// Passed test count + /// Failed test count + /// Skipped test count + public void ValidateSummaryStatusv15(int passed, int failed, int skipped) { // example: Total tests: 6. Passed: 2. Failed: 2. Skipped: 2. - var totalTestCount = passedTestsCount + failedTestsCount + skippedTestsCount; + var totalTestCount = passed + failed + skipped; if (totalTestCount == 0) { // No test should be found/run @@ -355,19 +365,19 @@ public void ValidateSummaryStatusv15(int passedTestsCount, int failedTestsCount, else { var summaryStatus = $"Total tests: {totalTestCount}."; - if (passedTestsCount != 0) + if (passed != 0) { - summaryStatus += $" Passed: {passedTestsCount}."; + summaryStatus += $" Passed: {passed}."; } - if (failedTestsCount != 0) + if (failed != 0) { - summaryStatus += $" Failed: {failedTestsCount}."; + summaryStatus += $" Failed: {failed}."; } - if (skippedTestsCount != 0) + if (skipped != 0) { - summaryStatus += $" Skipped: {skippedTestsCount}."; + summaryStatus += $" Skipped: {skipped}."; } Assert.IsTrue( @@ -612,7 +622,7 @@ public virtual string GetConsoleRunnerPath() Assert.Fail("Unknown Runner framework - [{0}]", _testEnvironment.RunnerFramework); } - Assert.IsTrue(File.Exists(consoleRunnerPath), "GetConsoleRunnerPath: Path not found: {0}", consoleRunnerPath); + Assert.IsTrue(File.Exists(consoleRunnerPath), "GetConsoleRunnerPath: Path not found: \"{0}\"", consoleRunnerPath); return consoleRunnerPath; } @@ -910,7 +920,7 @@ protected string BuildMultipleAssemblyPath(DllInfo dllInfo, params string[] asse { var path = GetAssetFullPath(assetNames[i]); var updatedPath = dllInfo.UpdatePath(path); - Assert.IsTrue(File.Exists(updatedPath), "GetTestAsset: Path not found: {0}. Most likely you need to build using build.cmd -s PrepareAcceptanceTests.", updatedPath); + Assert.IsTrue(File.Exists(updatedPath), "GetTestAsset: Path not found: \"{0}\". Most likely you need to build using build.cmd -s PrepareAcceptanceTests.", updatedPath); assetFullPaths[i] = updatedPath.AddDoubleQuote(); } @@ -925,7 +935,7 @@ protected string BuildMultipleAssemblyPath(TesthostInfo testhostInfo, DllInfo ad { var path = GetAssetFullPath(assetNames[i]); var updatedPath = testhostInfo.UpdatePath(adapterInfo.UpdatePath(path)); - Assert.IsTrue(File.Exists(updatedPath), "GetTestAsset: Path not found: {0}. Most likely you need to build using build.cmd -s PrepareAcceptanceTests.", updatedPath); + Assert.IsTrue(File.Exists(updatedPath), "GetTestAsset: Path not found: \"{0}\". Most likely you need to build using build.cmd -s PrepareAcceptanceTests.", updatedPath); assetFullPaths[i] = updatedPath.AddDoubleQuote(); } diff --git a/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestEnvironment.cs b/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestEnvironment.cs index 7d4429d58c..1762200b2f 100644 --- a/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestEnvironment.cs +++ b/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestEnvironment.cs @@ -240,8 +240,10 @@ public string GetTestAsset(string assetName, string targetFramework) targetFramework, assetName); - Assert.IsTrue(File.Exists(assetPath), "GetTestAsset: Path not found: {0}. Most likely you need to build using build.cmd -s PrepareAcceptanceTests.", assetPath); + Assert.IsTrue(File.Exists(assetPath), "GetTestAsset: Path not found: \"{0}\". Most likely you need to build using build.cmd -s PrepareAcceptanceTests.", assetPath); + // If you are thinking about wrapping the path in double quotes here, + // then don't. File.Exist cannot handle quoted paths, and we use it in a lot of places. return assetPath; } @@ -315,7 +317,7 @@ public string GetTestProject(string assetName) simpleAssetName, assetName); - Assert.IsTrue(File.Exists(assetPath), "GetTestAsset: Path not found: {0}.", assetPath); + Assert.IsTrue(File.Exists(assetPath), "GetTestAsset: Path not found: \"{0}\".", assetPath); return assetPath; } diff --git a/test/TestAssets/MSTestProject1/MSTestProject1.csproj b/test/TestAssets/MSTestProject1/MSTestProject1.csproj new file mode 100644 index 0000000000..6e15698454 --- /dev/null +++ b/test/TestAssets/MSTestProject1/MSTestProject1.csproj @@ -0,0 +1,28 @@ + + + + + + netcoreapp2.1;net451 + netcoreapp3.1 + false + Preview + + + + + $(MSTestFrameworkVersion) + + + $(MSTestAdapterVersion) + + + $(NETTestSdkVersion) + + + + + + + + diff --git a/test/TestAssets/MSTestProject1/UnitTest1.cs b/test/TestAssets/MSTestProject1/UnitTest1.cs new file mode 100644 index 0000000000..17c1760f18 --- /dev/null +++ b/test/TestAssets/MSTestProject1/UnitTest1.cs @@ -0,0 +1,28 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using Microsoft.VisualStudio.TestTools.UnitTesting; + +namespace MSTestProject1; + +[TestClass] +public class UnitTest1 +{ + [TestMethod] + public void PassingTest() + { + Assert.AreEqual(2, 2); + } + + [TestMethod] + public void FailingTest() + { + Assert.AreEqual(2, 3); + } + + [Ignore] + [TestMethod] + public void SkippedTest() + { + } +} diff --git a/test/TestAssets/MSTestProject2/MSTestProject2.csproj b/test/TestAssets/MSTestProject2/MSTestProject2.csproj index 18ce8041acd97ea3ce45df23fd308ba411dedd89..6e15698454aeb3c77999dfba81759f0cd428ef05 100644 GIT binary patch literal 1063 zcmb_b!A|2a5WV*+EDNb{Sck&GWs@Mb3#igAs+wv~y%=X&LX+4UPg?l2 zDTh~B-TPBF2t;PtisHYojp}qp3bIV>DF2xD-3F5ISb;RKkp72 zKze&XT^avJSDQ*)0NtZ3V6Aa!;}S?|1Zm8L82oTw->(K?@E`2(X*Selno+$g>z|zd z()0-Q(~otJx=3_|3eNg0iddhpfimWRR-%Y*e!Z>f{r}CHXdwmDr@Vv%d?pZ@ccN*u!AY zig#nWiRGWhYR+JoGoaacw7Q zoPv;06ld+3dHd$=%s9V)McUVmt~64h?@E=c(1orw)}A)BqfjsS4X})r@cb0(SZB^U z)kH&WCcz_6pQuY0aYS_DtOXuJtf4;XP$&F)uZKwP9J zpl`G4!7w8tC!*lK%le9s@eGJgc{~o#6POxtg5l|_6TsQH~v83*q)y1+LU%DLsB&)8@!A_{m zLl}%&ROwKyF - /// The unit test 1. - /// - [TestClass] - public class UnitTest1 + [TestMethod] + public void PassingTest() { - /// - /// The passing test. - /// - [Priority(2)] - [TestMethod] - public void PassingTest() - { - Assert.AreEqual(2, 2); - } + Assert.AreEqual(2, 2); + } - /// - /// The failing test. - /// - [TestCategory("CategoryA")] - [Priority(3)] - [TestMethod] - public void FailingTest() - { -#if NETFRAMEWORK - // current App domain should be write to file to test DisableAppDomain acceptance test. - var appDomainFilePath = Environment.GetEnvironmentVariable("TEST_ASSET_APPDOMAIN_TEST_PATH") ?? Path.Combine(Path.GetTempPath(), "appdomain_test.txt"); - File.WriteAllText(appDomainFilePath, "AppDomain FriendlyName: " + AppDomain.CurrentDomain.FriendlyName); -#endif - Assert.AreEqual(2, 3); - } + [TestMethod] + public void FailingTest() + { + Assert.AreEqual(2, 3); + } - /// - /// The skipping test. - /// - [Ignore] - [TestMethod] - public void SkippingTest() - { - } + [Ignore] + [TestMethod] + public void SkippedTest() + { } } diff --git a/test/TestAssets/TestAssets.sln b/test/TestAssets/TestAssets.sln index c5d6cba585..1a05bdef96 100644 --- a/test/TestAssets/TestAssets.sln +++ b/test/TestAssets/TestAssets.sln @@ -96,7 +96,11 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "ArchitectureSwitch", "Archi EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "SimpleTestProjectMessedUpTargetFramework", "SimpleTestProjectMessedUpTargetFramework\SimpleTestProjectMessedUpTargetFramework.csproj", "{08C44607-EB80-4EE5-927D-08C34AA277AF}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "MSTestProject2", "MSTestProject2\MSTestProject2.csproj", "{10AA955C-B412-41A8-899F-8609AAE19F61}" +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "MSTestProject2", "MSTestProject2\MSTestProject2.csproj", "{10AA955C-B412-41A8-899F-8609AAE19F61}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "MSTestProject1", "MSTestProject1\MSTestProject1.csproj", "{E166D337-4033-4209-863F-8F77675EAEE8}" +EndProject +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "basic", "basic", "{2633D125-64A7-456C-AD37-F8A6B56C2403}" EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution @@ -648,10 +652,26 @@ Global {10AA955C-B412-41A8-899F-8609AAE19F61}.Release|x64.Build.0 = Release|Any CPU {10AA955C-B412-41A8-899F-8609AAE19F61}.Release|x86.ActiveCfg = Release|Any CPU {10AA955C-B412-41A8-899F-8609AAE19F61}.Release|x86.Build.0 = Release|Any CPU + {E166D337-4033-4209-863F-8F77675EAEE8}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {E166D337-4033-4209-863F-8F77675EAEE8}.Debug|Any CPU.Build.0 = Debug|Any CPU + {E166D337-4033-4209-863F-8F77675EAEE8}.Debug|x64.ActiveCfg = Debug|Any CPU + {E166D337-4033-4209-863F-8F77675EAEE8}.Debug|x64.Build.0 = Debug|Any CPU + {E166D337-4033-4209-863F-8F77675EAEE8}.Debug|x86.ActiveCfg = Debug|Any CPU + {E166D337-4033-4209-863F-8F77675EAEE8}.Debug|x86.Build.0 = Debug|Any CPU + {E166D337-4033-4209-863F-8F77675EAEE8}.Release|Any CPU.ActiveCfg = Release|Any CPU + {E166D337-4033-4209-863F-8F77675EAEE8}.Release|Any CPU.Build.0 = Release|Any CPU + {E166D337-4033-4209-863F-8F77675EAEE8}.Release|x64.ActiveCfg = Release|Any CPU + {E166D337-4033-4209-863F-8F77675EAEE8}.Release|x64.Build.0 = Release|Any CPU + {E166D337-4033-4209-863F-8F77675EAEE8}.Release|x86.ActiveCfg = Release|Any CPU + {E166D337-4033-4209-863F-8F77675EAEE8}.Release|x86.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE EndGlobalSection + GlobalSection(NestedProjects) = preSolution + {10AA955C-B412-41A8-899F-8609AAE19F61} = {2633D125-64A7-456C-AD37-F8A6B56C2403} + {E166D337-4033-4209-863F-8F77675EAEE8} = {2633D125-64A7-456C-AD37-F8A6B56C2403} + EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {D2334DAA-F7B2-450E-ABA4-FBC185152500} EndGlobalSection From 82ad32023f5e4374a5e9ca69017e2196e5a483a2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Jare=C5=A1?= Date: Wed, 16 Mar 2022 14:27:38 +0100 Subject: [PATCH 054/112] Some tests run. --- .../AcceptanceTestBase.cs | 14 ++-- .../ExecutionTests.cs | 20 +++--- .../MSTestCompatibilityDataSource.cs | 33 +++++++--- .../Extension/NetCoreRunner.cs | 10 ++- .../NetCoreTargetFrameworkDataSource.cs | 9 ++- .../Extension/NetFrameworkRunner.cs | 11 +++- .../NetFullTargetFrameworkDataSource.cs | 30 ++++++++- .../Extension/RunnnerInfo.cs | 28 +++++++- .../TestPlatformCompatibilityDataSource.cs | 50 +++++++++----- .../TesthostCompatibilityDataSource.cs | 45 ++++++++----- ...TranslationLayerCompatibilityDataSource.cs | 25 +++++-- .../CustomTestHostTests.cs | 16 ++--- .../TranslationLayerTests/DiscoverTests.cs | 8 +-- .../TranslationLayerTests/RunTests.cs | 4 +- .../RunTestsWithFilterTests.cs | 4 +- .../DebugInfo.cs | 17 +++++ .../DllInfo.cs | 1 + .../IntegrationTestBase.cs | 65 +++++++------------ .../IntegrationTestEnvironment.cs | 9 ++- .../MSTestInfo.cs | 3 + .../TesthostInfo.cs | 7 +- .../VSTestConsoleInfo.cs | 3 + 22 files changed, 271 insertions(+), 141 deletions(-) create mode 100644 test/Microsoft.TestPlatform.TestUtilities/DebugInfo.cs diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/AcceptanceTestBase.cs b/test/Microsoft.TestPlatform.AcceptanceTests/AcceptanceTestBase.cs index 4f74da75e7..c44ec9775e 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/AcceptanceTestBase.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/AcceptanceTestBase.cs @@ -69,19 +69,15 @@ public static string And(string left, string right) protected string FrameworkArgValue => DeriveFrameworkArgValue(_testEnvironment); - protected static void SetTestEnvironment(IntegrationTestEnvironment testEnvironment, RunnerInfo runnerInfo, VSTestConsoleInfo vsTestConsoleInfo = null) + protected static void SetTestEnvironment(IntegrationTestEnvironment testEnvironment, RunnerInfo runnerInfo) { - if (vsTestConsoleInfo != null) - { - testEnvironment.VSTestConsolePath = vsTestConsoleInfo.Path; - } + testEnvironment.VSTestConsoleInfo = runnerInfo.VSTestConsoleInfo; + testEnvironment.DllInfos = runnerInfo.DllInfos; + testEnvironment.DebugInfo = runnerInfo.DebugInfo; + testEnvironment.RunnerFramework = runnerInfo.RunnerFramework; testEnvironment.TargetFramework = runnerInfo.TargetFramework; testEnvironment.InIsolationValue = runnerInfo.InIsolationValue; - testEnvironment.DebugVSTestConsole = runnerInfo.DebugVSTestConsole; - testEnvironment.DebugTesthost = runnerInfo.DebugTesthost; - testEnvironment.DebugDataCollector = runnerInfo.DebugDataCollector; - testEnvironment.NoDefaultBreakpoints = runnerInfo.NoDefaultBreakpoints; } protected static string DeriveFrameworkArgValue(IntegrationTestEnvironment testEnvironment) diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/ExecutionTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/ExecutionTests.cs index 14daacdf20..3faa9178ab 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/ExecutionTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/ExecutionTests.cs @@ -19,11 +19,11 @@ public class ExecutionTests : AcceptanceTestBase // TODO: It looks like the first 3 tests would be useful to multiply by all 3 test frameworks, should we make the test even more generic, or duplicate them? [TestMethod] [MSTestCompatibilityDataSource(InProcess = true)] - public void RunMultipleTestAssemblies(RunnerInfo runnerInfo, MSTestInfo msTestInfo) + public void RunMultipleTestAssemblies(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - var assemblyPaths = BuildMultipleAssemblyPath(msTestInfo, "SimpleTestProject.dll", "SimpleTestProject2.dll"); + var assemblyPaths = BuildMultipleAssemblyPath("SimpleTestProject.dll", "SimpleTestProject2.dll"); InvokeVsTestForExecution(assemblyPaths, testAdapterPath: null, FrameworkArgValue, string.Empty); @@ -32,13 +32,13 @@ public void RunMultipleTestAssemblies(RunnerInfo runnerInfo, MSTestInfo msTestIn } [TestMethod] - [TestPlatformCompatibilityDataSource(WithInProcess = true)] + [TestPlatformCompatibilityDataSource(WithInProcess = true, WithEveryVersionOfAdapter = false, WithEveryVersionOfHost = false, WithEveryVersionOfRunner = false, WithOlderConfigurations = false)] - public void RunTestsFromMultipleMSTestAssemblies(RunnerInfo runnerInfo, VSTestConsoleInfo consoleInfo, TesthostInfo testhostInfo, MSTestInfo msTestInfo) + public void RunTestsFromMultipleMSTestAssemblies(RunnerInfo runnerInfo) { - SetTestEnvironment(_testEnvironment, runnerInfo, consoleInfo); + SetTestEnvironment(_testEnvironment, runnerInfo); - var assemblyPaths = BuildMultipleAssemblyPath(testhostInfo, msTestInfo, "MSTestProject1.dll", "MSTestProject2.dll"); + var assemblyPaths = BuildMultipleAssemblyPath("MSTestProject1.dll", "MSTestProject2.dll"); InvokeVsTestForExecution(assemblyPaths, testAdapterPath: null, FrameworkArgValue, string.Empty); @@ -48,11 +48,11 @@ public void RunTestsFromMultipleMSTestAssemblies(RunnerInfo runnerInfo, VSTestCo [TestMethod] [TesthostCompatibilityDataSource] - public void RunMultipleMSTestAssembliesOnVstestConsoleAndTesthostCombinations(RunnerInfo runnerInfo, VSTestConsoleInfo vsTestConsoleInfo, TesthostInfo testhostInfo) + public void RunMultipleMSTestAssembliesOnVstestConsoleAndTesthostCombinations(RunnerInfo runnerInfo) { - SetTestEnvironment(_testEnvironment, runnerInfo, vsTestConsoleInfo); + SetTestEnvironment(_testEnvironment, runnerInfo); - var assemblyPaths = BuildMultipleAssemblyPath(testhostInfo, "SimpleTestProject.dll", "SimpleTestProject2.dll"); + var assemblyPaths = BuildMultipleAssemblyPath("SimpleTestProject.dll", "SimpleTestProject2.dll"); InvokeVsTestForExecution(assemblyPaths, testAdapterPath: null, FrameworkArgValue, string.Empty); @@ -93,7 +93,7 @@ public void RunMultipleTestAssembliesInParallel(RunnerInfo runnerInfo, MSTestInf SetTestEnvironment(_testEnvironment, runnerInfo); using var tempDir = new TempDirectory(); - var assemblyPaths = BuildMultipleAssemblyPath(msTestInfo,"SimpleTestProject.dll", "SimpleTestProject2.dll"); + var assemblyPaths = BuildMultipleAssemblyPath("SimpleTestProject.dll", "SimpleTestProject2.dll"); var arguments = PrepareArguments(assemblyPaths, testAdapterPath: null, runSettings: null, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: tempDir.Path); arguments = string.Concat(arguments, " /Parallel"); arguments = string.Concat(arguments, " /Platform:x86"); diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/MSTestCompatibilityDataSource.cs b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/MSTestCompatibilityDataSource.cs index 388217cea8..af3101c971 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/MSTestCompatibilityDataSource.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/MSTestCompatibilityDataSource.cs @@ -13,7 +13,7 @@ namespace Microsoft.TestPlatform.AcceptanceTests; -public sealed class MSTestCompatibilityDataSource : TestDataSource +public sealed class MSTestCompatibilityDataSource : TestDataSource { private static XmlDocument? s_depsXml; private readonly string[] _runnerFrameworks; @@ -53,11 +53,18 @@ public override void CreateData(MethodInfo methodInfo) { foreach (var msTestVersion in _msTestVersions) { - var runnerInfo = new RunnerInfo(AcceptanceTestBase.DEFAULT_RUNNER_NETFX, AcceptanceTestBase.DEFAULT_RUNNER_NETFX, InIsolationValue: null, - DebugVSTestConsole, DebugTesthost, DebugDataCollector, NoDefaultBreakpoints); - var msTestInfo = GetMSTestInfo(msTestVersion); // We run in the .NET Framework runner process, the runner and target framework must agree. - AddData(runnerInfo, msTestInfo); + var runnerInfo = new RunnerInfo(AcceptanceTestBase.DEFAULT_RUNNER_NETFX, AcceptanceTestBase.DEFAULT_RUNNER_NETFX, inIsolationValue: null); + runnerInfo.DebugInfo = new DebugInfo + { + DebugVSTestConsole = DebugVSTestConsole, + DebugTesthost = DebugTesthost, + DebugDataCollector = DebugDataCollector, + NoDefaultBreakpoints = NoDefaultBreakpoints, + }; + runnerInfo.DllInfos.Add(GetMSTestInfo(msTestVersion)); + + AddData(runnerInfo); } } @@ -67,11 +74,17 @@ public override void CreateData(MethodInfo methodInfo) { foreach (var msTestVersion in _msTestVersions) { - var runnerInfo = new RunnerInfo(runner, fmw, AcceptanceTestBase.InIsolation, - DebugVSTestConsole, DebugTesthost, DebugDataCollector, NoDefaultBreakpoints); - var msTestInfo = GetMSTestInfo(msTestVersion); - - AddData(runnerInfo, msTestInfo); + var runnerInfo = new RunnerInfo(runner, fmw, AcceptanceTestBase.InIsolation); + runnerInfo.DebugInfo = new DebugInfo + { + DebugVSTestConsole = DebugVSTestConsole, + DebugTesthost = DebugTesthost, + DebugDataCollector = DebugDataCollector, + NoDefaultBreakpoints = NoDefaultBreakpoints, + }; + runnerInfo.DllInfos.Add(GetMSTestInfo(msTestVersion)); + + AddData(runnerInfo); } } } diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/NetCoreRunner.cs b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/NetCoreRunner.cs index 1a3d855e29..e12d7f6011 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/NetCoreRunner.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/NetCoreRunner.cs @@ -47,7 +47,15 @@ public IEnumerable GetData(MethodInfo methodInfo) Func filter = tfm => isWindows || !tfm.StartsWith("net4"); foreach (var fmw in _targetFrameworks.Split(';').Where(filter)) { - dataRows.Add(new object[] { new RunnerInfo(IntegrationTestBase.CoreRunnerFramework, fmw, InIsolationValue: null, DebugVSTestConsole, DebugTesthost, DebugDataCollector, NoDefaultBreakpoints) }); + var runnerInfo = new RunnerInfo(IntegrationTestBase.CoreRunnerFramework, fmw, inIsolationValue: null); + runnerInfo.DebugInfo = new DebugInfo + { + DebugVSTestConsole = DebugVSTestConsole, + DebugTesthost = DebugTesthost, + DebugDataCollector = DebugDataCollector, + NoDefaultBreakpoints = NoDefaultBreakpoints, + }; + dataRows.Add(new object[] { runnerInfo }); } return dataRows; diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/NetCoreTargetFrameworkDataSource.cs b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/NetCoreTargetFrameworkDataSource.cs index 6788d56074..d5bfd3b4ce 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/NetCoreTargetFrameworkDataSource.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/NetCoreTargetFrameworkDataSource.cs @@ -55,7 +55,14 @@ public NetCoreTargetFrameworkDataSource( private void AddRunnerDataRow(List dataRows, string runnerFramework, string targetFramework) { - var runnerInfo = new RunnerInfo(runnerFramework, targetFramework, InIsolationValue: null, DebugVSTestConsole, DebugTesthost, DebugDataCollector, NoDefaultBreakpoints); + var runnerInfo = new RunnerInfo(runnerFramework, targetFramework, inIsolationValue: null); + runnerInfo.DebugInfo = new DebugInfo + { + DebugDataCollector = DebugDataCollector, + DebugTesthost = DebugTesthost, + DebugVSTestConsole = DebugVSTestConsole, + NoDefaultBreakpoints = NoDefaultBreakpoints, + }; dataRows.Add(new object[] { runnerInfo }); } diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/NetFrameworkRunner.cs b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/NetFrameworkRunner.cs index b8386149f0..07f30ee823 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/NetFrameworkRunner.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/NetFrameworkRunner.cs @@ -49,7 +49,16 @@ public IEnumerable GetData(MethodInfo methodInfo) foreach (var fmw in _targetFrameworks.Split(';')) { - dataRows.Add(new object[] { new RunnerInfo(IntegrationTestBase.DesktopRunnerFramework, fmw, AcceptanceTestBase.InIsolation, DebugVSTestConsole, DebugTesthost, DebugDataCollector, NoDefaultBreakpoints) }); + var runnerInfo = new RunnerInfo(IntegrationTestBase.DesktopRunnerFramework, fmw, AcceptanceTestBase.InIsolation); + runnerInfo.DebugInfo = new DebugInfo + { + DebugVSTestConsole = DebugVSTestConsole, + DebugTesthost = DebugTesthost, + DebugDataCollector = DebugDataCollector, + NoDefaultBreakpoints = NoDefaultBreakpoints, + }; + + dataRows.Add(new object[] { runnerInfo }); } return dataRows; diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/NetFullTargetFrameworkDataSource.cs b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/NetFullTargetFrameworkDataSource.cs index c3b0af0e6c..3b45a19e74 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/NetFullTargetFrameworkDataSource.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/NetFullTargetFrameworkDataSource.cs @@ -53,19 +53,43 @@ public IEnumerable GetData(MethodInfo methodInfo) var isWindows = Environment.OSVersion.Platform.ToString().StartsWith("Win"); if (_useCoreRunner && isWindows) { - dataRows.Add(new object[] { new RunnerInfo(IntegrationTestBase.CoreRunnerFramework, AcceptanceTestBase.DesktopTargetFramework, InIsolationValue: null, DebugVSTestConsole, DebugTesthost, DebugDataCollector, NoDefaultBreakpoints) }); + var runnerInfo = new RunnerInfo(IntegrationTestBase.CoreRunnerFramework, AcceptanceTestBase.DesktopTargetFramework, inIsolationValue: null); + runnerInfo.DebugInfo = new DebugInfo + { + DebugVSTestConsole = DebugVSTestConsole, + DebugTesthost = DebugTesthost, + DebugDataCollector = DebugDataCollector, + NoDefaultBreakpoints = NoDefaultBreakpoints, + }; + dataRows.Add(new object[] { runnerInfo }); } if (_useDesktopRunner && isWindows) { if (_inIsolation) { - dataRows.Add(new object[] { new RunnerInfo(IntegrationTestBase.DesktopRunnerFramework, AcceptanceTestBase.DesktopTargetFramework, AcceptanceTestBase.InIsolation, DebugVSTestConsole, DebugTesthost, DebugDataCollector, NoDefaultBreakpoints) }); + var runnerInfo = new RunnerInfo(IntegrationTestBase.DesktopRunnerFramework, AcceptanceTestBase.DesktopTargetFramework, AcceptanceTestBase.InIsolation); + runnerInfo.DebugInfo = new DebugInfo + { + DebugVSTestConsole = DebugVSTestConsole, + DebugTesthost = DebugTesthost, + DebugDataCollector = DebugDataCollector, + NoDefaultBreakpoints = NoDefaultBreakpoints, + }; + dataRows.Add(new object[] { runnerInfo }); } if (_inProcess) { - dataRows.Add(new object[] { new RunnerInfo(IntegrationTestBase.DesktopRunnerFramework, AcceptanceTestBase.DesktopTargetFramework, InIsolationValue: null, DebugVSTestConsole, DebugTesthost, DebugDataCollector, NoDefaultBreakpoints) }); + var runnerInfo = new RunnerInfo(IntegrationTestBase.DesktopRunnerFramework, AcceptanceTestBase.DesktopTargetFramework, inIsolationValue: null); + runnerInfo.DebugInfo = new DebugInfo + { + DebugVSTestConsole = DebugVSTestConsole, + DebugTesthost = DebugTesthost, + DebugDataCollector = DebugDataCollector, + NoDefaultBreakpoints = NoDefaultBreakpoints, + }; + dataRows.Add(new object[] { runnerInfo }); } } diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/RunnnerInfo.cs b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/RunnnerInfo.cs index da0d836e89..fcf9cca46e 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/RunnnerInfo.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/RunnnerInfo.cs @@ -2,6 +2,9 @@ // Licensed under the MIT license. See LICENSE file in the project root for full license information. using System; +using System.Collections.Generic; + +using Microsoft.TestPlatform.TestUtilities; namespace Microsoft.TestPlatform.AcceptanceTests; @@ -11,9 +14,28 @@ namespace Microsoft.TestPlatform.AcceptanceTests; /// /// /// Supported value = /InIsolation. -public record struct RunnerInfo(string RunnerFramework, string TargetFramework, string? InIsolationValue = "", - bool DebugVSTestConsole = false, bool DebugTesthost = false, bool DebugDataCollector = false, bool NoDefaultBreakpoints = true) +[Serializable] +public class RunnerInfo { + public RunnerInfo(string runnerFramework, string targetFramework, string? inIsolationValue = "") + { + RunnerFramework = runnerFramework; + TargetFramework = targetFramework; + InIsolationValue = inIsolationValue; + } + + public string RunnerFramework { get; set; } + + public VSTestConsoleInfo VSTestConsoleInfo { get; set; } + + + public string TargetFramework { get; set; } + public string? InIsolationValue { get; set; } + + public DebugInfo? DebugInfo { get; set; } + + public List DllInfos { get; set; } = new(); + /// /// Is running via .NET "Core" vstest.console? /// @@ -34,5 +56,5 @@ public record struct RunnerInfo(string RunnerFramework, string TargetFramework, /// public bool IsNetFrameworkTarget => TargetFramework.StartsWith("net4", StringComparison.InvariantCultureIgnoreCase); - public override string ToString() => $"RunnerFramework = {RunnerFramework}, TargetFramework = {TargetFramework}, {(string.IsNullOrEmpty(InIsolationValue) ? "InProcess" : "InIsolation")}"; + public override string ToString() => $"Runner = {RunnerFramework}, TargetFramework = {TargetFramework}, {(string.IsNullOrEmpty(InIsolationValue) ? "InProcess" : "InIsolation")}, {VSTestConsoleInfo}, {string.Join(",", DllInfos)}"; } diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/TestPlatformCompatibilityDataSource.cs b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/TestPlatformCompatibilityDataSource.cs index 262b3f1b85..eca9ac55e6 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/TestPlatformCompatibilityDataSource.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/TestPlatformCompatibilityDataSource.cs @@ -15,7 +15,7 @@ namespace Microsoft.TestPlatform.AcceptanceTests; -public sealed class TestPlatformCompatibilityDataSource : TestDataSource +public sealed class TestPlatformCompatibilityDataSource : TestDataSource { private static XmlDocument? s_depsXml; private readonly string[] _runnerFrameworks; @@ -54,7 +54,7 @@ public TestPlatformCompatibilityDataSource( /// /// Add run for in-process using the selected .NET Framework runners, and and all selected adapters. /// - public bool WithInProcess { get; set; } + public bool WithInProcess { get; set; } = true; public bool WithEveryVersionOfRunner { get; set; } = true; @@ -72,7 +72,7 @@ public TestPlatformCompatibilityDataSource( public override void CreateData(MethodInfo methodInfo) { - var dataRows = new List<(RunnerInfo runnerInfo, VSTestConsoleInfo vstestConsoleInfo, TesthostInfo testhostInfo, MSTestInfo mstestInfo)>(); + var dataRows = new List(); if (WithEveryVersionOfRunner) AddEveryVersionOfRunner(dataRows); @@ -121,11 +121,11 @@ public override void CreateData(MethodInfo methodInfo) foreach (var dataRow in dataRows) { - AddData(dataRow.runnerInfo, dataRow.vstestConsoleInfo, dataRow.testhostInfo, dataRow.mstestInfo); + AddData(dataRow); } } - private void AddInProcess(List<(RunnerInfo runnerInfo, VSTestConsoleInfo vstestConsoleInfo, TesthostInfo testhostInfo, MSTestInfo mstestInfo)> dataRows) + private void AddInProcess(List dataRows) { foreach (var runnerFramework in _runnerFrameworks) { @@ -147,7 +147,7 @@ private void AddInProcess(List<(RunnerInfo runnerInfo, VSTestConsoleInfo vstestC } } - private void AddOlderConfigurations(List<(RunnerInfo, VSTestConsoleInfo, TesthostInfo, MSTestInfo)> dataRows) + private void AddOlderConfigurations(List dataRows) { // Older configurations where the runner, host and adapter version are the same. // We already added the row where all are newest when adding combination with all runners. @@ -169,7 +169,7 @@ private void AddOlderConfigurations(List<(RunnerInfo, VSTestConsoleInfo, Testhos } } - private void AddEveryVersionOfAdapter(List<(RunnerInfo, VSTestConsoleInfo, TesthostInfo, MSTestInfo)> dataRows) + private void AddEveryVersionOfAdapter(List dataRows) { var runnerVersion = _runnerVersions[0]; foreach (var runnerFramework in _runnerFrameworks) @@ -192,7 +192,7 @@ private void AddEveryVersionOfAdapter(List<(RunnerInfo, VSTestConsoleInfo, Testh } } - private void AddEveryVersionOfHost(List<(RunnerInfo, VSTestConsoleInfo, TesthostInfo, MSTestInfo)> dataRows) + private void AddEveryVersionOfHost(List dataRows) { var runnerVersion = _runnerVersions[0]; @@ -219,7 +219,7 @@ private void AddEveryVersionOfHost(List<(RunnerInfo, VSTestConsoleInfo, Testhost } } - private void AddEveryVersionOfRunner(List<(RunnerInfo, VSTestConsoleInfo, TesthostInfo, MSTestInfo)> dataRows) + private void AddEveryVersionOfRunner(List dataRows) { foreach (var runnerVersion in _runnerVersions) { @@ -242,20 +242,38 @@ private void AddEveryVersionOfRunner(List<(RunnerInfo, VSTestConsoleInfo, Testho } } - private void AddRow(List<(RunnerInfo, VSTestConsoleInfo, TesthostInfo, MSTestInfo)> dataRows, + private void AddRow(List dataRows, string runnerVersion, string runnerFramework, string hostVersion, string hostFramework, string adapter, string adapterVersion, bool inIsolation) { RunnerInfo runnerInfo = GetRunnerInfo(runnerFramework, hostFramework, inIsolation); - var vstestConsoleInfo = GetVSTestConsoleInfo(runnerVersion, runnerInfo); - var testhostInfo = TesthostCompatibilityDataSource.GetTesthostInfo(hostVersion); - var mstestInfo = GetMSTestInfo(adapterVersion); - dataRows.Add(new(runnerInfo, vstestConsoleInfo, testhostInfo, mstestInfo)); + runnerInfo.DebugInfo = GetDebugInfo(); + runnerInfo.VSTestConsoleInfo = GetVSTestConsoleInfo(runnerVersion, runnerInfo); + + // The order in which we add them matters. We end up both modifying the same path + // and adding to it. So the first one added will be later in the path. E.g.: + // Adding testSdk first: + // C:\p\vstest\test\TestAssets\MSTestProject1\bin\MSTestLatestPreview-2.2.9-preview-20220210-07\NETTestSdkLatest-17.2.0-dev\Debug\net451\MSTestProject1.dll + // versus adding testSdk second: + // C:\p\vstest\test\TestAssets\MSTestProject1\bin\NETTestSdkLatest-17.2.0-dev\MSTestLatestPreview-2.2.9-preview-20220210-07\Debug\net451\MSTestProject1.dll + runnerInfo.DllInfos.Add(GetMSTestInfo(adapterVersion)); + runnerInfo.DllInfos.Add(TesthostCompatibilityDataSource.GetNetTestSdkInfo(hostVersion)); + dataRows.Add(runnerInfo); + } + + private DebugInfo GetDebugInfo() + { + return new DebugInfo + { + DebugDataCollector = DebugDataCollector, + DebugTesthost = DebugTesthost, + DebugVSTestConsole = DebugVSTestConsole, + NoDefaultBreakpoints = NoDefaultBreakpoints + }; } private RunnerInfo GetRunnerInfo(string runnerFramework, string hostFramework, bool inIsolation) { - return new RunnerInfo(runnerFramework, hostFramework, inIsolation ? AcceptanceTestBase.InIsolation : null, - DebugVSTestConsole, DebugTesthost, DebugDataCollector, NoDefaultBreakpoints); + return new RunnerInfo(runnerFramework, hostFramework, inIsolation ? AcceptanceTestBase.InIsolation : null); } public string GetDisplayName(MethodInfo methodInfo, object[] data) diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/TesthostCompatibilityDataSource.cs b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/TesthostCompatibilityDataSource.cs index 6892b4a210..a74c45adc9 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/TesthostCompatibilityDataSource.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/TesthostCompatibilityDataSource.cs @@ -13,7 +13,7 @@ namespace Microsoft.TestPlatform.AcceptanceTests; -public sealed class TesthostCompatibilityDataSource : TestDataSource +public sealed class TesthostCompatibilityDataSource : TestDataSource { private static XmlDocument? s_depsXml; private readonly string[] _runnerFrameworks; @@ -58,13 +58,18 @@ public override void CreateData(MethodInfo methodInfo) var testhostVersions = runner.StartsWith("net4") ? onlyLatest : _testhostVersions; foreach (var testhostVersion in testhostVersions) { - var runnerInfo = new RunnerInfo(runner, fmw, AcceptanceTestBase.InIsolation, - DebugVSTestConsole, DebugTesthost, DebugDataCollector, NoDefaultBreakpoints); - - var vstestConsoleInfo = TranslationLayerCompatibilityDataSource.GetVSTestConsoleInfo(AcceptanceTestBase.LATEST, runnerInfo); - var testhostInfo = GetTesthostInfo(testhostVersion); - - AddData(runnerInfo, vstestConsoleInfo, testhostInfo); + var runnerInfo = new RunnerInfo(runner, fmw, AcceptanceTestBase.InIsolation); + runnerInfo.DebugInfo = new DebugInfo + { + DebugDataCollector = DebugDataCollector, + DebugTesthost = DebugTesthost, + DebugVSTestConsole = DebugVSTestConsole, + NoDefaultBreakpoints = NoDefaultBreakpoints + }; + runnerInfo.VSTestConsoleInfo = TranslationLayerCompatibilityDataSource.GetVSTestConsoleInfo(AcceptanceTestBase.LATEST, runnerInfo); + runnerInfo.DllInfos.Add(GetNetTestSdkInfo(testhostVersion)); + + AddData(runnerInfo); } } } @@ -81,14 +86,18 @@ public override void CreateData(MethodInfo methodInfo) var consoleVersions = runner.StartsWith("net4") ? onlyLatest : _testhostVersions; foreach (var consoleVersion in consoleVersions) { - var runnerInfo = new RunnerInfo(runner, fmw, AcceptanceTestBase.InIsolation, - DebugVSTestConsole, DebugTesthost, DebugDataCollector, NoDefaultBreakpoints); - - var vstestConsoleInfo = TranslationLayerCompatibilityDataSource.GetVSTestConsoleInfo(consoleVersion, runnerInfo); - // Generate only for latest testhsot, - var testhostInfo = GetTesthostInfo(AcceptanceTestBase.LATEST); - - AddData(runnerInfo, vstestConsoleInfo, testhostInfo); + var runnerInfo = new RunnerInfo(runner, fmw, AcceptanceTestBase.InIsolation); + runnerInfo.DebugInfo = new DebugInfo + { + DebugDataCollector = DebugDataCollector, + DebugTesthost = DebugTesthost, + DebugVSTestConsole = DebugVSTestConsole, + NoDefaultBreakpoints = NoDefaultBreakpoints + }; + runnerInfo.VSTestConsoleInfo = TranslationLayerCompatibilityDataSource.GetVSTestConsoleInfo(consoleVersion, runnerInfo); + runnerInfo.DllInfos.Add(GetNetTestSdkInfo(AcceptanceTestBase.LATEST)); + + AddData(runnerInfo); } } } @@ -99,7 +108,7 @@ public string GetDisplayName(MethodInfo methodInfo, object[] data) return string.Format(CultureInfo.CurrentCulture, "{0} ({1})", methodInfo.Name, string.Join(",", data)); } - internal static TesthostInfo GetTesthostInfo(string testhostVersionType) + internal static NetTestSdkInfo GetNetTestSdkInfo(string testhostVersionType) { var depsXml = GetDependenciesXml(); @@ -119,7 +128,7 @@ internal static TesthostInfo GetTesthostInfo(string testhostVersionType) var slash = Path.DirectorySeparatorChar; var versionSpecificBinPath = $"{slash}bin{slash}NETTestSdk{testhostVersionType}-{version}{slash}"; - return new TesthostInfo(testhostVersionType, version, versionSpecificBinPath); + return new NetTestSdkInfo(testhostVersionType, version, versionSpecificBinPath); } private static XmlDocument GetDependenciesXml() diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/TranslationLayerCompatibilityDataSource.cs b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/TranslationLayerCompatibilityDataSource.cs index d5e8d2ee77..50dfd86bbf 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/TranslationLayerCompatibilityDataSource.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/TranslationLayerCompatibilityDataSource.cs @@ -85,8 +85,14 @@ public override void CreateData(MethodInfo methodInfo) { foreach (var vstestConsoleVersion in _vstestConsoleVersions) { - var runnerInfo = new RunnerInfo(runner, fmw, AcceptanceTestBase.InIsolation, - DebugVSTestConsole, DebugTesthost, DebugDataCollector, NoDefaultBreakpoints); + var runnerInfo = new RunnerInfo(runner, fmw, AcceptanceTestBase.InIsolation); + runnerInfo.DebugInfo = new DebugInfo + { + DebugVSTestConsole = DebugVSTestConsole, + DebugTesthost = DebugTesthost, + DebugDataCollector = DebugDataCollector, + NoDefaultBreakpoints = NoDefaultBreakpoints, + }; var vsTestConsoleInfo = GetVSTestConsoleInfo(vstestConsoleVersion, runnerInfo); if (beforeVersion != null && vsTestConsoleInfo.Version > beforeVersion) @@ -156,7 +162,18 @@ private static XmlDocument GetDependenciesXml() } } -public readonly record struct Feature(string Version, string Issue); +public class Feature +{ + public Feature (string version, string issue) + { + Version = version; + Issue = issue; + } + + public string Version { get; } + public string Issue { get; } +} + public static class Features { @@ -164,6 +181,6 @@ public static class Features public static Dictionary Table { get; } = new Dictionary { - [ATTACH_DEBUGGER] = new(Version: "v16.7.0-preview-20200519-01", Issue: "https://github.com/microsoft/vstest/pull/2325") + [ATTACH_DEBUGGER] = new(version: "v16.7.0-preview-20200519-01", issue: "https://github.com/microsoft/vstest/pull/2325") }; } diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/CustomTestHostTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/CustomTestHostTests.cs index f0524f871e..e6c341ca4f 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/CustomTestHostTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/CustomTestHostTests.cs @@ -35,9 +35,9 @@ public void Cleanup() [TestMethod] [TranslationLayerCompatibilityDataSource(BeforeFeature = Features.ATTACH_DEBUGGER)] - public void RunTestsWithCustomTestHostLauncherLaunchesTheProcessUsingTheProvidedLauncher(RunnerInfo runnerInfo, VSTestConsoleInfo vsTestConsoleInfo) + public void RunTestsWithCustomTestHostLauncherLaunchesTheProcessUsingTheProvidedLauncher(RunnerInfo runnerInfo) { - SetTestEnvironment(_testEnvironment, runnerInfo, vsTestConsoleInfo); + SetTestEnvironment(_testEnvironment, runnerInfo); _vstestConsoleWrapper = GetVsTestConsoleWrapper(); _runEventHandler = new RunEventHandler(); @@ -56,9 +56,9 @@ public void RunTestsWithCustomTestHostLauncherLaunchesTheProcessUsingTheProvided [TestMethod] [TranslationLayerCompatibilityDataSource(BeforeFeature = Features.ATTACH_DEBUGGER)] - public void RunTestsWithCustomTestHostLauncherLaunchesTheProcessUsingTheProvidedLauncherWhenITestHostLauncher2IsProvided(RunnerInfo runnerInfo, VSTestConsoleInfo vsTestConsoleInfo) + public void RunTestsWithCustomTestHostLauncherLaunchesTheProcessUsingTheProvidedLauncherWhenITestHostLauncher2IsProvided(RunnerInfo runnerInfo) { - SetTestEnvironment(_testEnvironment, runnerInfo, vsTestConsoleInfo); + SetTestEnvironment(_testEnvironment, runnerInfo); _vstestConsoleWrapper = GetVsTestConsoleWrapper(); _runEventHandler = new RunEventHandler(); @@ -78,9 +78,9 @@ public void RunTestsWithCustomTestHostLauncherLaunchesTheProcessUsingTheProvided [TestMethod] [TranslationLayerCompatibilityDataSource(AfterFeature = Features.ATTACH_DEBUGGER)] - public void RunTestsWithCustomTestHostLauncherAttachesToDebuggerUsingTheProvidedLauncher(RunnerInfo runnerInfo, VSTestConsoleInfo vsTestConsoleInfo) + public void RunTestsWithCustomTestHostLauncherAttachesToDebuggerUsingTheProvidedLauncher(RunnerInfo runnerInfo) { - SetTestEnvironment(_testEnvironment, runnerInfo, vsTestConsoleInfo); + SetTestEnvironment(_testEnvironment, runnerInfo); _vstestConsoleWrapper = GetVsTestConsoleWrapper(); _runEventHandler = new RunEventHandler(); @@ -103,9 +103,9 @@ public void RunTestsWithCustomTestHostLauncherAttachesToDebuggerUsingTheProvided + "It sees 6 but does not realize that the provided CustomTesthostLauncher is not supporting the new feature, it ends up calling back to EditoAttachDebugger" + "in translation layer, and that just silently skips the call.")] [TranslationLayerCompatibilityDataSource("net451", "net451", "Latest", AfterFeature = Features.ATTACH_DEBUGGER, DebugVSTestConsole = true, DebugTesthost=true)] - public void RunTestsWithCustomTestHostLauncherUsesLaunchWhenGivenAnOutdatedITestHostLauncher(RunnerInfo runnerInfo, VSTestConsoleInfo vsTestConsoleInfo) + public void RunTestsWithCustomTestHostLauncherUsesLaunchWhenGivenAnOutdatedITestHostLauncher(RunnerInfo runnerInfo) { - SetTestEnvironment(_testEnvironment, runnerInfo, vsTestConsoleInfo); + SetTestEnvironment(_testEnvironment, runnerInfo); _vstestConsoleWrapper = GetVsTestConsoleWrapper(); _runEventHandler = new RunEventHandler(); diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/DiscoverTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/DiscoverTests.cs index 3a59d0de0f..c0a278cf14 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/DiscoverTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/DiscoverTests.cs @@ -41,9 +41,9 @@ public void Cleanup() [TestMethod] [TranslationLayerCompatibilityDataSource] - public void DiscoverTestsUsingDiscoveryEventHandler1(RunnerInfo runnerInfo, VSTestConsoleInfo vsTestConsoleInfo) + public void DiscoverTestsUsingDiscoveryEventHandler1(RunnerInfo runnerInfo) { - SetTestEnvironment(_testEnvironment, runnerInfo, vsTestConsoleInfo); + SetTestEnvironment(_testEnvironment, runnerInfo); // Setup(); _discoveryEventHandler = new DiscoveryEventHandler(); @@ -58,9 +58,9 @@ public void DiscoverTestsUsingDiscoveryEventHandler1(RunnerInfo runnerInfo, VSTe [TestMethod] [TranslationLayerCompatibilityDataSource()] - public void DiscoverTestsUsingDiscoveryEventHandler2AndTelemetryOptedOut(RunnerInfo runnerInfo, VSTestConsoleInfo vsTestConsoleInfo) + public void DiscoverTestsUsingDiscoveryEventHandler2AndTelemetryOptedOut(RunnerInfo runnerInfo) { - SetTestEnvironment(_testEnvironment, runnerInfo, vsTestConsoleInfo); + SetTestEnvironment(_testEnvironment, runnerInfo); // Setup(); _discoveryEventHandler = new DiscoveryEventHandler(); diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/RunTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/RunTests.cs index d966022eeb..ed121eaa0e 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/RunTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/RunTests.cs @@ -41,9 +41,9 @@ public void Cleanup() [TestMethod] [TranslationLayerCompatibilityDataSource] - public void RunAllTests(RunnerInfo runnerInfo, VSTestConsoleInfo vsTestConsoleInfo) + public void RunAllTests(RunnerInfo runnerInfo) { - SetTestEnvironment(_testEnvironment, runnerInfo, vsTestConsoleInfo); + SetTestEnvironment(_testEnvironment, runnerInfo); var vstestConsoleWrapper = GetVsTestConsoleWrapper(); var runEventHandler = new RunEventHandler(); diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/RunTestsWithFilterTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/RunTestsWithFilterTests.cs index 9edda9281e..9661f18242 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/RunTestsWithFilterTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/RunTestsWithFilterTests.cs @@ -37,9 +37,9 @@ public void Cleanup() [TestMethod] [TranslationLayerCompatibilityDataSource] - public void RunTestsWithTestCaseFilter(RunnerInfo runnerInfo, VSTestConsoleInfo vsTestConsoleInfo) + public void RunTestsWithTestCaseFilter(RunnerInfo runnerInfo) { - SetTestEnvironment(_testEnvironment, runnerInfo, vsTestConsoleInfo); + SetTestEnvironment(_testEnvironment, runnerInfo); // Setup(); _runEventHandler = new RunEventHandler(); diff --git a/test/Microsoft.TestPlatform.TestUtilities/DebugInfo.cs b/test/Microsoft.TestPlatform.TestUtilities/DebugInfo.cs new file mode 100644 index 0000000000..3869ccb2ea --- /dev/null +++ b/test/Microsoft.TestPlatform.TestUtilities/DebugInfo.cs @@ -0,0 +1,17 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using System; + +namespace Microsoft.TestPlatform.TestUtilities; + +[Serializable] +public class DebugInfo +{ + public bool DebugVSTestConsole { get; set; } + public bool DebugTesthost { get; set; } + public bool DebugDataCollector { get; set; } + public bool NoDefaultBreakpoints { get; set; } = true; +} + + diff --git a/test/Microsoft.TestPlatform.TestUtilities/DllInfo.cs b/test/Microsoft.TestPlatform.TestUtilities/DllInfo.cs index e417ef9db7..8299661238 100644 --- a/test/Microsoft.TestPlatform.TestUtilities/DllInfo.cs +++ b/test/Microsoft.TestPlatform.TestUtilities/DllInfo.cs @@ -5,6 +5,7 @@ namespace Microsoft.TestPlatform.TestUtilities; +[Serializable] public abstract class DllInfo { protected DllInfo(string name, string propertyName, string versionType, string? version, string path) diff --git a/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestBase.cs b/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestBase.cs index 6155bfa57a..2c80715014 100644 --- a/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestBase.cs +++ b/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestBase.cs @@ -222,26 +222,26 @@ public void InvokeVsTestForExecution(string testAssembly, { var arguments = PrepareArguments(testAssembly, testAdapterPath, runSettings, framework, _testEnvironment.InIsolationValue, resultsDirectory: TempDirectory.Path); - if (_testEnvironment.DebugVSTestConsole || _testEnvironment.DebugTesthost || _testEnvironment.DebugDataCollector) + if (_testEnvironment.DebugInfo.DebugVSTestConsole || _testEnvironment.DebugInfo.DebugTesthost || _testEnvironment.DebugInfo.DebugDataCollector) { environmentVariables ??= new Dictionary(); - if (_testEnvironment.DebugVSTestConsole) + if (_testEnvironment.DebugInfo.DebugVSTestConsole) { environmentVariables.Add("VSTEST_RUNNER_DEBUG_ATTACHVS", "1"); } - if (_testEnvironment.DebugTesthost) + if (_testEnvironment.DebugInfo.DebugTesthost) { environmentVariables.Add("VSTEST_HOST_DEBUG_ATTACHVS", "1"); } - if (_testEnvironment.DebugDataCollector) + if (_testEnvironment.DebugInfo.DebugDataCollector) { environmentVariables.Add("VSTEST_DATACOLLECTOR_DEBUG_ATTACHVS", "1"); } - if (_testEnvironment.NoDefaultBreakpoints) + if (_testEnvironment.DebugInfo.NoDefaultBreakpoints) { environmentVariables.Add("VSTEST_DEBUG_NOBP", "1"); } @@ -286,7 +286,7 @@ public void ExecuteNotSupportedRunnerFrameworkTests(string runnerFramework, stri public void ValidateSummaryStatus(int passed, int failed, int skipped) { // TODO: Switch on the actual version of vstest console when we have that set on test environment. - if (_testEnvironment.VSTestConsolePath.Contains($"{Path.DirectorySeparatorChar}15.")) + if (_testEnvironment.VSTestConsoleInfo != null && _testEnvironment.VSTestConsoleInfo.Path.Contains($"{Path.DirectorySeparatorChar}15.")) { ValidateSummaryStatusv15(passed, failed, skipped); return; @@ -603,9 +603,9 @@ public virtual string GetConsoleRunnerPath() if (IsDesktopRunner()) { - if (!string.IsNullOrWhiteSpace(_testEnvironment.VSTestConsolePath)) + if (!string.IsNullOrWhiteSpace(_testEnvironment?.VSTestConsoleInfo.Path)) { - consoleRunnerPath = _testEnvironment.VSTestConsolePath; + consoleRunnerPath = _testEnvironment.VSTestConsoleInfo.Path; } else { @@ -672,27 +672,27 @@ public IVsTestConsoleWrapper GetVsTestConsoleWrapper() Console.WriteLine($"Console runner path: {consoleRunnerPath}"); VsTestConsoleWrapper vstestConsoleWrapper; - if (_testEnvironment.DebugVSTestConsole || _testEnvironment.DebugTesthost || _testEnvironment.DebugDataCollector) + if (_testEnvironment.DebugInfo.DebugVSTestConsole || _testEnvironment.DebugInfo.DebugTesthost || _testEnvironment.DebugInfo.DebugDataCollector) { var environmentVariables = new Dictionary(); Environment.GetEnvironmentVariables().OfType().ToList().ForEach(e => environmentVariables.Add(e.Key.ToString(), e.Value.ToString())); - if (_testEnvironment.DebugVSTestConsole) + if (_testEnvironment.DebugInfo.DebugVSTestConsole) { environmentVariables.Add("VSTEST_RUNNER_DEBUG_ATTACHVS", "1"); } - if (_testEnvironment.DebugTesthost) + if (_testEnvironment.DebugInfo.DebugTesthost) { environmentVariables.Add("VSTEST_HOST_DEBUG_ATTACHVS", "1"); } - if (_testEnvironment.DebugDataCollector) + if (_testEnvironment.DebugInfo.DebugDataCollector) { environmentVariables.Add("VSTEST_DATACOLLECTOR_DEBUG_ATTACHVS", "1"); } - if (_testEnvironment.NoDefaultBreakpoints) + if (_testEnvironment.DebugInfo.NoDefaultBreakpoints) { environmentVariables.Add("VSTEST_DEBUG_NOBP", "1"); } @@ -903,41 +903,22 @@ public static void CreateRunSettingsFile(string destinationRunsettingsPath, stri } protected string BuildMultipleAssemblyPath(params string[] assetNames) - { - var assetFullPath = new string[assetNames.Length]; - for (var i = 0; i < assetNames.Length; i++) - { - assetFullPath[i] = GetAssetFullPath(assetNames[i]).AddDoubleQuote(); - } - - return string.Join(" ", assetFullPath); - } - - protected string BuildMultipleAssemblyPath(DllInfo dllInfo, params string[] assetNames) { var assetFullPaths = new string[assetNames.Length]; for (var i = 0; i < assetNames.Length; i++) { var path = GetAssetFullPath(assetNames[i]); - var updatedPath = dllInfo.UpdatePath(path); - Assert.IsTrue(File.Exists(updatedPath), "GetTestAsset: Path not found: \"{0}\". Most likely you need to build using build.cmd -s PrepareAcceptanceTests.", updatedPath); - - assetFullPaths[i] = updatedPath.AddDoubleQuote(); - } - - return string.Join(" ", assetFullPaths); - } + if (_testEnvironment.DllInfos.Count > 0) + { + foreach (var dllInfo in _testEnvironment.DllInfos) + { + path = dllInfo.UpdatePath(path); + } - protected string BuildMultipleAssemblyPath(TesthostInfo testhostInfo, DllInfo adapterInfo, params string[] assetNames) - { - var assetFullPaths = new string[assetNames.Length]; - for (var i = 0; i < assetNames.Length; i++) - { - var path = GetAssetFullPath(assetNames[i]); - var updatedPath = testhostInfo.UpdatePath(adapterInfo.UpdatePath(path)); - Assert.IsTrue(File.Exists(updatedPath), "GetTestAsset: Path not found: \"{0}\". Most likely you need to build using build.cmd -s PrepareAcceptanceTests.", updatedPath); + Assert.IsTrue(File.Exists(path), "GetTestAsset: Path not found: \"{0}\". Most likely you need to build using build.cmd -s PrepareAcceptanceTests.", path); + } - assetFullPaths[i] = updatedPath.AddDoubleQuote(); + assetFullPaths[i] = path.AddDoubleQuote(); } return string.Join(" ", assetFullPaths); @@ -979,5 +960,5 @@ protected static string GetDownloadedDotnetMuxerFromTools(string architecture) return path; } - protected string GetDotnetRunnerPath() => _testEnvironment.VSTestConsolePath ?? Path.Combine(_testEnvironment.PublishDirectory, "vstest.console.dll"); + protected string GetDotnetRunnerPath() => _testEnvironment?.VSTestConsoleInfo.Path ?? Path.Combine(_testEnvironment.PublishDirectory, "vstest.console.dll"); } diff --git a/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestEnvironment.cs b/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestEnvironment.cs index 1762200b2f..b0af339915 100644 --- a/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestEnvironment.cs +++ b/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestEnvironment.cs @@ -4,6 +4,7 @@ using System; using System.Collections.Generic; using System.IO; +using System.Linq.Expressions; using System.Xml; using Microsoft.VisualStudio.TestTools.UnitTesting; @@ -193,11 +194,9 @@ public string RunnerFramework // A known AzureDevOps env variable meaning we are running in CI. public static bool IsCI { get; } = Environment.GetEnvironmentVariable("TF_BUILD") == "True"; - public bool DebugVSTestConsole { get; set; } - public bool DebugTesthost { get; set; } - public bool DebugDataCollector { get; set; } - public bool NoDefaultBreakpoints { get; set; } - public string VSTestConsolePath { get; set; } + public DebugInfo DebugInfo { get; set; } + public VSTestConsoleInfo VSTestConsoleInfo { get; set; } + public List DllInfos { get; set; } /// /// Gets the full path to a test asset. diff --git a/test/Microsoft.TestPlatform.TestUtilities/MSTestInfo.cs b/test/Microsoft.TestPlatform.TestUtilities/MSTestInfo.cs index e4c128774e..6529493ab0 100644 --- a/test/Microsoft.TestPlatform.TestUtilities/MSTestInfo.cs +++ b/test/Microsoft.TestPlatform.TestUtilities/MSTestInfo.cs @@ -1,8 +1,11 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT license. See LICENSE file in the project root for full license information. +using System; + namespace Microsoft.TestPlatform.TestUtilities; +[Serializable] public class MSTestInfo : DllInfo { public MSTestInfo(string versionType, string? version, string path) diff --git a/test/Microsoft.TestPlatform.TestUtilities/TesthostInfo.cs b/test/Microsoft.TestPlatform.TestUtilities/TesthostInfo.cs index f29006fd38..a9be09a3d8 100644 --- a/test/Microsoft.TestPlatform.TestUtilities/TesthostInfo.cs +++ b/test/Microsoft.TestPlatform.TestUtilities/TesthostInfo.cs @@ -1,11 +1,14 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT license. See LICENSE file in the project root for full license information. +using System; + namespace Microsoft.TestPlatform.TestUtilities; -public class TesthostInfo : DllInfo +[Serializable] +public class NetTestSdkInfo : DllInfo { - public TesthostInfo(string versionType, string? version, string path) + public NetTestSdkInfo(string versionType, string? version, string path) : base(name: "Testhost", propertyName: "VSTestConsole", versionType, version, path) { } diff --git a/test/Microsoft.TestPlatform.TestUtilities/VSTestConsoleInfo.cs b/test/Microsoft.TestPlatform.TestUtilities/VSTestConsoleInfo.cs index d88ad20375..41d441e3b8 100644 --- a/test/Microsoft.TestPlatform.TestUtilities/VSTestConsoleInfo.cs +++ b/test/Microsoft.TestPlatform.TestUtilities/VSTestConsoleInfo.cs @@ -1,8 +1,11 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT license. See LICENSE file in the project root for full license information. +using System; + namespace Microsoft.TestPlatform.TestUtilities; +[Serializable] public class VSTestConsoleInfo { public VSTestConsoleInfo(string versionType, string? version, string path) From 5216498a73defacbe18668d84cf798782881653e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Jare=C5=A1?= Date: Wed, 16 Mar 2022 19:22:24 +0100 Subject: [PATCH 055/112] More fun with sources. --- playground/TestPlatform.Playground/Program.cs | 3 +- .../TestPlatform.Playground.csproj | 2 +- scripts/build.ps1 | 72 ++-- sln.cmd | 14 - src/SettingsMigrator/ConsoleParameters.cs | 93 ----- .../AcceptanceTestBase.cs | 9 +- .../ExecutionTests.cs | 27 +- .../Extension/CompatibilityRowsBuilder.cs | 380 ++++++++++++++++++ .../Extension/Feature.cs | 16 + .../Extension/Features.cs | 23 ++ .../MSTestCompatibilityDataSource.cs | 152 +++---- .../Extension/NetCoreRunner.cs | 7 +- .../NetCoreTargetFrameworkDataSource.cs | 7 +- .../Extension/NetFrameworkRunner.cs | 7 +- .../NetFullTargetFrameworkDataSource.cs | 21 +- .../Extension/RunnnerInfo.cs | 17 +- .../Extension/TestDataSource.cs | 1 + .../TestPlatformCompatibilityDataSource.cs | 333 ++------------- .../TesthostCompatibilityDataSource.cs | 176 +++----- ...TranslationLayerCompatibilityDataSource.cs | 205 +++------- .../CustomTestHostTests.cs | 15 +- .../TranslationLayerTests/DiscoverTests.cs | 4 +- .../TranslationLayerTests/RunTests.cs | 2 +- .../RunTestsWithFilterTests.cs | 3 +- .../IntegrationTestBase.cs | 14 +- ...icrosoft.TestPlatform.TestUtilities.csproj | 1 + 26 files changed, 737 insertions(+), 867 deletions(-) delete mode 100644 sln.cmd delete mode 100644 src/SettingsMigrator/ConsoleParameters.cs create mode 100644 test/Microsoft.TestPlatform.AcceptanceTests/Extension/CompatibilityRowsBuilder.cs create mode 100644 test/Microsoft.TestPlatform.AcceptanceTests/Extension/Feature.cs create mode 100644 test/Microsoft.TestPlatform.AcceptanceTests/Extension/Features.cs diff --git a/playground/TestPlatform.Playground/Program.cs b/playground/TestPlatform.Playground/Program.cs index 2b1fcb0a0c..5d46ab9328 100644 --- a/playground/TestPlatform.Playground/Program.cs +++ b/playground/TestPlatform.Playground/Program.cs @@ -55,7 +55,8 @@ static void Main(string[] args) "; var sources = new[] { - Path.Combine(playground, "MSTest1", "bin", "Debug", "net472", "MSTest1.dll") + @"C:\p\vstest\test\TestAssets\MSTestProject1\bin\NETTestSdkLatest-17.2.0-dev\MSTestLatestPreview-2.2.9-preview-20220210-07\Debug\net451\MSTestProject1.dll", + @"C:\p\vstest\test\TestAssets\MSTestProject2\bin\NETTestSdkLatest-17.2.0-dev\MSTestLatestPreview-2.2.9-preview-20220210-07\Debug\net451\MSTestProject2.dll" }; var options = new TestPlatformOptions(); diff --git a/playground/TestPlatform.Playground/TestPlatform.Playground.csproj b/playground/TestPlatform.Playground/TestPlatform.Playground.csproj index 77ccfd0b7e..7bdbeb915e 100644 --- a/playground/TestPlatform.Playground/TestPlatform.Playground.csproj +++ b/playground/TestPlatform.Playground/TestPlatform.Playground.csproj @@ -38,7 +38,7 @@ - + diff --git a/scripts/build.ps1 b/scripts/build.ps1 index 23c5872736..c5723cb933 100644 --- a/scripts/build.ps1 +++ b/scripts/build.ps1 @@ -276,7 +276,7 @@ function Invoke-TestAssetsBuild { $dirMSTestVersion = $mstestVersion -replace "\[|\]" $dirMSTestPropertyName = $propertyName -replace "Framework" -replace "Version" - Invoke-Exe $dotnetExe -Arguments "build $project --configuration $TPB_Configuration --no-restore -v:minimal -p:CIBuild=$TPB_CIBuild -p:LocalizedBuild=$TPB_LocalizedBuild -p:MSTestFrameworkVersion=$mstestVersion -p:MSTestAdapterVersion=$mstestVersion -p:NETTestSdkVersion=$netTestSdkVersion -p:BaseOutputPath=""bin\$dirNetTestSdkPropertyName-$dirNetTestSdkVersion\$dirMSTestPropertyName-$dirMSTestVersion\\"" -bl:""$env:TP_OUT_DIR\log\$Configuration\perm.binlog""" + Invoke-Exe $dotnetExe -Arguments "build $project --configuration $TPB_Configuration -v:minimal -p:CIBuild=$TPB_CIBuild -p:LocalizedBuild=$TPB_LocalizedBuild -p:MSTestFrameworkVersion=$mstestVersion -p:MSTestAdapterVersion=$mstestVersion -p:NETTestSdkVersion=$netTestSdkVersion -p:BaseOutputPath=""bin\$dirNetTestSdkPropertyName-$dirNetTestSdkVersion\$dirMSTestPropertyName-$dirMSTestVersion\\"" -bl:""$env:TP_OUT_DIR\log\$Configuration\perm.binlog""" } } } @@ -1276,49 +1276,49 @@ if ($ProjectNamePatterns.Count -ne 0) { } # Execute build -# $timer = Start-Timer -# Write-Log "Build started: args = '$args'" -# Write-Log "Test platform environment variables: " -# Get-ChildItem env: | Where-Object -FilterScript { $_.Name.StartsWith("TP_") } | Format-Table -# Write-Log "Test platform build variables: " -# Get-Variable | Where-Object -FilterScript { $_.Name.StartsWith("TPB_") } | Format-Table +$timer = Start-Timer +Write-Log "Build started: args = '$args'" +Write-Log "Test platform environment variables: " +Get-ChildItem env: | Where-Object -FilterScript { $_.Name.StartsWith("TP_") } | Format-Table +Write-Log "Test platform build variables: " +Get-Variable | Where-Object -FilterScript { $_.Name.StartsWith("TPB_") } | Format-Table if ($Force -or $Steps -contains "InstallDotnet") { Install-DotNetCli } -# if ($Force -or $Steps -contains "Restore") { -# Clear-Package -# Restore-Package -# } - -# if ($Force -or $Steps -contains "UpdateLocalization") { -# Update-LocalizedResources -# } - -# if ($Force -or $Steps -contains "Build") { -# Invoke-Build -# } - -# if ($Force -or $Steps -contains "Publish") { -# Publish-Package -# Create-VsixPackage -# Create-NugetPackages -# } - -# if ($Force -or $Steps -contains "Publish" -or $Steps -contains "Manifest") { -# Generate-Manifest -PackageFolder $TPB_PackageOutDir -# if (Test-Path $TPB_SourceBuildPackageOutDir) -# { -# Generate-Manifest -PackageFolder $TPB_SourceBuildPackageOutDir -# } -# Copy-PackageIntoStaticDirectory -# } +if ($Force -or $Steps -contains "Restore") { + Clear-Package + Restore-Package +} + +if ($Force -or $Steps -contains "UpdateLocalization") { + Update-LocalizedResources +} + +if ($Force -or $Steps -contains "Build") { + Invoke-Build +} + +if ($Force -or $Steps -contains "Publish") { + Publish-Package + Create-VsixPackage + Create-NugetPackages +} + +if ($Force -or $Steps -contains "Publish" -or $Steps -contains "Manifest") { + Generate-Manifest -PackageFolder $TPB_PackageOutDir + if (Test-Path $TPB_SourceBuildPackageOutDir) + { + Generate-Manifest -PackageFolder $TPB_SourceBuildPackageOutDir + } + Copy-PackageIntoStaticDirectory +} if ($Force -or $Steps -contains "PrepareAcceptanceTests") { - #Publish-PatchedDotnet + Publish-PatchedDotnet Invoke-TestAssetsBuild - #Publish-Tests + Publish-Tests } if ($Script:ScriptFailed) { diff --git a/sln.cmd b/sln.cmd deleted file mode 100644 index 8ba8ab0bb9..0000000000 --- a/sln.cmd +++ /dev/null @@ -1,14 +0,0 @@ -@echo off - -REM Copyright (c) Microsoft. All rights reserved. - -REM set DOTNET_ROOT to point at the locally installed dotnet, -REM to avoid problems with .NET Core 2.1 that we run our tests and tools against, -REM but that is regularly corrupted. - -set DOTNET_ROOT=%~dp0tools\dotnet -set DOTNET_ROOT(x86)=%~dp0tools\dotnet_x86 - -start %~dp0TestPlatform.sln - -if %errorlevel% neq 0 exit /b %errorlevel% diff --git a/src/SettingsMigrator/ConsoleParameters.cs b/src/SettingsMigrator/ConsoleParameters.cs deleted file mode 100644 index 80470abdff..0000000000 --- a/src/SettingsMigrator/ConsoleParameters.cs +++ /dev/null @@ -1,93 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT license. See LICENSE file in the project root for full license information. - -#if NETFRAMEWORK -using System.Collections.Generic; -#endif -using System.Collections.Generic; -using System.Diagnostics; -using System.IO; - -using Microsoft.VisualStudio.TestPlatform.CoreUtilities.Extensions; -using Microsoft.VisualStudio.TestPlatform.ObjectModel; -using Microsoft.VisualStudio.TestPlatform.Utilities.Helpers; -using Microsoft.VisualStudio.TestPlatform.Utilities.Helpers.Interfaces; - -#nullable disable - -namespace Microsoft.TestPlatform.VsTestConsole.TranslationLayer; - -/// -/// Class which defines additional specifiable parameters for vstest.console.exe -/// -public class ConsoleParameters -{ - internal static readonly ConsoleParameters Default = new(); - - private string _logFilePath; - private readonly IFileHelper _fileHelper; - - /// - /// Create instance of - /// - public ConsoleParameters() : this(new FileHelper()) - { } - - /// - /// Create instance of - /// - /// Object of type - public ConsoleParameters(IFileHelper fileHelper) - { - _fileHelper = fileHelper; - } - - /// - /// Environment variables to be set for the process - /// - public Dictionary EnvironmentVariables { get; set; } - - /// - /// Trace level for logs. - /// - public TraceLevel TraceLevel { get; set; } = TraceLevel.Verbose; - - /// - /// Full path for the log file - /// - public string LogFilePath - { - get - { - return _logFilePath; - } - - set - { - ValidateArg.NotNullOrEmpty(value, "LogFilePath"); - var directoryPath = Path.GetDirectoryName(value); - if (!string.IsNullOrEmpty(directoryPath) && !_fileHelper.DirectoryExists(directoryPath)) - { - Directory.CreateDirectory(directoryPath); - } - - // Ensure path is double quoted. if path has white space then it can create problem. - _logFilePath = value.AddDoubleQuote(); - } - } - - /// - /// Port Number for communication - /// vstest.console will need this port number to communicate with this component - translation layer - /// Currently Internal as we are not intentionally exposing this to consumers of translation layer - /// - internal int PortNumber { get; set; } - - /// - /// Parent Process ID of the process whose lifetime should dictate the life time of vstest.console.exe - /// vstest.console will need this process ID to know when the process exits. - /// If parent process dies/crashes without invoking EndSession, vstest.console should exit immediately - /// Currently Internal as we are not intentionally exposing this to consumers of translation layer - /// - internal int ParentProcessId { get; set; } -} diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/AcceptanceTestBase.cs b/test/Microsoft.TestPlatform.AcceptanceTests/AcceptanceTestBase.cs index c44ec9775e..574e742f2e 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/AcceptanceTestBase.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/AcceptanceTestBase.cs @@ -45,7 +45,7 @@ public class AcceptanceTestBase : IntegrationTestBase public const string DesktopRunnerTargetRuntime = "win7-x64"; public const string CoreRunnerTargetRuntime = ""; public const string InIsolation = "/InIsolation"; - + public const string NETFX452_48 = "net452;net461;net472;net48"; public const string NETFX451_48 = "net452;net461;net472;net48"; public const string NETCORE21_50 = "netcoreapp2.1;netcoreapp3.1;net5.0"; @@ -55,12 +55,13 @@ public class AcceptanceTestBase : IntegrationTestBase /// /// Our current defaults for .NET and .NET Framework. /// - public const string DEFAULT_RUNNER_NETFX_AND_NET = $"{DEFAULT_RUNNER_NETFX};netcoreapp2.1"; + public const string DEFAULT_RUNNER_NETFX_AND_NET = $"{DEFAULT_RUNNER_NETFX};netcoreapp2.1"; public const string DEFAULT_HOST_NETFX_AND_NET = "net451;netcoreapp2.1"; public const string LATEST_TO_LEGACY = "Latest;LatestPreview;LatestStable;RecentStable;MostDownloaded;PreviousStable;LegacyStable"; public const string LATESTPREVIEW_TO_LEGACY = "LatestPreview;LatestStable;RecentStable;MostDownloaded;PreviousStable;LegacyStable"; public const string LATEST = "Latest"; - internal const string MSTEST="MSTest"; + public const string LATESTSTABLE= "LatestStable"; + internal const string MSTEST = "MSTest"; public static string And(string left, string right) { @@ -74,7 +75,7 @@ protected static void SetTestEnvironment(IntegrationTestEnvironment testEnvironm testEnvironment.VSTestConsoleInfo = runnerInfo.VSTestConsoleInfo; testEnvironment.DllInfos = runnerInfo.DllInfos; testEnvironment.DebugInfo = runnerInfo.DebugInfo; - + testEnvironment.RunnerFramework = runnerInfo.RunnerFramework; testEnvironment.TargetFramework = runnerInfo.TargetFramework; testEnvironment.InIsolationValue = runnerInfo.InIsolationValue; diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/ExecutionTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/ExecutionTests.cs index 3faa9178ab..b39e6b76a1 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/ExecutionTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/ExecutionTests.cs @@ -23,16 +23,18 @@ public void RunMultipleTestAssemblies(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - var assemblyPaths = BuildMultipleAssemblyPath("SimpleTestProject.dll", "SimpleTestProject2.dll"); + var assemblyPaths = BuildMultipleAssemblyPath("MSTestProject1.dll", "MSTestProject2.dll"); InvokeVsTestForExecution(assemblyPaths, testAdapterPath: null, FrameworkArgValue, string.Empty); ValidateSummaryStatus(2, 2, 2); ExitCodeEquals(1); // failing tests + StdErrHasTestRunFailedMessageButNoOtherError(); + StdOutHasNoWarnings(); } [TestMethod] - [TestPlatformCompatibilityDataSource(WithInProcess = true, WithEveryVersionOfAdapter = false, WithEveryVersionOfHost = false, WithEveryVersionOfRunner = false, WithOlderConfigurations = false)] + [TestPlatformCompatibilityDataSource(BeforeFeature = Features.ATTACH_DEBUGGER, AfterAdapterFeature = Features.MSTEST_IFRAMEWORK_HANDLE_99)] public void RunTestsFromMultipleMSTestAssemblies(RunnerInfo runnerInfo) { @@ -44,15 +46,32 @@ public void RunTestsFromMultipleMSTestAssemblies(RunnerInfo runnerInfo) ValidateSummaryStatus(passed: 2, failed: 2, skipped: 2); ExitCodeEquals(1); // failing tests + StdErrHasTestRunFailedMessageButNoOtherError(); + StdOutHasNoWarnings(); } [TestMethod] - [TesthostCompatibilityDataSource] + [HostCompatibilityDataSource] public void RunMultipleMSTestAssembliesOnVstestConsoleAndTesthostCombinations(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - var assemblyPaths = BuildMultipleAssemblyPath("SimpleTestProject.dll", "SimpleTestProject2.dll"); + var assemblyPaths = BuildMultipleAssemblyPath("MSTestProject1.dll", "MSTestProject2.dll"); + + InvokeVsTestForExecution(assemblyPaths, testAdapterPath: null, FrameworkArgValue, string.Empty); + + ValidateSummaryStatus(2, 2, 2); + ExitCodeEquals(1); // failing tests + } + + + [TestMethod] + [RunnerCompatibilityDataSource] + public void RunMultipleMSTestAssembliesOnVstestConsoleAndTesthostCombinations2(RunnerInfo runnerInfo) + { + SetTestEnvironment(_testEnvironment, runnerInfo); + + var assemblyPaths = BuildMultipleAssemblyPath("MSTestProject1.dll", "MSTestProject2.dll"); InvokeVsTestForExecution(assemblyPaths, testAdapterPath: null, FrameworkArgValue, string.Empty); diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/CompatibilityRowsBuilder.cs b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/CompatibilityRowsBuilder.cs new file mode 100644 index 0000000000..f4fa979dbd --- /dev/null +++ b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/CompatibilityRowsBuilder.cs @@ -0,0 +1,380 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Xml; + +using Microsoft.TestPlatform.TestUtilities; + +using Semver; + +namespace Microsoft.TestPlatform.AcceptanceTests; + +public class CompatibilityRowsBuilder +{ + private static XmlDocument? s_depsXml; + private readonly string[] _runnerFrameworks; + private readonly string[] _runnerVersions; + private readonly string[] _hostFrameworks; + private readonly string[] _adapterVersions; + private readonly string[] _adapters; + private readonly string[] _hostVersions; + + public CompatibilityRowsBuilder(string runnerFrameworks = AcceptanceTestBase.DEFAULT_HOST_NETFX_AND_NET, + string runnerVersions = AcceptanceTestBase.LATEST_TO_LEGACY, + string hostFrameworks = AcceptanceTestBase.DEFAULT_HOST_NETFX_AND_NET, + string hostVersions = AcceptanceTestBase.LATEST_TO_LEGACY, + string adapterVersions = AcceptanceTestBase.LATESTPREVIEW_TO_LEGACY, + string adapters = AcceptanceTestBase.MSTEST) + { + _runnerFrameworks = runnerFrameworks.Split(';'); + _runnerVersions = runnerVersions.Split(';'); + _hostFrameworks = hostFrameworks.Split(';'); + _hostVersions = hostVersions.Split(';'); + _adapterVersions = adapterVersions.Split(';'); + _adapters = adapters.Split(';'); + } + + /// + /// Add run for in-process using the selected .NET Framework runners, and and all selected adapters. + /// + public bool WithInProcess { get; set; } = true; + public bool WithEveryVersionOfRunner { get; set; } = true; + public bool WithEveryVersionOfHost { get; set; } = true; + public bool WithEveryVersionOfAdapter { get; set; } = true; + public bool WithOlderConfigurations { get; set; } = true; + + public string? BeforeFeature { get; set; } + public string? AfterFeature { get; set; } + public string? BeforeAdapterFeature { get; set; } + public string? AfterAdapterFeature { get; set; } + + public bool DebugVSTestConsole { get; set; } + public bool DebugTesthost { get; set; } + public bool DebugDataCollector { get; set; } + public bool NoDefaultBreakpoints { get; set; } = true; + + + public List CreateData() + { + var dataRows = new List(); + + if (WithEveryVersionOfRunner) + AddEveryVersionOfRunner(dataRows); + + if (WithEveryVersionOfHost) + AddEveryVersionOfHost(dataRows); + + if (WithEveryVersionOfAdapter) + AddEveryVersionOfAdapter(dataRows); + + if (WithOlderConfigurations) + AddOlderConfigurations(dataRows); + + if (WithInProcess) + AddInProcess(dataRows); + + var minVersion = SemVersion.Parse("0.0.0-alpha.1"); + var maxVersion = SemVersion.Parse("9999.0.0"); + SemVersion? beforeVersion = maxVersion; + SemVersion? afterVersion = minVersion; + SemVersion? beforeAdapterVersion = maxVersion; + SemVersion? afterAdapterVersion = minVersion; + + if (BeforeFeature != null) + { + var feature = Features.TestPlatformFeatures[BeforeFeature]; + beforeVersion = SemVersion.Parse(feature.Version.TrimStart('v')); + } + + if (AfterFeature != null) + { + var feature = Features.TestPlatformFeatures[AfterFeature]; + afterVersion = SemVersion.Parse(feature.Version.TrimStart('v')); + } + + if (BeforeFeature != null) + { + var feature = Features.TestPlatformFeatures[BeforeFeature]; + beforeAdapterVersion = SemVersion.Parse(feature.Version.TrimStart('v')); + } + + if (AfterAdapterFeature != null) + { + var feature = Features.AdapterFeatures[AfterAdapterFeature]; + afterAdapterVersion = SemVersion.Parse(feature.Version.TrimStart('v')); + } + + var isWindows = Environment.OSVersion.Platform.ToString().StartsWith("Win"); + // Run .NET Framework tests only on Windows. + Func filter = tfm => isWindows || !tfm.StartsWith("net4"); + + // TODO: maybe we should throw if we don't end up generating any data + // because none of the versions match, or some other way to identify tests that will never run because they are very outdated. + // We probably don't have that need right now, because legacy version is 15.x.x, which is very old, and we are still keeping + // compatibility. + + Func isInRange = (version, before, after) => version < before && after < version; + + var rows = dataRows.Where(r => r.VSTestConsoleInfo != null + && isInRange(r.VSTestConsoleInfo.Version, beforeVersion, afterVersion) + && r.DllInfos.All(d => d is NetTestSdkInfo ? isInRange(d.Version, beforeVersion, afterVersion) : isInRange(d.Version, beforeAdapterVersion, afterAdapterVersion))).ToList(); + + if (rows.Count == 0) + { + // TODO: This needs to be way more specific about what happened. + throw new InvalidOperationException("There were no rows that matched the specified criteria."); + } + + return rows; + } + + private void AddInProcess(List dataRows) + { + foreach (var runnerFramework in _runnerFrameworks) + { + if (!runnerFramework.StartsWith("net4")) + { + continue; + } + + foreach (var runnerVersion in _runnerVersions) + { + foreach (var adapter in _adapters) + { + foreach (var adapterVersion in _adapterVersions) + { + AddRow(dataRows, runnerVersion, runnerFramework, runnerVersion, runnerFramework, adapter, adapterVersion, inIsolation: false); + } + } + } + } + } + + private void AddOlderConfigurations(List dataRows) + { + // Older configurations where the runner, host and adapter version are the same. + // We already added the row where all are newest when adding combination with all runners. + foreach (var runnerVersion in _runnerVersions.Skip(1)) + { + foreach (var runnerFramework in _runnerFrameworks) + { + foreach (var hostFramework in _hostFrameworks) + { + var isNetFramework = hostFramework.StartsWith("net4"); + var hostVersion = runnerVersion; + foreach (var adapter in _adapters) + { + var adapterVersion = runnerVersion; + AddRow(dataRows, runnerVersion, runnerFramework, hostVersion, hostFramework, adapter, adapterVersion, inIsolation: true); + } + } + } + } + } + + + private void AddEveryVersionOfAdapter(List dataRows) + { + var runnerVersion = _runnerVersions[0]; + foreach (var runnerFramework in _runnerFrameworks) + { + foreach (var hostFramework in _hostFrameworks) + { + var isNetFramework = hostFramework.StartsWith("net4"); + // .NET Framework testhost ships with the runner, and the version from the + // runner directory is always selected, otherwise select the newest version from _hostFrameworks. + var hostVersion = isNetFramework ? runnerVersion : _hostVersions[0]; + foreach (var adapter in _adapters) + { + // We already used the newest when adding combination with every runner + foreach (var adapterVersion in _adapterVersions.Skip(1)) + { + AddRow(dataRows, runnerVersion, runnerFramework, hostVersion, hostFramework, adapter, adapterVersion, inIsolation: true); + } + } + } + } + } + + private void AddEveryVersionOfHost(List dataRows) + { + var runnerVersion = _runnerVersions[0]; + + foreach (var runnerFramework in _runnerFrameworks) + { + foreach (var hostFramework in _hostFrameworks) + { + var isNetFramework = hostFramework.StartsWith("net4"); + // .NET Framework testhost ships with the runner, and the version from the + // runner directory is always the same as the runner. There are no variations + // so we just need to add host versions for .NET testhosts. We also skip the + // newest version because we already added it when AddEveryVersionOfRunner + var hostVersions = isNetFramework ? Array.Empty() : _hostVersions.Skip(1).ToArray(); + foreach (var hostVersion in hostVersions) + { + foreach (var adapter in _adapters) + { + // use the newest + var adapterVersion = _adapterVersions[0]; + AddRow(dataRows, runnerVersion, runnerFramework, hostVersion, hostFramework, adapter, adapterVersion, inIsolation: true); + } + } + } + } + } + + private void AddEveryVersionOfRunner(List dataRows) + { + foreach (var runnerVersion in _runnerVersions) + { + foreach (var runnerFramework in _runnerFrameworks) + { + foreach (var hostFramework in _hostFrameworks) + { + var isNetFramework = hostFramework.StartsWith("net4"); + // .NET Framework testhost ships with the runner, and the version from the + // runner directory is always selected, otherwise select the newest version from _hostFrameworks. + var hostVersion = isNetFramework ? runnerVersion : _hostVersions[0]; + foreach (var adapter in _adapters) + { + // use the newest + var adapterVersion = _adapterVersions[0]; + AddRow(dataRows, runnerVersion, runnerFramework, hostVersion, hostFramework, adapter, adapterVersion, inIsolation: true); + } + } + } + } + } + + private void AddRow(List dataRows, +string runnerVersion, string runnerFramework, string hostVersion, string hostFramework, string adapter, string adapterVersion, bool inIsolation) + { + RunnerInfo runnerInfo = GetRunnerInfo(runnerFramework, hostFramework, inIsolation); + runnerInfo.DebugInfo = GetDebugInfo(); + runnerInfo.VSTestConsoleInfo = GetVSTestConsoleInfo(runnerVersion, runnerInfo); + + // The order in which we add them matters. We end up both modifying the same path + // and adding to it. So the first one added will be later in the path. E.g.: + // Adding testSdk first: + // C:\p\vstest\test\TestAssets\MSTestProject1\bin\MSTestLatestPreview-2.2.9-preview-20220210-07\NETTestSdkLatest-17.2.0-dev\Debug\net451\MSTestProject1.dll + // versus adding testSdk second: + // C:\p\vstest\test\TestAssets\MSTestProject1\bin\NETTestSdkLatest-17.2.0-dev\MSTestLatestPreview-2.2.9-preview-20220210-07\Debug\net451\MSTestProject1.dll + runnerInfo.DllInfos.Add(GetMSTestInfo(adapterVersion)); + runnerInfo.DllInfos.Add(GetNetTestSdkInfo(hostVersion)); + dataRows.Add(runnerInfo); + } + + private DebugInfo GetDebugInfo() + { + return new DebugInfo + { + DebugDataCollector = DebugDataCollector, + DebugTesthost = DebugTesthost, + DebugVSTestConsole = DebugVSTestConsole, + NoDefaultBreakpoints = NoDefaultBreakpoints + }; + } + + + private RunnerInfo GetRunnerInfo(string runnerFramework, string hostFramework, bool inIsolation) + { + return new RunnerInfo + { + RunnerFramework = runnerFramework, + TargetFramework = hostFramework, + InIsolationValue = inIsolation ? AcceptanceTestBase.InIsolation : null + }; + } + + private MSTestInfo GetMSTestInfo(string msTestVersion) + { + var depsXml = GetDependenciesXml(); + + // It is okay when node is null, we check that Version has value when we update paths by using MSTestInfo, and throw. + // This way it throws in the body of the test which has better error reporting than throwing in the data source. + XmlNode? node = depsXml.DocumentElement?.SelectSingleNode($"PropertyGroup/MSTestFramework{msTestVersion}Version"); + var version = node?.InnerText.Replace("[", "").Replace("]", ""); + var slash = Path.DirectorySeparatorChar; + var versionSpecificBinPath = $"{slash}bin{slash}MSTest{msTestVersion}-{version}{slash}"; + + return new MSTestInfo(msTestVersion, version, versionSpecificBinPath); + } + + private static VSTestConsoleInfo GetVSTestConsoleInfo(string vstestConsoleVersion, RunnerInfo runnerInfo) + { + var depsXml = GetDependenciesXml(); + + // When version is Latest, we built it locally, but it gets restored into our nuget cache on build + // same as other versions, we just need to grab the version from a different property. + + var propertyName = vstestConsoleVersion == AcceptanceTestBase.LATEST + ? $"NETTestSdkVersion" + : $"VSTestConsole{vstestConsoleVersion}Version"; + + var packageName = runnerInfo.IsNetFrameworkRunner + ? "microsoft.testplatform" + : "microsoft.testplatform.cli"; + + // It is okay when node is null, we will fail to find the executable later, and throw. + // This way it throws in the body of the test which has better error reporting than throwing in the data source. + // And we can easily find out what is going on because --WRONG-VERSION-- sticks out, and is easy to find in the codebase. + XmlNode? node = depsXml.DocumentElement?.SelectSingleNode($"PropertyGroup/{propertyName}"); + var version = node?.InnerText.Replace("[", "").Replace("]", "") ?? "--WRONG-VERSION--"; + var vstestConsolePath = runnerInfo.IsNetFrameworkRunner + ? Path.Combine(IntegrationTestEnvironment.TestPlatformRootDirectory, "packages", packageName, version, + "tools", "net451", "Common7", "IDE", "Extensions", "TestPlatform", "vstest.console.exe") + : Path.Combine(IntegrationTestEnvironment.TestPlatformRootDirectory, "packages", packageName, version, + "contentFiles", "any", "netcoreapp2.1", "vstest.console.dll"); + + if (version.StartsWith("15.")) + { + vstestConsolePath = vstestConsolePath.Replace("netcoreapp2.1", "netcoreapp2.0"); + } + + return new VSTestConsoleInfo(vstestConsoleVersion, version, vstestConsolePath); + } + + private static NetTestSdkInfo GetNetTestSdkInfo(string testhostVersionType) + { + var depsXml = GetDependenciesXml(); + + // When version is Latest, we built it locally, but it gets restored into our nuget cache on build + // same as other versions, we just need to grab the version from a different property. + + var propertyName = testhostVersionType == AcceptanceTestBase.LATEST + ? $"NETTestSdkVersion" + : $"VSTestConsole{testhostVersionType}Version"; + + // It is okay when node is null, we check that Version has value when we update paths by using TesthostInfo, and throw. + // This way it throws in the body of the test which has better error reporting than throwing in the data source. + // + // We use the VSTestConsole properties to figure out testhost version, for now. + XmlNode? node = depsXml.DocumentElement?.SelectSingleNode($"PropertyGroup/{propertyName}"); + var version = node?.InnerText.Replace("[", "").Replace("]", ""); + var slash = Path.DirectorySeparatorChar; + var versionSpecificBinPath = $"{slash}bin{slash}NETTestSdk{testhostVersionType}-{version}{slash}"; + + return new NetTestSdkInfo(testhostVersionType, version, versionSpecificBinPath); + } + + + private static XmlDocument GetDependenciesXml() + { + if (s_depsXml != null) + return s_depsXml; + + var depsXmlPath = Path.Combine(IntegrationTestEnvironment.TestPlatformRootDirectory, "scripts", "build", "TestPlatform.Dependencies.props"); + var fileStream = File.OpenRead(depsXmlPath); + var xmlTextReader = new XmlTextReader(fileStream) { Namespaces = false }; + var depsXml = new XmlDocument(); + depsXml.Load(xmlTextReader); + + s_depsXml = depsXml; + return depsXml; + } +} + diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/Feature.cs b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/Feature.cs new file mode 100644 index 0000000000..9014349f06 --- /dev/null +++ b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/Feature.cs @@ -0,0 +1,16 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +namespace Microsoft.TestPlatform.AcceptanceTests; + +public class Feature +{ + public Feature(string version, string issue) + { + Version = version; + Issue = issue; + } + + public string Version { get; } + public string Issue { get; } +} diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/Features.cs b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/Features.cs new file mode 100644 index 0000000000..e82389e404 --- /dev/null +++ b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/Features.cs @@ -0,0 +1,23 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using System.Collections.Generic; + +namespace Microsoft.TestPlatform.AcceptanceTests; + +public static class Features +{ + public const string ATTACH_DEBUGGER = nameof(ATTACH_DEBUGGER); + public const string MSTEST_IFRAMEWORK_HANDLE_99 = nameof(MSTEST_IFRAMEWORK_HANDLE_99); + + + public static Dictionary TestPlatformFeatures { get; } = new Dictionary + { + [ATTACH_DEBUGGER] = new(version: "v16.7.0-preview-20200519-01", issue: "https://github.com/microsoft/vstest/pull/2325"), + }; + + public static Dictionary AdapterFeatures { get; internal set; } = new Dictionary + { + [MSTEST_IFRAMEWORK_HANDLE_99] = new("2.2.8", issue: "idk"), + }; +} diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/MSTestCompatibilityDataSource.cs b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/MSTestCompatibilityDataSource.cs index af3101c971..30875f6886 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/MSTestCompatibilityDataSource.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/MSTestCompatibilityDataSource.cs @@ -1,127 +1,73 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT license. See LICENSE file in the project root for full license information. -using System; -using System.Globalization; -using System.IO; -using System.Linq; using System.Reflection; -using System.Xml; - -using Microsoft.TestPlatform.TestUtilities; namespace Microsoft.TestPlatform.AcceptanceTests; -public sealed class MSTestCompatibilityDataSource : TestDataSource + +public class MSTestCompatibilityDataSource : TestDataSource { - private static XmlDocument? s_depsXml; - private readonly string[] _runnerFrameworks; - private readonly string[] _targetFrameworks; - private readonly string[] _msTestVersions; + private readonly CompatibilityRowsBuilder _builder; - /// - /// Initializes a new instance. - /// - /// To run tests with desktop runner(vstest.console.exe), use AcceptanceTestBase.Net452TargetFramework or alike values. - public MSTestCompatibilityDataSource(string runners = AcceptanceTestBase.DEFAULT_RUNNER_NETFX_AND_NET, string targetFrameworks = AcceptanceTestBase.DEFAULT_HOST_NETFX_AND_NET, string msTestVersions = AcceptanceTestBase.LATESTPREVIEW_TO_LEGACY) + public MSTestCompatibilityDataSource( + string runnerFrameworks = AcceptanceTestBase.DEFAULT_HOST_NETFX_AND_NET, + string hostFrameworks = AcceptanceTestBase.DEFAULT_HOST_NETFX_AND_NET, + string adapterVersions = AcceptanceTestBase.LATESTPREVIEW_TO_LEGACY) { - _runnerFrameworks = runners.Split(';'); - _targetFrameworks = targetFrameworks.Split(';'); - _msTestVersions = msTestVersions.Split(';'); - - // Do not generate the data rows here, properties (e.g. InProcess) are not populated until after constructor is done. + // TODO: We actually don't generate values to use different translation layers, because we don't have a good way to do + // that right now. Translation layer is loaded directly into the acceptance test, and so we don't have easy way to substitute it. + + _builder = new CompatibilityRowsBuilder( + runnerFrameworks, + // runner versions + AcceptanceTestBase.LATEST_TO_LEGACY, + hostFrameworks, + // host versions + AcceptanceTestBase.LATEST_TO_LEGACY, + adapterVersions, + // adapters + AcceptanceTestBase.MSTEST); + + // Do not generate the data rows here, properties (e.g. DebugVSTestConsole) are not populated until after constructor is done. } - /// - /// Add also run for in-process using the runner. - /// - // TODO: Can we somehow assert that we actually ran in process? - public bool InProcess { get; set; } public bool DebugVSTestConsole { get; set; } public bool DebugTesthost { get; set; } public bool DebugDataCollector { get; set; } public bool NoDefaultBreakpoints { get; set; } = true; - public override void CreateData(MethodInfo methodInfo) - { - var isWindows = Environment.OSVersion.Platform.ToString().StartsWith("Win"); - // Run .NET Framework tests only on Windows. - Func filter = tfm => isWindows || !tfm.StartsWith("net4"); - - if (InProcess) - { - foreach (var msTestVersion in _msTestVersions) - { - // We run in the .NET Framework runner process, the runner and target framework must agree. - var runnerInfo = new RunnerInfo(AcceptanceTestBase.DEFAULT_RUNNER_NETFX, AcceptanceTestBase.DEFAULT_RUNNER_NETFX, inIsolationValue: null); - runnerInfo.DebugInfo = new DebugInfo - { - DebugVSTestConsole = DebugVSTestConsole, - DebugTesthost = DebugTesthost, - DebugDataCollector = DebugDataCollector, - NoDefaultBreakpoints = NoDefaultBreakpoints, - }; - runnerInfo.DllInfos.Add(GetMSTestInfo(msTestVersion)); - - AddData(runnerInfo); - } - } - - foreach (var runner in _runnerFrameworks.Where(filter)) - { - foreach (var fmw in _targetFrameworks.Where(filter)) - { - foreach (var msTestVersion in _msTestVersions) - { - var runnerInfo = new RunnerInfo(runner, fmw, AcceptanceTestBase.InIsolation); - runnerInfo.DebugInfo = new DebugInfo - { - DebugVSTestConsole = DebugVSTestConsole, - DebugTesthost = DebugTesthost, - DebugDataCollector = DebugDataCollector, - NoDefaultBreakpoints = NoDefaultBreakpoints, - }; - runnerInfo.DllInfos.Add(GetMSTestInfo(msTestVersion)); - - AddData(runnerInfo); - } - } - } - } - - public string GetDisplayName(MethodInfo methodInfo, object[] data) - { - return string.Format(CultureInfo.CurrentCulture, "{0} ({1})", methodInfo.Name, string.Join(",", data)); - } - - private MSTestInfo GetMSTestInfo(string msTestVersion) - { - var depsXml = GetDependenciesXml(); + /// + /// Add run for in-process using the selected .NET Framework runners, and and all selected adapters. + /// + public bool InProcess { get; set; } - // It is okay when node is null, we check that Version has value when we update paths by using MSTestInfo, and throw. - // This way it throws in the body of the test which has better error reporting than throwing in the data source. - XmlNode? node = depsXml.DocumentElement?.SelectSingleNode($"PropertyGroup/MSTestFramework{msTestVersion}Version"); - var version = node?.InnerText.Replace("[", "").Replace("]", ""); - var slash = Path.DirectorySeparatorChar; - var versionSpecificBinPath = $"{slash}bin{slash}MSTest{msTestVersion}-{version}{slash}"; + public string? BeforeFeature { get; set; } + public string? AfterFeature { get; set; } - return new MSTestInfo(msTestVersion, version, versionSpecificBinPath); - } + public string? BeforeAdapterFeature { get; set; } + public string? AfterAdapterFeature { get; set; } - private static XmlDocument GetDependenciesXml() + public override void CreateData(MethodInfo methodInfo) { - if (s_depsXml != null) - return s_depsXml; - - var depsXmlPath = Path.Combine(IntegrationTestEnvironment.TestPlatformRootDirectory, "scripts", "build", "TestPlatform.Dependencies.props"); - var fileStream = File.OpenRead(depsXmlPath); - var xmlTextReader = new XmlTextReader(fileStream) { Namespaces = false }; - var depsXml = new XmlDocument(); - depsXml.Load(xmlTextReader); - - s_depsXml = depsXml; - return depsXml; + _builder.WithEveryVersionOfRunner = false; + _builder.WithEveryVersionOfHost = false; + _builder.WithEveryVersionOfAdapter = true; + _builder.WithOlderConfigurations = false; + _builder.WithInProcess = InProcess; + + _builder.BeforeFeature = BeforeFeature; + _builder.AfterFeature = AfterFeature; + _builder.BeforeAdapterFeature = BeforeAdapterFeature; + _builder.AfterAdapterFeature = AfterAdapterFeature; + + _builder.DebugDataCollector = DebugDataCollector; + _builder.DebugVSTestConsole = DebugVSTestConsole; + _builder.DebugTesthost = DebugTesthost; + _builder.NoDefaultBreakpoints = NoDefaultBreakpoints; + + var data = _builder.CreateData(); + data.ForEach(AddData); } } - diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/NetCoreRunner.cs b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/NetCoreRunner.cs index e12d7f6011..60d49baf1a 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/NetCoreRunner.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/NetCoreRunner.cs @@ -47,7 +47,12 @@ public IEnumerable GetData(MethodInfo methodInfo) Func filter = tfm => isWindows || !tfm.StartsWith("net4"); foreach (var fmw in _targetFrameworks.Split(';').Where(filter)) { - var runnerInfo = new RunnerInfo(IntegrationTestBase.CoreRunnerFramework, fmw, inIsolationValue: null); + var runnerInfo = new RunnerInfo + { + RunnerFramework = IntegrationTestBase.CoreRunnerFramework, + TargetFramework = fmw, + InIsolationValue = null + }; runnerInfo.DebugInfo = new DebugInfo { DebugVSTestConsole = DebugVSTestConsole, diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/NetCoreTargetFrameworkDataSource.cs b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/NetCoreTargetFrameworkDataSource.cs index d5bfd3b4ce..e4d646a8b8 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/NetCoreTargetFrameworkDataSource.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/NetCoreTargetFrameworkDataSource.cs @@ -55,7 +55,12 @@ public NetCoreTargetFrameworkDataSource( private void AddRunnerDataRow(List dataRows, string runnerFramework, string targetFramework) { - var runnerInfo = new RunnerInfo(runnerFramework, targetFramework, inIsolationValue: null); + var runnerInfo = new RunnerInfo + { + RunnerFramework = runnerFramework, + TargetFramework = targetFramework, + InIsolationValue = null + }; runnerInfo.DebugInfo = new DebugInfo { DebugDataCollector = DebugDataCollector, diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/NetFrameworkRunner.cs b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/NetFrameworkRunner.cs index 07f30ee823..c7be429345 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/NetFrameworkRunner.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/NetFrameworkRunner.cs @@ -49,7 +49,12 @@ public IEnumerable GetData(MethodInfo methodInfo) foreach (var fmw in _targetFrameworks.Split(';')) { - var runnerInfo = new RunnerInfo(IntegrationTestBase.DesktopRunnerFramework, fmw, AcceptanceTestBase.InIsolation); + var runnerInfo = new RunnerInfo + { + RunnerFramework = IntegrationTestBase.DesktopRunnerFramework, + TargetFramework = fmw, + InIsolationValue = AcceptanceTestBase.InIsolation + }; runnerInfo.DebugInfo = new DebugInfo { DebugVSTestConsole = DebugVSTestConsole, diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/NetFullTargetFrameworkDataSource.cs b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/NetFullTargetFrameworkDataSource.cs index 3b45a19e74..468aded7f7 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/NetFullTargetFrameworkDataSource.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/NetFullTargetFrameworkDataSource.cs @@ -53,7 +53,12 @@ public IEnumerable GetData(MethodInfo methodInfo) var isWindows = Environment.OSVersion.Platform.ToString().StartsWith("Win"); if (_useCoreRunner && isWindows) { - var runnerInfo = new RunnerInfo(IntegrationTestBase.CoreRunnerFramework, AcceptanceTestBase.DesktopTargetFramework, inIsolationValue: null); + var runnerInfo = new RunnerInfo + { + RunnerFramework = IntegrationTestBase.CoreRunnerFramework, + TargetFramework = AcceptanceTestBase.DesktopTargetFramework, + InIsolationValue = null + }; runnerInfo.DebugInfo = new DebugInfo { DebugVSTestConsole = DebugVSTestConsole, @@ -68,7 +73,12 @@ public IEnumerable GetData(MethodInfo methodInfo) { if (_inIsolation) { - var runnerInfo = new RunnerInfo(IntegrationTestBase.DesktopRunnerFramework, AcceptanceTestBase.DesktopTargetFramework, AcceptanceTestBase.InIsolation); + var runnerInfo = new RunnerInfo + { + RunnerFramework = IntegrationTestBase.DesktopRunnerFramework, + TargetFramework = AcceptanceTestBase.DesktopTargetFramework, + InIsolationValue = AcceptanceTestBase.InIsolation + }; runnerInfo.DebugInfo = new DebugInfo { DebugVSTestConsole = DebugVSTestConsole, @@ -81,7 +91,12 @@ public IEnumerable GetData(MethodInfo methodInfo) if (_inProcess) { - var runnerInfo = new RunnerInfo(IntegrationTestBase.DesktopRunnerFramework, AcceptanceTestBase.DesktopTargetFramework, inIsolationValue: null); + var runnerInfo = new RunnerInfo + { + RunnerFramework = IntegrationTestBase.DesktopRunnerFramework, + TargetFramework = AcceptanceTestBase.DesktopTargetFramework, + InIsolationValue = null + }; runnerInfo.DebugInfo = new DebugInfo { DebugVSTestConsole = DebugVSTestConsole, diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/RunnnerInfo.cs b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/RunnnerInfo.cs index fcf9cca46e..bb30619479 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/RunnnerInfo.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/RunnnerInfo.cs @@ -17,19 +17,12 @@ namespace Microsoft.TestPlatform.AcceptanceTests; [Serializable] public class RunnerInfo { - public RunnerInfo(string runnerFramework, string targetFramework, string? inIsolationValue = "") - { - RunnerFramework = runnerFramework; - TargetFramework = targetFramework; - InIsolationValue = inIsolationValue; - } + public string? RunnerFramework { get; set; } - public string RunnerFramework { get; set; } + public VSTestConsoleInfo? VSTestConsoleInfo { get; set; } - public VSTestConsoleInfo VSTestConsoleInfo { get; set; } - - public string TargetFramework { get; set; } + public string? TargetFramework { get; set; } public string? InIsolationValue { get; set; } public DebugInfo? DebugInfo { get; set; } @@ -44,7 +37,7 @@ public RunnerInfo(string runnerFramework, string targetFramework, string? inIsol /// /// Is running via .NET Framework vstest.console? /// - public bool IsNetFrameworkRunner => RunnerFramework.StartsWith("net4", StringComparison.InvariantCultureIgnoreCase); + public bool IsNetFrameworkRunner => RunnerFramework!.StartsWith("net4", StringComparison.InvariantCultureIgnoreCase); /// /// Is running via .NET "Core" testhost? @@ -54,7 +47,7 @@ public RunnerInfo(string runnerFramework, string targetFramework, string? inIsol /// /// Is running via .NET Framework testhost? /// - public bool IsNetFrameworkTarget => TargetFramework.StartsWith("net4", StringComparison.InvariantCultureIgnoreCase); + public bool IsNetFrameworkTarget => TargetFramework!.StartsWith("net4", StringComparison.InvariantCultureIgnoreCase); public override string ToString() => $"Runner = {RunnerFramework}, TargetFramework = {TargetFramework}, {(string.IsNullOrEmpty(InIsolationValue) ? "InProcess" : "InIsolation")}, {VSTestConsoleInfo}, {string.Join(",", DllInfos)}"; } diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/TestDataSource.cs b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/TestDataSource.cs index abfead224e..d041d36e79 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/TestDataSource.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/TestDataSource.cs @@ -106,6 +106,7 @@ public abstract class TestDataSource : Attribute, ITestDataSourc where T1 : notnull where T2 : notnull where T3 : notnull + where T4 : notnull { private readonly List _data = new(); diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/TestPlatformCompatibilityDataSource.cs b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/TestPlatformCompatibilityDataSource.cs index eca9ac55e6..b607811116 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/TestPlatformCompatibilityDataSource.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/TestPlatformCompatibilityDataSource.cs @@ -1,29 +1,30 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT license. See LICENSE file in the project root for full license information. -using System; -using System.Collections.Generic; -using System.Globalization; -using System.IO; -using System.Linq; using System.Reflection; -using System.Xml; - -using Microsoft.TestPlatform.TestUtilities; - -using Semver; namespace Microsoft.TestPlatform.AcceptanceTests; -public sealed class TestPlatformCompatibilityDataSource : TestDataSource +/// +/// A data source that provides a huge mix of runners, hosts and mstest adapter, to add up >100 tests. +/// You can control which runner versions, host versions, and adapter versions will be used. This should be +/// used only to test the most common scenarios, or special configurations that are candidates for their own +/// specialized source. +/// +/// By default net451 and netcoreapp2.1 are used for both runner and host. (4 combinations) +/// Then run with every version of runner is added. +/// Then run with every version of test.sdk is added. +/// Then run with every combination of testhost and adapter is added. +/// And then run in process is added. +/// +/// All of those are filtered down to have no duplicates, and to pass the +/// Before and After platform version filters, and adapter filters. +/// +/// When that adds up to no configuration exception is thrown. +/// +public class TestPlatformCompatibilityDataSource : TestDataSource { - private static XmlDocument? s_depsXml; - private readonly string[] _runnerFrameworks; - private readonly string[] _runnerVersions; - private readonly string[] _hostFrameworks; - private readonly string[] _adapterVersions; - private readonly string[] _adapters; - private readonly string[] _hostVersions; + private readonly CompatibilityRowsBuilder _builder; public TestPlatformCompatibilityDataSource( string runnerFrameworks = AcceptanceTestBase.DEFAULT_HOST_NETFX_AND_NET, @@ -35,14 +36,8 @@ public TestPlatformCompatibilityDataSource( { // TODO: We actually don't generate values to use different translation layers, because we don't have a good way to do // that right now. Translation layer is loaded directly into the acceptance test, and so we don't have easy way to substitute it. - // I am keeping this source separate from vstest console compatibility data source, to be able to easily add this feature later. - _runnerFrameworks = runnerFrameworks.Split(';'); - _runnerVersions = runnerVersions.Split(';'); - _hostFrameworks = hostFrameworks.Split(';'); - _hostVersions = hostVersions.Split(';'); - _adapterVersions = adapterVersions.Split(';'); - _adapters = adapters.Split(';'); + _builder = new CompatibilityRowsBuilder(runnerFrameworks, runnerVersions, hostFrameworks, hostVersions, adapterVersions, adapters); // Do not generate the data rows here, properties (e.g. DebugVSTestConsole) are not populated until after constructor is done. } @@ -72,275 +67,23 @@ public TestPlatformCompatibilityDataSource( public override void CreateData(MethodInfo methodInfo) { - var dataRows = new List(); - - if (WithEveryVersionOfRunner) - AddEveryVersionOfRunner(dataRows); - - if (WithEveryVersionOfHost) - AddEveryVersionOfHost(dataRows); - - if (WithEveryVersionOfAdapter) - AddEveryVersionOfAdapter(dataRows); - - if (WithOlderConfigurations) - AddOlderConfigurations(dataRows); - - if (WithInProcess) - AddInProcess(dataRows); - - var c = dataRows.Count(); - - if (BeforeFeature != null && AfterFeature != null) - { - throw new InvalidOperationException($"You cannot specify {nameof(BeforeFeature)} and {nameof(AfterFeature)} at the same time"); - } - - var minVersion = SemVersion.Parse("0.0.0-alpha.1"); - SemVersion? beforeVersion = null; - SemVersion? afterVersion = null; - if (BeforeFeature != null) - { - var feature = Features.Table[BeforeFeature]; - beforeVersion = SemVersion.Parse(feature.Version.TrimStart('v')); - } - if (AfterFeature != null) - { - var feature = Features.Table[AfterFeature]; - afterVersion = SemVersion.Parse(feature.Version.TrimStart('v')); - } - - var isWindows = Environment.OSVersion.Platform.ToString().StartsWith("Win"); - // Run .NET Framework tests only on Windows. - Func filter = tfm => isWindows || !tfm.StartsWith("net4"); - - // TODO: maybe we should throw if we don't end up generating any data - // because none of the versions match, or some other way to identify tests that will never run because they are very outdated. - // We probably don't have that need right now, because legacy version is 15.x.x, which is very old, and we are still keeping - // compatibility. - - foreach (var dataRow in dataRows) - { - AddData(dataRow); - } - } - - private void AddInProcess(List dataRows) - { - foreach (var runnerFramework in _runnerFrameworks) - { - if (!runnerFramework.StartsWith("net4")) - { - continue; - } - - foreach (var runnerVersion in _runnerVersions) - { - foreach (var adapter in _adapters) - { - foreach (var adapterVersion in _adapterVersions) - { - AddRow(dataRows, runnerVersion, runnerFramework, runnerVersion, runnerFramework, adapter, adapterVersion, inIsolation: false); - } - } - } - } - } - - private void AddOlderConfigurations(List dataRows) - { - // Older configurations where the runner, host and adapter version are the same. - // We already added the row where all are newest when adding combination with all runners. - foreach (var runnerVersion in _runnerVersions.Skip(1)) - { - foreach (var runnerFramework in _runnerFrameworks) - { - foreach (var hostFramework in _hostFrameworks) - { - var isNetFramework = hostFramework.StartsWith("net4"); - var hostVersion = runnerVersion; - foreach (var adapter in _adapters) - { - var adapterVersion = runnerVersion; - AddRow(dataRows, runnerVersion, runnerFramework, hostVersion, hostFramework, adapter, adapterVersion, inIsolation: true); - } - } - } - } - } - - private void AddEveryVersionOfAdapter(List dataRows) - { - var runnerVersion = _runnerVersions[0]; - foreach (var runnerFramework in _runnerFrameworks) - { - foreach (var hostFramework in _hostFrameworks) - { - var isNetFramework = hostFramework.StartsWith("net4"); - // .NET Framework testhost ships with the runner, and the version from the - // runner directory is always selected, otherwise select the newest version from _hostFrameworks. - var hostVersion = isNetFramework ? runnerVersion : _hostVersions[0]; - foreach (var adapter in _adapters) - { - // We already used the newest when adding combination with every runner - foreach (var adapterVersion in _adapterVersions.Skip(1)) - { - AddRow(dataRows, runnerVersion, runnerFramework, hostVersion, hostFramework, adapter, adapterVersion, inIsolation: true); - } - } - } - } - } - - private void AddEveryVersionOfHost(List dataRows) - { - var runnerVersion = _runnerVersions[0]; - - foreach (var runnerFramework in _runnerFrameworks) - { - foreach (var hostFramework in _hostFrameworks) - { - var isNetFramework = hostFramework.StartsWith("net4"); - // .NET Framework testhost ships with the runner, and the version from the - // runner directory is always the same as the runner. There are no variations - // so we just need to add host versions for .NET testhosts. We also skip the - // newest version because we already added it when AddEveryVersionOfRunner - var hostVersions = isNetFramework ? Array.Empty() : _hostVersions.Skip(1).ToArray(); - foreach (var hostVersion in hostVersions) - { - foreach (var adapter in _adapters) - { - // use the newest - var adapterVersion = _adapterVersions[0]; - AddRow(dataRows, runnerVersion, runnerFramework, hostVersion, hostFramework, adapter, adapterVersion, inIsolation: true); - } - } - } - } - } - - private void AddEveryVersionOfRunner(List dataRows) - { - foreach (var runnerVersion in _runnerVersions) - { - foreach (var runnerFramework in _runnerFrameworks) - { - foreach (var hostFramework in _hostFrameworks) - { - var isNetFramework = hostFramework.StartsWith("net4"); - // .NET Framework testhost ships with the runner, and the version from the - // runner directory is always selected, otherwise select the newest version from _hostFrameworks. - var hostVersion = isNetFramework ? runnerVersion : _hostVersions[0]; - foreach (var adapter in _adapters) - { - // use the newest - var adapterVersion = _adapterVersions[0]; - AddRow(dataRows, runnerVersion, runnerFramework, hostVersion, hostFramework, adapter, adapterVersion, inIsolation: true); - } - } - } - } - } - - private void AddRow(List dataRows, - string runnerVersion, string runnerFramework, string hostVersion, string hostFramework, string adapter, string adapterVersion, bool inIsolation) - { - RunnerInfo runnerInfo = GetRunnerInfo(runnerFramework, hostFramework, inIsolation); - runnerInfo.DebugInfo = GetDebugInfo(); - runnerInfo.VSTestConsoleInfo = GetVSTestConsoleInfo(runnerVersion, runnerInfo); - - // The order in which we add them matters. We end up both modifying the same path - // and adding to it. So the first one added will be later in the path. E.g.: - // Adding testSdk first: - // C:\p\vstest\test\TestAssets\MSTestProject1\bin\MSTestLatestPreview-2.2.9-preview-20220210-07\NETTestSdkLatest-17.2.0-dev\Debug\net451\MSTestProject1.dll - // versus adding testSdk second: - // C:\p\vstest\test\TestAssets\MSTestProject1\bin\NETTestSdkLatest-17.2.0-dev\MSTestLatestPreview-2.2.9-preview-20220210-07\Debug\net451\MSTestProject1.dll - runnerInfo.DllInfos.Add(GetMSTestInfo(adapterVersion)); - runnerInfo.DllInfos.Add(TesthostCompatibilityDataSource.GetNetTestSdkInfo(hostVersion)); - dataRows.Add(runnerInfo); - } - - private DebugInfo GetDebugInfo() - { - return new DebugInfo - { - DebugDataCollector = DebugDataCollector, - DebugTesthost = DebugTesthost, - DebugVSTestConsole = DebugVSTestConsole, - NoDefaultBreakpoints = NoDefaultBreakpoints - }; - } - - private RunnerInfo GetRunnerInfo(string runnerFramework, string hostFramework, bool inIsolation) - { - return new RunnerInfo(runnerFramework, hostFramework, inIsolation ? AcceptanceTestBase.InIsolation : null); - } - - public string GetDisplayName(MethodInfo methodInfo, object[] data) - { - return string.Format(CultureInfo.CurrentCulture, "{0} ({1})", methodInfo.Name, string.Join(",", data)); - } - - private MSTestInfo GetMSTestInfo(string msTestVersion) - { - var depsXml = GetDependenciesXml(); - - // It is okay when node is null, we check that Version has value when we update paths by using MSTestInfo, and throw. - // This way it throws in the body of the test which has better error reporting than throwing in the data source. - XmlNode? node = depsXml.DocumentElement?.SelectSingleNode($"PropertyGroup/MSTestFramework{msTestVersion}Version"); - var version = node?.InnerText.Replace("[", "").Replace("]", ""); - var slash = Path.DirectorySeparatorChar; - var versionSpecificBinPath = $"{slash}bin{slash}MSTest{msTestVersion}-{version}{slash}"; - - return new MSTestInfo(msTestVersion, version, versionSpecificBinPath); - } - - internal static VSTestConsoleInfo GetVSTestConsoleInfo(string vstestConsoleVersion, RunnerInfo runnerInfo) - { - var depsXml = GetDependenciesXml(); - - // When version is Latest, we built it locally, but it gets restored into our nuget cache on build - // same as other versions, we just need to grab the version from a different property. - - var propertyName = vstestConsoleVersion == AcceptanceTestBase.LATEST - ? $"NETTestSdkVersion" - : $"VSTestConsole{vstestConsoleVersion}Version"; - - var packageName = runnerInfo.IsNetFrameworkRunner - ? "microsoft.testplatform" - : "microsoft.testplatform.cli"; - - // It is okay when node is null, we will fail to find the executable later, and throw. - // This way it throws in the body of the test which has better error reporting than throwing in the data source. - // And we can easily find out what is going on because --WRONG-VERSION-- sticks out, and is easy to find in the codebase. - XmlNode? node = depsXml.DocumentElement?.SelectSingleNode($"PropertyGroup/{propertyName}"); - var version = node?.InnerText.Replace("[", "").Replace("]", "") ?? "--WRONG-VERSION--"; - var vstestConsolePath = runnerInfo.IsNetFrameworkRunner - ? Path.Combine(IntegrationTestEnvironment.TestPlatformRootDirectory, "packages", packageName, version, - "tools", "net451", "Common7", "IDE", "Extensions", "TestPlatform", "vstest.console.exe") - : Path.Combine(IntegrationTestEnvironment.TestPlatformRootDirectory, "packages", packageName, version, - "contentFiles", "any", "netcoreapp2.1", "vstest.console.dll"); - - if (version.StartsWith("15.")) - { - vstestConsolePath = vstestConsolePath.Replace("netcoreapp2.1", "netcoreapp2.0"); - } - - return new VSTestConsoleInfo(vstestConsoleVersion, version, vstestConsolePath); - } - - private static XmlDocument GetDependenciesXml() - { - if (s_depsXml != null) - return s_depsXml; - - var depsXmlPath = Path.Combine(IntegrationTestEnvironment.TestPlatformRootDirectory, "scripts", "build", "TestPlatform.Dependencies.props"); - var fileStream = File.OpenRead(depsXmlPath); - var xmlTextReader = new XmlTextReader(fileStream) { Namespaces = false }; - var depsXml = new XmlDocument(); - depsXml.Load(xmlTextReader); - - s_depsXml = depsXml; - return depsXml; + _builder.WithEveryVersionOfRunner = WithEveryVersionOfRunner; + _builder.WithEveryVersionOfHost = WithEveryVersionOfHost; + _builder.WithEveryVersionOfAdapter = WithEveryVersionOfAdapter; + _builder.WithOlderConfigurations = WithOlderConfigurations; + _builder.WithInProcess = WithInProcess; + + _builder.BeforeFeature = BeforeFeature; + _builder.AfterFeature = AfterFeature; + _builder.BeforeAdapterFeature = BeforeAdapterFeature; + _builder.AfterAdapterFeature = AfterAdapterFeature; + + _builder.DebugDataCollector = DebugDataCollector; + _builder.DebugVSTestConsole = DebugVSTestConsole; + _builder.DebugTesthost = DebugTesthost; + _builder.NoDefaultBreakpoints = NoDefaultBreakpoints; + + var data = _builder.CreateData(); + data.ForEach(AddData); } } diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/TesthostCompatibilityDataSource.cs b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/TesthostCompatibilityDataSource.cs index a74c45adc9..e32c32aa87 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/TesthostCompatibilityDataSource.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/TesthostCompatibilityDataSource.cs @@ -1,37 +1,39 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT license. See LICENSE file in the project root for full license information. -using System; -using System.Globalization; -using System.IO; -using System.Linq; using System.Reflection; -using System.Xml; - -using Microsoft.TestPlatform.TestUtilities; - namespace Microsoft.TestPlatform.AcceptanceTests; -public sealed class TesthostCompatibilityDataSource : TestDataSource +/// +/// A data source that provides every testhost. +/// +/// When that adds up to no configuration exception is thrown. +/// +public class HostCompatibilityDataSource : TestDataSource { - private static XmlDocument? s_depsXml; - private readonly string[] _runnerFrameworks; - private readonly string[] _targetFrameworks; - private readonly string[] _testhostVersions; + private readonly CompatibilityRowsBuilder _builder; - /// - /// Initializes a new instance. - /// - /// To run tests with desktop runner(vstest.console.exe), use AcceptanceTestBase.Net452TargetFramework or alike values. - public TesthostCompatibilityDataSource(string runners = AcceptanceTestBase.DEFAULT_RUNNER_NETFX_AND_NET, string targetFrameworks = AcceptanceTestBase.DEFAULT_HOST_NETFX_AND_NET, - string runnerVersions = AcceptanceTestBase.LATESTPREVIEW_TO_LEGACY, string testhostVersions = AcceptanceTestBase.LATESTPREVIEW_TO_LEGACY) + public HostCompatibilityDataSource( + string runnerFrameworks = AcceptanceTestBase.DEFAULT_HOST_NETFX_AND_NET, + string hostFrameworks = AcceptanceTestBase.DEFAULT_HOST_NETFX_AND_NET, + string hostVersions = AcceptanceTestBase.LATEST_TO_LEGACY) { - _runnerFrameworks = runners.Split(';'); - _targetFrameworks = targetFrameworks.Split(';'); - _testhostVersions = testhostVersions.Split(';'); - - // Do not generate the data rows here, properties (e.g. InProcess) are not populated until after constructor is done. + // TODO: We actually don't generate values to use different translation layers, because we don't have a good way to do + // that right now. Translation layer is loaded directly into the acceptance test, and so we don't have easy way to substitute it. + + _builder = new CompatibilityRowsBuilder( + runnerFrameworks, + // runner versions + AcceptanceTestBase.LATEST, + hostFrameworks, + hostVersions, + // adapter versions + AcceptanceTestBase.LATESTSTABLE, + // adapter + AcceptanceTestBase.MSTEST); + + // Do not generate the data rows here, properties (e.g. DebugVSTestConsole) are not populated until after constructor is done. } public bool DebugVSTestConsole { get; set; } @@ -39,110 +41,30 @@ public TesthostCompatibilityDataSource(string runners = AcceptanceTestBase.DEFAU public bool DebugDataCollector { get; set; } public bool NoDefaultBreakpoints { get; set; } = true; - public override void CreateData(MethodInfo methodInfo) - { - var isWindows = Environment.OSVersion.Platform.ToString().StartsWith("Win"); - // Run .NET Framework tests only on Windows. - Func filter = tfm => isWindows || tfm.StartsWith("net4"); - var onlyLatest = new[] { AcceptanceTestBase.LATEST }; - - // Test different versions of vstest console together with latest testhost. - // There is no point in proving that old testhost does not work with old vstest.console - // when we can patch neither. - foreach (var runner in _runnerFrameworks.Where(filter)) - { - foreach (var fmw in _targetFrameworks.Where(filter)) - { - // For .NET Framework generate only latest console with latest testhost, - // we cannot control the version of testhost because it is shipped with the console. - var testhostVersions = runner.StartsWith("net4") ? onlyLatest : _testhostVersions; - foreach (var testhostVersion in testhostVersions) - { - var runnerInfo = new RunnerInfo(runner, fmw, AcceptanceTestBase.InIsolation); - runnerInfo.DebugInfo = new DebugInfo - { - DebugDataCollector = DebugDataCollector, - DebugTesthost = DebugTesthost, - DebugVSTestConsole = DebugVSTestConsole, - NoDefaultBreakpoints = NoDefaultBreakpoints - }; - runnerInfo.VSTestConsoleInfo = TranslationLayerCompatibilityDataSource.GetVSTestConsoleInfo(AcceptanceTestBase.LATEST, runnerInfo); - runnerInfo.DllInfos.Add(GetNetTestSdkInfo(testhostVersion)); + public string? BeforeFeature { get; set; } + public string? AfterFeature { get; set; } + public string? BeforeAdapterFeature { get; set; } + public string? AfterAdapterFeature { get; set; } - AddData(runnerInfo); - } - } - } - - // Test different versions of vstest console together with latest vstest.console. - // There is no point in proving that old testhost does not work with old vstest.console - // when we can patch neither. - foreach (var runner in _runnerFrameworks.Where(filter)) - { - foreach (var fmw in _targetFrameworks.Where(filter)) - { - // For .NET Framework generate only latest console with latest testhost, - // we cannot control the version of testhost because it is shipped with the console. - var consoleVersions = runner.StartsWith("net4") ? onlyLatest : _testhostVersions; - foreach (var consoleVersion in consoleVersions) - { - var runnerInfo = new RunnerInfo(runner, fmw, AcceptanceTestBase.InIsolation); - runnerInfo.DebugInfo = new DebugInfo - { - DebugDataCollector = DebugDataCollector, - DebugTesthost = DebugTesthost, - DebugVSTestConsole = DebugVSTestConsole, - NoDefaultBreakpoints = NoDefaultBreakpoints - }; - runnerInfo.VSTestConsoleInfo = TranslationLayerCompatibilityDataSource.GetVSTestConsoleInfo(consoleVersion, runnerInfo); - runnerInfo.DllInfos.Add(GetNetTestSdkInfo(AcceptanceTestBase.LATEST)); - - AddData(runnerInfo); - } - } - } - } - - public string GetDisplayName(MethodInfo methodInfo, object[] data) - { - return string.Format(CultureInfo.CurrentCulture, "{0} ({1})", methodInfo.Name, string.Join(",", data)); - } - - internal static NetTestSdkInfo GetNetTestSdkInfo(string testhostVersionType) - { - var depsXml = GetDependenciesXml(); - - // When version is Latest, we built it locally, but it gets restored into our nuget cache on build - // same as other versions, we just need to grab the version from a different property. - - var propertyName = testhostVersionType == AcceptanceTestBase.LATEST - ? $"NETTestSdkVersion" - : $"VSTestConsole{testhostVersionType}Version"; - - // It is okay when node is null, we check that Version has value when we update paths by using TesthostInfo, and throw. - // This way it throws in the body of the test which has better error reporting than throwing in the data source. - // - // We use the VSTestConsole properties to figure out testhost version, for now. - XmlNode? node = depsXml.DocumentElement?.SelectSingleNode($"PropertyGroup/{propertyName}"); - var version = node?.InnerText.Replace("[", "").Replace("]", ""); - var slash = Path.DirectorySeparatorChar; - var versionSpecificBinPath = $"{slash}bin{slash}NETTestSdk{testhostVersionType}-{version}{slash}"; - - return new NetTestSdkInfo(testhostVersionType, version, versionSpecificBinPath); - } - - private static XmlDocument GetDependenciesXml() + public override void CreateData(MethodInfo methodInfo) { - if (s_depsXml != null) - return s_depsXml; - - var depsXmlPath = Path.Combine(IntegrationTestEnvironment.TestPlatformRootDirectory, "scripts", "build", "TestPlatform.Dependencies.props"); - var fileStream = File.OpenRead(depsXmlPath); - var xmlTextReader = new XmlTextReader(fileStream) { Namespaces = false }; - var depsXml = new XmlDocument(); - depsXml.Load(xmlTextReader); - - s_depsXml = depsXml; - return depsXml; + _builder.WithEveryVersionOfRunner = false; + _builder.WithEveryVersionOfHost = true; + _builder.WithEveryVersionOfAdapter = false; + _builder.WithOlderConfigurations = false; + _builder.WithInProcess = false; + + _builder.BeforeFeature = BeforeFeature; + _builder.AfterFeature = AfterFeature; + _builder.BeforeAdapterFeature = BeforeAdapterFeature; + _builder.AfterAdapterFeature = AfterAdapterFeature; + + _builder.DebugDataCollector = DebugDataCollector; + _builder.DebugVSTestConsole = DebugVSTestConsole; + _builder.DebugTesthost = DebugTesthost; + _builder.NoDefaultBreakpoints = NoDefaultBreakpoints; + + var data = _builder.CreateData(); + data.ForEach(AddData); } } diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/TranslationLayerCompatibilityDataSource.cs b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/TranslationLayerCompatibilityDataSource.cs index 50dfd86bbf..d55cfe0c62 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/TranslationLayerCompatibilityDataSource.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/TranslationLayerCompatibilityDataSource.cs @@ -1,43 +1,38 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT license. See LICENSE file in the project root for full license information. -using System; -using System.Collections.Generic; -using System.Globalization; -using System.IO; -using System.Linq; using System.Reflection; -using System.Xml; - -using Microsoft.TestPlatform.TestUtilities; - -using Semver; namespace Microsoft.TestPlatform.AcceptanceTests; -public sealed class TranslationLayerCompatibilityDataSource : TestDataSource + +/// +/// A data source that provides every version of runner. +/// +/// When that adds up to no configuration exception is thrown. +/// +public class RunnerCompatibilityDataSource : TestDataSource { - private static XmlDocument? s_depsXml; - private readonly string[] _runnerFrameworks; - private readonly string[] _targetFrameworks; - // private readonly string[] _translationLayerVersions; - private readonly string[] _vstestConsoleVersions; + private readonly CompatibilityRowsBuilder _builder; - /// - /// Initializes a new instance. - /// - /// To run tests with desktop runner(vstest.console.exe), use AcceptanceTestBase.Net452TargetFramework or alike values. - public TranslationLayerCompatibilityDataSource( - string runners = AcceptanceTestBase.DEFAULT_RUNNER_NETFX_AND_NET, - string targetFrameworks = AcceptanceTestBase.DEFAULT_HOST_NETFX_AND_NET, - string vstestConsoleVersions = AcceptanceTestBase.LATEST_TO_LEGACY) + public RunnerCompatibilityDataSource( + string runnerFrameworks = AcceptanceTestBase.DEFAULT_HOST_NETFX_AND_NET, + string runnerVersions = AcceptanceTestBase.LATEST_TO_LEGACY, + string hostFrameworks = AcceptanceTestBase.DEFAULT_HOST_NETFX_AND_NET) { // TODO: We actually don't generate values to use different translation layers, because we don't have a good way to do // that right now. Translation layer is loaded directly into the acceptance test, and so we don't have easy way to substitute it. - // I am keeping this source separate from vstest console compatibility data source, to be able to easily add this feature later. - _runnerFrameworks = runners.Split(';'); - _targetFrameworks = targetFrameworks.Split(';'); - _vstestConsoleVersions = vstestConsoleVersions.Split(';'); + + _builder = new CompatibilityRowsBuilder( + runnerFrameworks, + runnerVersions, + hostFrameworks, + // host versions + AcceptanceTestBase.LATEST, + // adapter versions + AcceptanceTestBase.LATESTSTABLE, + // adapters + AcceptanceTestBase.MSTEST); // Do not generate the data rows here, properties (e.g. DebugVSTestConsole) are not populated until after constructor is done. } @@ -47,140 +42,36 @@ public TranslationLayerCompatibilityDataSource( public bool DebugDataCollector { get; set; } public bool NoDefaultBreakpoints { get; set; } = true; + /// + /// Add run for in-process using the selected .NET Framework runners, and and all selected adapters. + /// + public bool InProcess { get; set; } + public string? BeforeFeature { get; set; } public string? AfterFeature { get; set; } + public string? BeforeAdapterFeature { get; set; } + public string? AfterAdapterFeature { get; set; } public override void CreateData(MethodInfo methodInfo) { - if (BeforeFeature != null && AfterFeature != null) - { - throw new InvalidOperationException($"You cannot specify {nameof(BeforeFeature)} and {nameof(AfterFeature)} at the same time"); - } - - var minVersion = SemVersion.Parse("0.0.0-alpha.1"); - SemVersion? beforeVersion = null; - SemVersion? afterVersion = null; - if (BeforeFeature != null) - { - var feature = Features.Table[BeforeFeature]; - beforeVersion = SemVersion.Parse(feature.Version.TrimStart('v')); - } - if (AfterFeature != null) - { - var feature = Features.Table[AfterFeature]; - afterVersion = SemVersion.Parse(feature.Version.TrimStart('v')); - } - - var isWindows = Environment.OSVersion.Platform.ToString().StartsWith("Win"); - // Run .NET Framework tests only on Windows. - Func filter = tfm => isWindows || !tfm.StartsWith("net4"); - - // TODO: maybe we should throw if we don't end up generating any data - // because none of the versions match, or some other way to identify tests that will never run because they are very outdated. - // We probably don't have that need right now, because legacy version is 15.x.x, which is very old, and we are still keeping - // compatibility. - foreach (var runner in _runnerFrameworks.Where(filter)) - { - foreach (var fmw in _targetFrameworks.Where(filter)) - { - foreach (var vstestConsoleVersion in _vstestConsoleVersions) - { - var runnerInfo = new RunnerInfo(runner, fmw, AcceptanceTestBase.InIsolation); - runnerInfo.DebugInfo = new DebugInfo - { - DebugVSTestConsole = DebugVSTestConsole, - DebugTesthost = DebugTesthost, - DebugDataCollector = DebugDataCollector, - NoDefaultBreakpoints = NoDefaultBreakpoints, - }; - var vsTestConsoleInfo = GetVSTestConsoleInfo(vstestConsoleVersion, runnerInfo); - - if (beforeVersion != null && vsTestConsoleInfo.Version > beforeVersion) - continue; - - if (afterVersion != null && vsTestConsoleInfo.Version < afterVersion) - continue; - - AddData(runnerInfo, vsTestConsoleInfo); - } - } - } - } - - public string GetDisplayName(MethodInfo methodInfo, object[] data) - { - return string.Format(CultureInfo.CurrentCulture, "{0} ({1})", methodInfo.Name, string.Join(",", data)); - } - - internal static VSTestConsoleInfo GetVSTestConsoleInfo(string vstestConsoleVersion, RunnerInfo runnerInfo) - { - var depsXml = GetDependenciesXml(); - - // When version is Latest, we built it locally, but it gets restored into our nuget cache on build - // same as other versions, we just need to grab the version from a different property. - - var propertyName = vstestConsoleVersion == AcceptanceTestBase.LATEST - ? $"NETTestSdkVersion" - : $"VSTestConsole{vstestConsoleVersion}Version"; - - var packageName = runnerInfo.IsNetFrameworkRunner - ? "microsoft.testplatform" - : "microsoft.testplatform.cli"; - - // It is okay when node is null, we will fail to find the executable later, and throw. - // This way it throws in the body of the test which has better error reporting than throwing in the data source. - // And we can easily find out what is going on because --WRONG-VERSION-- sticks out, and is easy to find in the codebase. - XmlNode? node = depsXml.DocumentElement?.SelectSingleNode($"PropertyGroup/{propertyName}"); - var version = node?.InnerText.Replace("[", "").Replace("]", "") ?? "--WRONG-VERSION--"; - var vstestConsolePath = runnerInfo.IsNetFrameworkRunner - ? Path.Combine(IntegrationTestEnvironment.TestPlatformRootDirectory, "packages", packageName, version, - "tools", "net451", "Common7", "IDE", "Extensions", "TestPlatform", "vstest.console.exe") - : Path.Combine(IntegrationTestEnvironment.TestPlatformRootDirectory, "packages", packageName, version, - "contentFiles", "any", "netcoreapp2.1", "vstest.console.dll"); - - if (version.StartsWith("15.")) - { - vstestConsolePath = vstestConsolePath.Replace("netcoreapp2.1", "netcoreapp2.0"); - } - - return new VSTestConsoleInfo(vstestConsoleVersion, version, vstestConsolePath); - } - - private static XmlDocument GetDependenciesXml() - { - if (s_depsXml != null) - return s_depsXml; - - var depsXmlPath = Path.Combine(IntegrationTestEnvironment.TestPlatformRootDirectory, "scripts", "build", "TestPlatform.Dependencies.props"); - var fileStream = File.OpenRead(depsXmlPath); - var xmlTextReader = new XmlTextReader(fileStream) { Namespaces = false }; - var depsXml = new XmlDocument(); - depsXml.Load(xmlTextReader); - - s_depsXml = depsXml; - return depsXml; + _builder.WithEveryVersionOfRunner = true; + _builder.WithEveryVersionOfHost = false; + _builder.WithEveryVersionOfAdapter = false; + _builder.WithOlderConfigurations = false; + _builder.WithInProcess = InProcess; + + _builder.BeforeFeature = BeforeFeature; + _builder.AfterFeature = AfterFeature; + _builder.BeforeAdapterFeature = BeforeAdapterFeature; + _builder.AfterAdapterFeature = AfterAdapterFeature; + + _builder.DebugDataCollector = DebugDataCollector; + _builder.DebugVSTestConsole = DebugVSTestConsole; + _builder.DebugTesthost = DebugTesthost; + _builder.NoDefaultBreakpoints = NoDefaultBreakpoints; + + var data = _builder.CreateData(); + data.ForEach(AddData); } } -public class Feature -{ - public Feature (string version, string issue) - { - Version = version; - Issue = issue; - } - - public string Version { get; } - public string Issue { get; } -} - - -public static class Features -{ - public const string ATTACH_DEBUGGER = nameof(ATTACH_DEBUGGER); - - public static Dictionary Table { get; } = new Dictionary - { - [ATTACH_DEBUGGER] = new(version: "v16.7.0-preview-20200519-01", issue: "https://github.com/microsoft/vstest/pull/2325") - }; -} diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/CustomTestHostTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/CustomTestHostTests.cs index e6c341ca4f..996adfec65 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/CustomTestHostTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/CustomTestHostTests.cs @@ -8,7 +8,6 @@ using FluentAssertions; -using Microsoft.TestPlatform.TestUtilities; using Microsoft.TestPlatform.VsTestConsole.TranslationLayer.Interfaces; using Microsoft.VisualStudio.TestPlatform.ObjectModel; using Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.Interfaces; @@ -34,7 +33,7 @@ public void Cleanup() } [TestMethod] - [TranslationLayerCompatibilityDataSource(BeforeFeature = Features.ATTACH_DEBUGGER)] + [RunnerCompatibilityDataSource(BeforeFeature = Features.ATTACH_DEBUGGER)] public void RunTestsWithCustomTestHostLauncherLaunchesTheProcessUsingTheProvidedLauncher(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); @@ -55,7 +54,7 @@ public void RunTestsWithCustomTestHostLauncherLaunchesTheProcessUsingTheProvided } [TestMethod] - [TranslationLayerCompatibilityDataSource(BeforeFeature = Features.ATTACH_DEBUGGER)] + [RunnerCompatibilityDataSource(BeforeFeature = Features.ATTACH_DEBUGGER)] public void RunTestsWithCustomTestHostLauncherLaunchesTheProcessUsingTheProvidedLauncherWhenITestHostLauncher2IsProvided(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); @@ -77,7 +76,7 @@ public void RunTestsWithCustomTestHostLauncherLaunchesTheProcessUsingTheProvided } [TestMethod] - [TranslationLayerCompatibilityDataSource(AfterFeature = Features.ATTACH_DEBUGGER)] + [RunnerCompatibilityDataSource(AfterFeature = Features.ATTACH_DEBUGGER)] public void RunTestsWithCustomTestHostLauncherAttachesToDebuggerUsingTheProvidedLauncher(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); @@ -99,10 +98,10 @@ public void RunTestsWithCustomTestHostLauncherAttachesToDebuggerUsingTheProvided } [TestMethod] - [Ignore("This is not working. The compatibility code only checks the protocol version (in handler), which is dictated by testhost. " - + "It sees 6 but does not realize that the provided CustomTesthostLauncher is not supporting the new feature, it ends up calling back to EditoAttachDebugger" + - "in translation layer, and that just silently skips the call.")] - [TranslationLayerCompatibilityDataSource("net451", "net451", "Latest", AfterFeature = Features.ATTACH_DEBUGGER, DebugVSTestConsole = true, DebugTesthost=true)] + //[Ignore("This is not working. The compatibility code only checks the protocol version (in handler), which is dictated by testhost. " + // + "It sees 6 but does not realize that the provided CustomTesthostLauncher is not supporting the new feature, it ends up calling back to EditoAttachDebugger" + + // "in translation layer, and that just silently skips the call.")] + [RunnerCompatibilityDataSource("net451", "net451", "Latest", AfterFeature = Features.ATTACH_DEBUGGER, DebugVSTestConsole = true, DebugTesthost=true)] public void RunTestsWithCustomTestHostLauncherUsesLaunchWhenGivenAnOutdatedITestHostLauncher(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/DiscoverTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/DiscoverTests.cs index c0a278cf14..38ec639a3f 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/DiscoverTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/DiscoverTests.cs @@ -40,7 +40,7 @@ public void Cleanup() } [TestMethod] - [TranslationLayerCompatibilityDataSource] + [RunnerCompatibilityDataSource] public void DiscoverTestsUsingDiscoveryEventHandler1(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); @@ -57,7 +57,7 @@ public void DiscoverTestsUsingDiscoveryEventHandler1(RunnerInfo runnerInfo) } [TestMethod] - [TranslationLayerCompatibilityDataSource()] + [RunnerCompatibilityDataSource()] public void DiscoverTestsUsingDiscoveryEventHandler2AndTelemetryOptedOut(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/RunTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/RunTests.cs index ed121eaa0e..ffe2449e15 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/RunTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/RunTests.cs @@ -40,7 +40,7 @@ public void Cleanup() } [TestMethod] - [TranslationLayerCompatibilityDataSource] + [RunnerCompatibilityDataSource] public void RunAllTests(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/RunTestsWithFilterTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/RunTestsWithFilterTests.cs index 9661f18242..ea4c38ae0f 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/RunTestsWithFilterTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/RunTestsWithFilterTests.cs @@ -4,7 +4,6 @@ using System.Collections.Generic; using System.Linq; -using Microsoft.TestPlatform.TestUtilities; using Microsoft.TestPlatform.VsTestConsole.TranslationLayer.Interfaces; using Microsoft.VisualStudio.TestPlatform.ObjectModel; using Microsoft.VisualStudio.TestPlatform.ObjectModel.Client; @@ -36,7 +35,7 @@ public void Cleanup() } [TestMethod] - [TranslationLayerCompatibilityDataSource] + [RunnerCompatibilityDataSource] public void RunTestsWithTestCaseFilter(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); diff --git a/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestBase.cs b/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestBase.cs index 2c80715014..dbafe748fb 100644 --- a/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestBase.cs +++ b/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestBase.cs @@ -11,6 +11,8 @@ using System.Text.RegularExpressions; using System.Xml; +using FluentAssertions; + using Microsoft.TestPlatform.VsTestConsole.TranslationLayer; using Microsoft.TestPlatform.VsTestConsole.TranslationLayer.Interfaces; using Microsoft.VisualStudio.TestPlatform.CoreUtilities.Extensions; @@ -960,5 +962,15 @@ protected static string GetDownloadedDotnetMuxerFromTools(string architecture) return path; } - protected string GetDotnetRunnerPath() => _testEnvironment?.VSTestConsoleInfo.Path ?? Path.Combine(_testEnvironment.PublishDirectory, "vstest.console.dll"); + protected string GetDotnetRunnerPath() => _testEnvironment.VSTestConsoleInfo?.Path ?? Path.Combine(_testEnvironment.PublishDirectory, "vstest.console.dll"); + + protected void StdOutHasNoWarnings() + { + StdOut.Should().NotContainEquivalentOf("warning"); + } + + protected void StdErrHasTestRunFailedMessageButNoOtherError() + { + StdErr?.Trim().Should().Be("Test Run Failed."); + } } diff --git a/test/Microsoft.TestPlatform.TestUtilities/Microsoft.TestPlatform.TestUtilities.csproj b/test/Microsoft.TestPlatform.TestUtilities/Microsoft.TestPlatform.TestUtilities.csproj index 7ed26529d4..baea232289 100644 --- a/test/Microsoft.TestPlatform.TestUtilities/Microsoft.TestPlatform.TestUtilities.csproj +++ b/test/Microsoft.TestPlatform.TestUtilities/Microsoft.TestPlatform.TestUtilities.csproj @@ -19,6 +19,7 @@ + From 0f087c0968ccbd79117ee5c032040bd63b4ae64d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Jare=C5=A1?= Date: Thu, 17 Mar 2022 16:08:33 +0100 Subject: [PATCH 056/112] Tests tests. --- TestPlatform.sln | 22 ++-- scripts/build/TestPlatform.Dependencies.props | 2 +- .../DesignMode/DesignModeClient.cs | 2 +- .../AcceptanceTestBase.cs | 40 ++++-- .../AppDomainTests.cs | 2 +- .../ArgumentProcessorTests.cs | 2 +- .../BlameDataCollectorTests.cs | 14 +-- .../CodeCoverageTests.cs | 2 +- .../DataCollectionTests.cs | 4 +- .../DeprecateExtensionsPathWarningTests.cs | 2 +- .../DifferentTestFrameworkSimpleTests.cs | 2 +- .../DiscoveryTests.cs | 6 +- .../DotnetArchitectureSwitchTests.cs | 8 +- .../ExecutionTests.cs | 7 +- .../Extension/CompatibilityRowsBuilder.cs | 117 ++++++++++++------ .../Extension/Features.cs | 4 +- .../MSTestCompatibilityDataSource.cs | 19 ++- .../Extension/NetCoreRunner.cs | 4 +- .../NetCoreTargetFrameworkDataSource.cs | 4 +- .../Extension/NetFrameworkRunner.cs | 4 +- .../NetFullTargetFrameworkDataSource.cs | 8 +- ...ce.cs => RunnerCompatibilityDataSource.cs} | 16 ++- .../Extension/RunnnerInfo.cs | 24 ++-- .../TestPlatformCompatibilityDataSource.cs | 19 ++- .../TesthostCompatibilityDataSource.cs | 16 +-- .../FilePatternParserTests.cs | 6 +- .../FrameworkTests.cs | 8 +- .../LoggerTests.cs | 24 ++-- .../PlatformTests.cs | 2 +- .../PortableNugetPackageTests.cs | 2 +- .../ProcessesInteractionTests.cs | 2 +- .../ResultsDirectoryTests.cs | 4 +- .../RunsettingsTests.cs | 16 +-- .../TelemetryTests.cs | 4 +- .../TestCaseFilterTests.cs | 14 +-- .../CodeCoverageTests.cs | 2 +- .../CustomTestHostTests.cs | 105 +++++++++------- .../DataCollectorAttachmentProcessor.cs | 2 +- .../DifferentTestFrameworkSimpleTests.cs | 8 +- .../TranslationLayerTests/DiscoverTests.cs | 22 ++-- .../LiveUnitTestingTests.cs | 4 +- .../TranslationLayerTests/RunSelectedTests.cs | 12 +- .../TranslationLayerTests/RunTests.cs | 20 +-- ...RunTestsWithDifferentConfigurationTests.cs | 12 +- .../RunTestsWithFilterTests.cs | 8 +- .../DiaSessionTests.cs | 10 +- .../PerformanceTests.cs | 10 +- .../DiscoveryTests.cs | 2 +- .../ExecutionTests.cs | 4 +- .../DebugInfo.cs | 7 +- .../DllInfo.cs | 27 ++-- .../IntegrationTestBase.cs | 39 ++---- .../IntegrationTestEnvironment.cs | 12 +- .../MSTestInfo.cs | 15 --- .../{TesthostInfo.cs => NetTestSdkInfo.cs} | 5 +- .../VSTestConsoleInfo.cs | 15 +-- 56 files changed, 417 insertions(+), 356 deletions(-) rename test/Microsoft.TestPlatform.AcceptanceTests/Extension/{TranslationLayerCompatibilityDataSource.cs => RunnerCompatibilityDataSource.cs} (86%) delete mode 100644 test/Microsoft.TestPlatform.TestUtilities/MSTestInfo.cs rename test/Microsoft.TestPlatform.TestUtilities/{TesthostInfo.cs => NetTestSdkInfo.cs} (64%) diff --git a/TestPlatform.sln b/TestPlatform.sln index 05b0609b3c..f4b952a31f 100644 --- a/TestPlatform.sln +++ b/TestPlatform.sln @@ -181,21 +181,13 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "AttachmentProcessorDataColl EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "vstest.ProgrammerTests", "test\vstest.ProgrammerTests\vstest.ProgrammerTests.csproj", "{B1F84FD8-6150-4ECA-9AD7-C316E04E17D8}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Intent", "test\Intent\Intent.csproj", "{BFBB35C9-6437-480A-8DCC-AE3700110E7D}" +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Intent", "test\Intent\Intent.csproj", "{BFBB35C9-6437-480A-8DCC-AE3700110E7D}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Intent.Primitives", "test\Intent.Primitives\Intent.Primitives.csproj", "{29270853-90DC-4C39-9621-F47AE40A79B6}" +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Intent.Primitives", "test\Intent.Primitives\Intent.Primitives.csproj", "{29270853-90DC-4C39-9621-F47AE40A79B6}" EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "testhost.arm64", "src\testhost.arm64\testhost.arm64.csproj", "{186069FE-E1E8-4DE1-BEA4-0FF1484D22D1}" EndProject Global - GlobalSection(SharedMSBuildProjectFiles) = preSolution - src\Microsoft.TestPlatform.Execution.Shared\Microsoft.TestPlatform.Execution.Shared.projitems*{10b6ade1-f808-4612-801d-4452f5b52242}*SharedItemsImports = 5 - src\Microsoft.TestPlatform.Execution.Shared\Microsoft.TestPlatform.Execution.Shared.projitems*{186069fe-e1e8-4de1-bea4-0ff1484d22d1}*SharedItemsImports = 5 - src\Microsoft.TestPlatform.Execution.Shared\Microsoft.TestPlatform.Execution.Shared.projitems*{27dfbd04-64b2-4f1b-82b2-006620cca6f8}*SharedItemsImports = 5 - src\Microsoft.TestPlatform.Execution.Shared\Microsoft.TestPlatform.Execution.Shared.projitems*{2c7ce1f8-e73e-4987-8023-b5a0ebac86e8}*SharedItemsImports = 5 - src\Microsoft.TestPlatform.Execution.Shared\Microsoft.TestPlatform.Execution.Shared.projitems*{71cb42ff-e750-4a3b-9c3a-ac938853cc89}*SharedItemsImports = 5 - src\Microsoft.TestPlatform.Execution.Shared\Microsoft.TestPlatform.Execution.Shared.projitems*{7f26eda3-c8c4-4b7f-a9b6-d278c2f40a13}*SharedItemsImports = 13 - EndGlobalSection GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU Debug|x64 = Debug|x64 @@ -955,7 +947,7 @@ Global {61F7F446-9EF3-4768-B33A-4D75F60E1059} = {7D4082EA-7AC9-4DFB-98E8-C5E08BDC0EC3} {D5296435-3A3F-4B1A-81D1-434EC9E97DEF} = {5E7F18A8-F843-4C8A-AB02-4C7D9205C6CF} {790B8030-00C2-4121-B125-EDC4CE329BA3} = {ED0C35EB-7F31-4841-A24F-8EB708FFA959} - {27DFBD04-64B2-4F1B-82B2-006620CCA6F8} = {ED0C35EB-7F31-4841-A24F-8EB708FFA959} + {27DFBD04-64B2-4F1B-82B2-006620CCA6F8} = {D9A30E32-D466-4EC5-B4F2-62E17562279B} {71CB42FF-E750-4A3B-9C3A-AC938853CC89} = {ED0C35EB-7F31-4841-A24F-8EB708FFA959} {10B6ADE1-F808-4612-801D-4452F5B52242} = {ED0C35EB-7F31-4841-A24F-8EB708FFA959} {46250E12-4CF1-4051-B4A7-80C8C06E0068} = {B27FAFDF-2DBA-4AB0-BA85-FD5F21D359D6} @@ -1020,4 +1012,12 @@ Global GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {0541B30C-FF51-4E28-B172-83F5F3934BCD} EndGlobalSection + GlobalSection(SharedMSBuildProjectFiles) = preSolution + src\Microsoft.TestPlatform.Execution.Shared\Microsoft.TestPlatform.Execution.Shared.projitems*{10b6ade1-f808-4612-801d-4452f5b52242}*SharedItemsImports = 5 + src\Microsoft.TestPlatform.Execution.Shared\Microsoft.TestPlatform.Execution.Shared.projitems*{186069fe-e1e8-4de1-bea4-0ff1484d22d1}*SharedItemsImports = 5 + src\Microsoft.TestPlatform.Execution.Shared\Microsoft.TestPlatform.Execution.Shared.projitems*{27dfbd04-64b2-4f1b-82b2-006620cca6f8}*SharedItemsImports = 5 + src\Microsoft.TestPlatform.Execution.Shared\Microsoft.TestPlatform.Execution.Shared.projitems*{2c7ce1f8-e73e-4987-8023-b5a0ebac86e8}*SharedItemsImports = 5 + src\Microsoft.TestPlatform.Execution.Shared\Microsoft.TestPlatform.Execution.Shared.projitems*{71cb42ff-e750-4a3b-9c3a-ac938853cc89}*SharedItemsImports = 5 + src\Microsoft.TestPlatform.Execution.Shared\Microsoft.TestPlatform.Execution.Shared.projitems*{7f26eda3-c8c4-4b7f-a9b6-d278c2f40a13}*SharedItemsImports = 13 + EndGlobalSection EndGlobal diff --git a/scripts/build/TestPlatform.Dependencies.props b/scripts/build/TestPlatform.Dependencies.props index 03fd5958bd..d57a941d3d 100644 --- a/scripts/build/TestPlatform.Dependencies.props +++ b/scripts/build/TestPlatform.Dependencies.props @@ -49,7 +49,7 @@ [17.2.0-preview-20220131-20] [17.1.0] [17.0.0] - [16.9.4] + [16.6.1] [16.11.0] [15.9.2] diff --git a/src/Microsoft.TestPlatform.Client/DesignMode/DesignModeClient.cs b/src/Microsoft.TestPlatform.Client/DesignMode/DesignModeClient.cs index 9bbbca6b84..fc251b1c00 100644 --- a/src/Microsoft.TestPlatform.Client/DesignMode/DesignModeClient.cs +++ b/src/Microsoft.TestPlatform.Client/DesignMode/DesignModeClient.cs @@ -357,7 +357,7 @@ public bool AttachDebuggerToProcess(int pid, CancellationToken cancellationToken var ackPayload = _dataSerializer.DeserializePayload(ackMessage); if (!ackPayload.Attached) { - EqtTrace.Warning(ackPayload.ErrorMessage); + EqtTrace.Warning($"DesignModeClient.AttachDebuggerToProcess: Attaching to process failed: {ackPayload.ErrorMessage}"); } return ackPayload.Attached; diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/AcceptanceTestBase.cs b/test/Microsoft.TestPlatform.AcceptanceTests/AcceptanceTestBase.cs index 574e742f2e..bd7fe5afcd 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/AcceptanceTestBase.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/AcceptanceTestBase.cs @@ -2,6 +2,7 @@ // Licensed under the MIT license. See LICENSE file in the project root for full license information. using System; +using System.Linq; using Microsoft.TestPlatform.TestUtilities; @@ -73,7 +74,8 @@ public static string And(string left, string right) protected static void SetTestEnvironment(IntegrationTestEnvironment testEnvironment, RunnerInfo runnerInfo) { testEnvironment.VSTestConsoleInfo = runnerInfo.VSTestConsoleInfo; - testEnvironment.DllInfos = runnerInfo.DllInfos; + // The order here matters, it changes how the resulting path is built when we resolve test dlls and other assets. + testEnvironment.DllInfos = new[] { runnerInfo.AdapterInfo, runnerInfo.TestHostInfo }.Where(d => d != null).ToList(); testEnvironment.DebugInfo = runnerInfo.DebugInfo; testEnvironment.RunnerFramework = runnerInfo.RunnerFramework; @@ -111,17 +113,37 @@ protected string GetTargetFramworkForRunsettings() } /// - /// Default RunSettings + /// Empty runsettings, just with the RunSettings tag that we require. /// /// - public string GetDefaultRunSettings() + public string GetEmptyRunsettings() { - string runSettingsXml = $@" - - - {FrameworkArgValue} - - "; + return ""; + } + + /// + /// Almost empty runsettings, just specifying the target framework from the currently set test environment. + /// + public string GetRunSettingsWithCurrentTargetFramework() + { + return GetRunSettingsWithTargetFramework(FrameworkArgValue); + } + + /// + /// Almost empty runsettings, just specifying the given target framework. + /// Use the overload without any parameters to get the target framework from the currently set test environment. + /// + /// + public string GetRunSettingsWithTargetFramework(string targetFramework) + { + string runSettingsXml = + $@" + + + {targetFramework} + + "; + return runSettingsXml; } } diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/AppDomainTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/AppDomainTests.cs index 17e8f88423..a413e8d7d6 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/AppDomainTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/AppDomainTests.cs @@ -38,7 +38,7 @@ public void RunTestExecutionWithDisableAppDomain(RunnerInfo runnerInfo) var runsettingsFilePath = GetInProcDataCollectionRunsettingsFile(true, TempDirectory); var arguments = PrepareArguments( - GetSampleTestAssembly(), + GetSampleTestDll(), GetTestAdapterPath(), runsettingsFilePath, FrameworkArgValue, diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/ArgumentProcessorTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/ArgumentProcessorTests.cs index 44a4a39dbd..8b9dd95d40 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/ArgumentProcessorTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/ArgumentProcessorTests.cs @@ -40,7 +40,7 @@ public void PassingInvalidArgumentsToVsTestConsoleShouldNotPrintHelpMessage(Runn { SetTestEnvironment(_testEnvironment, runnerInfo); - var arguments = PrepareArguments(GetSampleTestAssembly(), GetTestAdapterPath(), string.Empty, FrameworkArgValue, resultsDirectory: TempDirectory.Path); + var arguments = PrepareArguments(GetSampleTestDll(), GetTestAdapterPath(), string.Empty, FrameworkArgValue, resultsDirectory: TempDirectory.Path); arguments = string.Concat(arguments, " /badArgument"); InvokeVsTest(arguments); diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/BlameDataCollectorTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/BlameDataCollectorTests.cs index e9e6af25ee..8df3e8bc6a 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/BlameDataCollectorTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/BlameDataCollectorTests.cs @@ -46,7 +46,7 @@ public BlameDataCollectorTests() public void BlameDataCollectorShouldGiveCorrectTestCaseName(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - var assemblyPaths = GetAssetFullPath("BlameUnitTestProject.dll"); + var assemblyPaths = GetTestDll("BlameUnitTestProject.dll"); var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue); arguments = string.Concat(arguments, $" /Blame"); arguments = string.Concat(arguments, $" /ResultsDirectory:{TempDirectory.Path}"); @@ -137,7 +137,7 @@ public void BlameDataCollectorShouldOutputDumpFileWhenNoCrashOccursButCollectAlw public void HangDumpOnTimeout(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - var assemblyPaths = GetAssetFullPath("timeout.dll"); + var assemblyPaths = GetTestDll("timeout.dll"); var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, string.Empty, runnerInfo.InIsolationValue); arguments = string.Concat(arguments, $@" /Blame:""CollectHangDump;HangDumpType=full;TestTimeout=3s"""); @@ -160,7 +160,7 @@ public void HangDumpOnTimeout(RunnerInfo runnerInfo) public void CrashDumpWhenThereIsNoTimeout(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - var assemblyPaths = GetAssetFullPath("timeout.dll"); + var assemblyPaths = GetTestDll("timeout.dll"); var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, string.Empty, runnerInfo.InIsolationValue); arguments = string.Concat(arguments, $@" /Blame:""CollectDump;DumpType=full;CollectAlways=true;CollectHangDump"""); @@ -183,7 +183,7 @@ public void CrashDumpWhenThereIsNoTimeout(RunnerInfo runnerInfo) public void CrashDumpOnExit(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - var assemblyPaths = GetAssetFullPath("timeout.dll"); + var assemblyPaths = GetTestDll("timeout.dll"); var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, string.Empty, runnerInfo.InIsolationValue); arguments = string.Concat(arguments, $@" /Blame:""CollectDump;DumpType=full;CollectAlways=true"""); @@ -204,7 +204,7 @@ public void CrashDumpOnExit(RunnerInfo runnerInfo) public void CrashDumpOnStackOverflow(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - var assemblyPaths = GetAssetFullPath("crash.dll"); + var assemblyPaths = GetTestDll("crash.dll"); var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, string.Empty, runnerInfo.InIsolationValue); arguments = string.Concat(arguments, $@" /Blame:""CollectDump;DumpType=full"""); @@ -225,7 +225,7 @@ public void CrashDumpOnStackOverflow(RunnerInfo runnerInfo) public void CrashDumpChildProcesses(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - var assemblyPaths = GetAssetFullPath("child-crash.dll"); + var assemblyPaths = GetTestDll("child-crash.dll"); var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, string.Empty, runnerInfo.InIsolationValue); arguments = string.Concat(arguments, $@" /Blame:""CollectDump;DumpType=full"""); InvokeVsTest(arguments); @@ -240,7 +240,7 @@ public void CrashDumpChildProcesses(RunnerInfo runnerInfo) public void HangDumpChildProcesses(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - var assemblyPaths = GetAssetFullPath("child-hang.dll"); + var assemblyPaths = GetTestDll("child-hang.dll"); var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, string.Empty, runnerInfo.InIsolationValue); arguments = string.Concat(arguments, $@" /Blame:""CollectHangDump;HangDumpType=full;TestTimeout=15s"""); InvokeVsTest(arguments); diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/CodeCoverageTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/CodeCoverageTests.cs index b426fc7fd1..c7901f2326 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/CodeCoverageTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/CodeCoverageTests.cs @@ -251,7 +251,7 @@ private string CreateArguments( TestParameters testParameters, out string trxFilePath) { - var assemblyPaths = GetAssetFullPath(testParameters.AssemblyName); + var assemblyPaths = GetTestDll(testParameters.AssemblyName); string traceDataCollectorDir = Path.Combine(IntegrationTestEnvironment.TestPlatformRootDirectory, "artifacts", IntegrationTestEnvironment.BuildConfiguration, "Microsoft.CodeCoverage"); diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/DataCollectionTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/DataCollectionTests.cs index 2486a605ff..1308f4345a 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/DataCollectionTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/DataCollectionTests.cs @@ -87,7 +87,7 @@ public void DataCollectorAssemblyLoadingShouldNotThrowErrorForNetCore(RunnerInfo { SetTestEnvironment(_testEnvironment, runnerInfo); - var arguments = PrepareArguments(GetAssetFullPath("AppDomainGetAssembliesTestProject.dll", "netcoreapp2.1"), string.Empty, string.Empty, FrameworkArgValue, resultsDirectory: TempDirectory.Path); + var arguments = PrepareArguments(GetTestDllForFramework("AppDomainGetAssembliesTestProject.dll", "netcoreapp2.1"), string.Empty, string.Empty, FrameworkArgValue, resultsDirectory: TempDirectory.Path); InvokeVsTest(arguments); ValidateSummaryStatus(1, 0, 0); @@ -100,7 +100,7 @@ public void DataCollectorAssemblyLoadingShouldNotThrowErrorForFullFramework(Runn { SetTestEnvironment(_testEnvironment, runnerInfo); - var arguments = PrepareArguments(GetAssetFullPath("AppDomainGetAssembliesTestProject.dll"), string.Empty, string.Empty, FrameworkArgValue, resultsDirectory: TempDirectory.Path); + var arguments = PrepareArguments(GetTestDll("AppDomainGetAssembliesTestProject.dll"), string.Empty, string.Empty, FrameworkArgValue, resultsDirectory: TempDirectory.Path); InvokeVsTest(arguments); ValidateSummaryStatus(1, 0, 0); diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/DeprecateExtensionsPathWarningTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/DeprecateExtensionsPathWarningTests.cs index 4c10549775..d33857347a 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/DeprecateExtensionsPathWarningTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/DeprecateExtensionsPathWarningTests.cs @@ -61,7 +61,7 @@ public void CopyAdapterToExtensions() [TestMethod] public void VerifyDeprecatedWarningIsThrownWhenAdaptersPickedFromExtensionDirectory() { - var arguments = PrepareArguments(GetSampleTestAssembly(), null, null, FrameworkArgValue, resultsDirectory: TempDirectory.Path); + var arguments = PrepareArguments(GetSampleTestDll(), null, null, FrameworkArgValue, resultsDirectory: TempDirectory.Path); InvokeVsTest(arguments); StdOutputContains("Adapter lookup is being changed, please follow"); diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/DifferentTestFrameworkSimpleTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/DifferentTestFrameworkSimpleTests.cs index faaeb1fd7e..8f112a83e5 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/DifferentTestFrameworkSimpleTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/DifferentTestFrameworkSimpleTests.cs @@ -153,7 +153,7 @@ public void NUnitRunAllTestExecution(RunnerInfo runnerInfo) SetTestEnvironment(_testEnvironment, runnerInfo); var arguments = PrepareArguments( - GetAssetFullPath("NUTestProject.dll"), + GetTestDll("NUTestProject.dll"), GetTestAdapterPath(UnitTestFramework.NUnit), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, TempDirectory.Path); diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/DiscoveryTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/DiscoveryTests.cs index fa9bb8627b..444cc873ec 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/DiscoveryTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/DiscoveryTests.cs @@ -25,7 +25,7 @@ public void DiscoverAllTests(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - InvokeVsTestForDiscovery(GetSampleTestAssembly(), GetTestAdapterPath(), string.Empty, FrameworkArgValue); + InvokeVsTestForDiscovery(GetSampleTestDll(), GetTestAdapterPath(), string.Empty, FrameworkArgValue); var listOfTests = new[] { "SampleUnitTestProject.UnitTest1.PassingTest", "SampleUnitTestProject.UnitTest1.FailingTest", "SampleUnitTestProject.UnitTest1.SkippingTest" }; ValidateDiscoveredTests(listOfTests); @@ -65,7 +65,7 @@ public void DiscoverFullyQualifiedTests(RunnerInfo runnerInfo) var listOfTests = new[] { "SampleUnitTestProject.UnitTest1.PassingTest", "SampleUnitTestProject.UnitTest1.FailingTest", "SampleUnitTestProject.UnitTest1.SkippingTest" }; - var arguments = PrepareArguments(GetSampleTestAssembly(), GetTestAdapterPath(), string.Empty, FrameworkArgValue, _testEnvironment.InIsolationValue, resultsDirectory: TempDirectory.Path); + var arguments = PrepareArguments(GetSampleTestDll(), GetTestAdapterPath(), string.Empty, FrameworkArgValue, _testEnvironment.InIsolationValue, resultsDirectory: TempDirectory.Path); arguments = string.Concat(arguments, " /ListFullyQualifiedTests", " /ListTestsTargetPath:\"" + dummyFilePath + "\""); InvokeVsTest(arguments); @@ -80,7 +80,7 @@ public void DiscoverTestsShouldShowProperWarningIfNoTestsOnTestCaseFilter(Runner { SetTestEnvironment(_testEnvironment, runnerInfo); - var assetFullPath = GetAssetFullPath("SimpleTestProject2.dll"); + var assetFullPath = GetTestDll("SimpleTestProject2.dll"); var arguments = PrepareArguments(assetFullPath, GetTestAdapterPath(), string.Empty, FrameworkArgValue, _testEnvironment.InIsolationValue, resultsDirectory: TempDirectory.Path); arguments = string.Concat(arguments, " /listtests"); arguments = string.Concat(arguments, " /testcasefilter:NonExistTestCaseName"); diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/DotnetArchitectureSwitchTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/DotnetArchitectureSwitchTests.cs index e60f46ed94..5ff96f6cd2 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/DotnetArchitectureSwitchTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/DotnetArchitectureSwitchTests.cs @@ -78,7 +78,7 @@ public void GlobalInstallation() AssertSwitch(stdOut); // Verify switch using test container - var buildAssemblyPath = GetAssetFullPath("ArchitectureSwitch.dll", "net6.0"); + var buildAssemblyPath = GetTestDllForFramework("ArchitectureSwitch.dll", "net6.0"); ExecuteApplication(GetDefaultDotnetMuxerLocation, $"test {buildAssemblyPath} --arch x64", out stdOut, out _, out _, env, projectDirectory); AssertSwitch(stdOut); @@ -148,7 +148,7 @@ public void DOTNET_ROOTS_EnvironmentVariables(bool dotnetRoot, bool dotnetRootX6 AssertSwitch(stdOut); // Verify switch using test container - var buildAssemblyPath = GetAssetFullPath("ArchitectureSwitch.dll", "net6.0"); + var buildAssemblyPath = GetTestDllForFramework("ArchitectureSwitch.dll", "net6.0"); ExecuteApplication($"{s_privateX64Installation}/{GetMuxerName}", $"test {buildAssemblyPath} --framework net6.0 --arch x64", out stdOut, out _, out _, env, projectDirectory); AssertSwitch(stdOut); @@ -192,7 +192,7 @@ public void PrivateX64BuildToGlobalArmInstallation() AssertSwitch(stdOut); // Verify switch using test container - var buildAssemblyPath = GetAssetFullPath("ArchitectureSwitch.dll", "net6.0"); + var buildAssemblyPath = GetTestDllForFramework("ArchitectureSwitch.dll", "net6.0"); ExecuteApplication($"{s_privateX64Installation}/{GetMuxerName}", $"test {buildAssemblyPath} --framework net6.0 --arch arm64", out stdOut, out _, out _, env, projectDirectory); AssertSwitch(stdOut); @@ -249,7 +249,7 @@ public void PrivateX64BuildToDOTNET_ROOTS_EnvironmentVariables(bool dotnetRoot, AssertSwitch(stdOut); // Verify switch using test container - var buildAssemblyPath = GetAssetFullPath("ArchitectureSwitch.dll", "net6.0"); + var buildAssemblyPath = GetTestDllForFramework("ArchitectureSwitch.dll", "net6.0"); ExecuteApplication($"{s_privateX64Installation}/{GetMuxerName}", $"test {buildAssemblyPath} --framework net6.0 --arch arm64", out stdOut, out _, out _, env, projectDirectory); AssertSwitch(stdOut); diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/ExecutionTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/ExecutionTests.cs index b39e6b76a1..60b60036ed 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/ExecutionTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/ExecutionTests.cs @@ -34,7 +34,8 @@ public void RunMultipleTestAssemblies(RunnerInfo runnerInfo) } [TestMethod] - [TestPlatformCompatibilityDataSource(BeforeFeature = Features.ATTACH_DEBUGGER, AfterAdapterFeature = Features.MSTEST_IFRAMEWORK_HANDLE_99)] + [TestPlatformCompatibilityDataSource()] + //[TestPlatformCompatibilityDataSource(BeforeFeature = Features.ATTACH_DEBUGGER, AfterAdapterFeature = Features.MSTEST_IFRAMEWORK_HANDLE_99)] public void RunTestsFromMultipleMSTestAssemblies(RunnerInfo runnerInfo) { @@ -51,7 +52,7 @@ public void RunTestsFromMultipleMSTestAssemblies(RunnerInfo runnerInfo) } [TestMethod] - [HostCompatibilityDataSource] + [TestHostCompatibilityDataSource] public void RunMultipleMSTestAssembliesOnVstestConsoleAndTesthostCombinations(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); @@ -107,7 +108,7 @@ public void RunMultipleTestAssembliesWithoutTestAdapterPath(RunnerInfo runnerInf [TestCategory("Windows")] [TestMethod] [MSTestCompatibilityDataSource] - public void RunMultipleTestAssembliesInParallel(RunnerInfo runnerInfo, MSTestInfo msTestInfo) + public void RunMultipleTestAssembliesInParallel(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); using var tempDir = new TempDirectory(); diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/CompatibilityRowsBuilder.cs b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/CompatibilityRowsBuilder.cs index f4fa979dbd..dccbd03525 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/CompatibilityRowsBuilder.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/CompatibilityRowsBuilder.cs @@ -47,13 +47,15 @@ public CompatibilityRowsBuilder(string runnerFrameworks = AcceptanceTestBase.DEF public bool WithEveryVersionOfAdapter { get; set; } = true; public bool WithOlderConfigurations { get; set; } = true; - public string? BeforeFeature { get; set; } - public string? AfterFeature { get; set; } + public string? BeforeRunnerFeature { get; set; } + public string? AfterRunnerFeature { get; set; } + public string? BeforeTestHostFeature { get; set; } + public string? AfterTestHostFeature { get; set; } public string? BeforeAdapterFeature { get; set; } public string? AfterAdapterFeature { get; set; } public bool DebugVSTestConsole { get; set; } - public bool DebugTesthost { get; set; } + public bool DebugTestHost { get; set; } public bool DebugDataCollector { get; set; } public bool NoDefaultBreakpoints { get; set; } = true; @@ -79,26 +81,40 @@ public List CreateData() var minVersion = SemVersion.Parse("0.0.0-alpha.1"); var maxVersion = SemVersion.Parse("9999.0.0"); - SemVersion? beforeVersion = maxVersion; - SemVersion? afterVersion = minVersion; + SemVersion? beforeRunnerVersion = maxVersion; + SemVersion? afterRunnerVersion = minVersion; + SemVersion? beforeTestHostVersion = maxVersion; + SemVersion? afterTestHostVersion = minVersion; SemVersion? beforeAdapterVersion = maxVersion; SemVersion? afterAdapterVersion = minVersion; - if (BeforeFeature != null) + if (BeforeRunnerFeature != null) { - var feature = Features.TestPlatformFeatures[BeforeFeature]; - beforeVersion = SemVersion.Parse(feature.Version.TrimStart('v')); + var feature = Features.TestPlatformFeatures[BeforeRunnerFeature]; + beforeRunnerVersion = SemVersion.Parse(feature.Version.TrimStart('v')); } - if (AfterFeature != null) + if (AfterRunnerFeature != null) { - var feature = Features.TestPlatformFeatures[AfterFeature]; - afterVersion = SemVersion.Parse(feature.Version.TrimStart('v')); + var feature = Features.TestPlatformFeatures[AfterRunnerFeature]; + afterRunnerVersion = SemVersion.Parse(feature.Version.TrimStart('v')); } - if (BeforeFeature != null) + if (BeforeTestHostFeature != null) { - var feature = Features.TestPlatformFeatures[BeforeFeature]; + var feature = Features.TestPlatformFeatures[BeforeTestHostFeature]; + beforeTestHostVersion = SemVersion.Parse(feature.Version.TrimStart('v')); + } + + if (AfterTestHostFeature != null) + { + var feature = Features.TestPlatformFeatures[AfterTestHostFeature]; + afterTestHostVersion = SemVersion.Parse(feature.Version.TrimStart('v')); + } + + if (BeforeAdapterFeature != null) + { + var feature = Features.TestPlatformFeatures[BeforeAdapterFeature]; beforeAdapterVersion = SemVersion.Parse(feature.Version.TrimStart('v')); } @@ -120,16 +136,21 @@ public List CreateData() Func isInRange = (version, before, after) => version < before && after < version; var rows = dataRows.Where(r => r.VSTestConsoleInfo != null - && isInRange(r.VSTestConsoleInfo.Version, beforeVersion, afterVersion) - && r.DllInfos.All(d => d is NetTestSdkInfo ? isInRange(d.Version, beforeVersion, afterVersion) : isInRange(d.Version, beforeAdapterVersion, afterAdapterVersion))).ToList(); + && isInRange(r.VSTestConsoleInfo.Version, beforeRunnerVersion, afterRunnerVersion) + && r.TestHostInfo != null && isInRange(r.TestHostInfo.Version, beforeTestHostVersion, afterTestHostVersion) + && r.AdapterInfo != null && isInRange(r.AdapterInfo.Version, beforeAdapterVersion, afterAdapterVersion)).ToList(); - if (rows.Count == 0) + // We use ToString to determine which values are unique. Not great solution, but works better than using records. + var distinctRows = new Dictionary(); + rows.ForEach(r => distinctRows[r.ToString()] = r); + + if (distinctRows.Count == 0) { - // TODO: This needs to be way more specific about what happened. + // TODO: This needs to be way more specific about what happened. And possibly propagate as inconclusive state if we decide to update versions automatically? throw new InvalidOperationException("There were no rows that matched the specified criteria."); } - return rows; + return distinctRows.Values.ToList(); } private void AddInProcess(List dataRows) @@ -147,7 +168,7 @@ private void AddInProcess(List dataRows) { foreach (var adapterVersion in _adapterVersions) { - AddRow(dataRows, runnerVersion, runnerFramework, runnerVersion, runnerFramework, adapter, adapterVersion, inIsolation: false); + AddRow(dataRows, "In process", runnerVersion, runnerFramework, runnerVersion, runnerFramework, adapter, adapterVersion, inIsolation: false); } } } @@ -158,7 +179,7 @@ private void AddOlderConfigurations(List dataRows) { // Older configurations where the runner, host and adapter version are the same. // We already added the row where all are newest when adding combination with all runners. - foreach (var runnerVersion in _runnerVersions.Skip(1)) + foreach (var runnerVersion in _runnerVersions) { foreach (var runnerFramework in _runnerFrameworks) { @@ -168,8 +189,8 @@ private void AddOlderConfigurations(List dataRows) var hostVersion = runnerVersion; foreach (var adapter in _adapters) { - var adapterVersion = runnerVersion; - AddRow(dataRows, runnerVersion, runnerFramework, hostVersion, hostFramework, adapter, adapterVersion, inIsolation: true); + var adapterVersion = _adapterVersions[0]; + AddRow(dataRows, "Older", runnerVersion, runnerFramework, hostVersion, hostFramework, adapter, adapterVersion, inIsolation: true); } } } @@ -190,10 +211,9 @@ private void AddEveryVersionOfAdapter(List dataRows) var hostVersion = isNetFramework ? runnerVersion : _hostVersions[0]; foreach (var adapter in _adapters) { - // We already used the newest when adding combination with every runner - foreach (var adapterVersion in _adapterVersions.Skip(1)) + foreach (var adapterVersion in _adapterVersions) { - AddRow(dataRows, runnerVersion, runnerFramework, hostVersion, hostFramework, adapter, adapterVersion, inIsolation: true); + AddRow(dataRows, "Every adapter", runnerVersion, runnerFramework, hostVersion, hostFramework, adapter, adapterVersion, inIsolation: true); } } } @@ -211,16 +231,15 @@ private void AddEveryVersionOfHost(List dataRows) var isNetFramework = hostFramework.StartsWith("net4"); // .NET Framework testhost ships with the runner, and the version from the // runner directory is always the same as the runner. There are no variations - // so we just need to add host versions for .NET testhosts. We also skip the - // newest version because we already added it when AddEveryVersionOfRunner - var hostVersions = isNetFramework ? Array.Empty() : _hostVersions.Skip(1).ToArray(); + // so we just need to add host versions for .NET testhosts. + var hostVersions = isNetFramework ? Array.Empty() : _hostVersions.ToArray(); foreach (var hostVersion in hostVersions) { foreach (var adapter in _adapters) { // use the newest var adapterVersion = _adapterVersions[0]; - AddRow(dataRows, runnerVersion, runnerFramework, hostVersion, hostFramework, adapter, adapterVersion, inIsolation: true); + AddRow(dataRows, "Every host", runnerVersion, runnerFramework, hostVersion, hostFramework, adapter, adapterVersion, inIsolation: true); } } } @@ -243,17 +262,17 @@ private void AddEveryVersionOfRunner(List dataRows) { // use the newest var adapterVersion = _adapterVersions[0]; - AddRow(dataRows, runnerVersion, runnerFramework, hostVersion, hostFramework, adapter, adapterVersion, inIsolation: true); + AddRow(dataRows, "Every runner", runnerVersion, runnerFramework, hostVersion, hostFramework, adapter, adapterVersion, inIsolation: true); } } } } } - private void AddRow(List dataRows, + private void AddRow(List dataRows, string batch, string runnerVersion, string runnerFramework, string hostVersion, string hostFramework, string adapter, string adapterVersion, bool inIsolation) { - RunnerInfo runnerInfo = GetRunnerInfo(runnerFramework, hostFramework, inIsolation); + RunnerInfo runnerInfo = GetRunnerInfo(batch, runnerFramework, hostFramework, inIsolation); runnerInfo.DebugInfo = GetDebugInfo(); runnerInfo.VSTestConsoleInfo = GetVSTestConsoleInfo(runnerVersion, runnerInfo); @@ -263,8 +282,8 @@ private void AddRow(List dataRows, // C:\p\vstest\test\TestAssets\MSTestProject1\bin\MSTestLatestPreview-2.2.9-preview-20220210-07\NETTestSdkLatest-17.2.0-dev\Debug\net451\MSTestProject1.dll // versus adding testSdk second: // C:\p\vstest\test\TestAssets\MSTestProject1\bin\NETTestSdkLatest-17.2.0-dev\MSTestLatestPreview-2.2.9-preview-20220210-07\Debug\net451\MSTestProject1.dll - runnerInfo.DllInfos.Add(GetMSTestInfo(adapterVersion)); - runnerInfo.DllInfos.Add(GetNetTestSdkInfo(hostVersion)); + runnerInfo.TestHostInfo = GetNetTestSdkInfo(hostVersion); + runnerInfo.AdapterInfo = GetMSTestInfo(adapterVersion); dataRows.Add(runnerInfo); } @@ -273,24 +292,25 @@ private DebugInfo GetDebugInfo() return new DebugInfo { DebugDataCollector = DebugDataCollector, - DebugTesthost = DebugTesthost, + DebugTestHost = DebugTestHost, DebugVSTestConsole = DebugVSTestConsole, NoDefaultBreakpoints = NoDefaultBreakpoints }; } - private RunnerInfo GetRunnerInfo(string runnerFramework, string hostFramework, bool inIsolation) + private RunnerInfo GetRunnerInfo(string batch, string runnerFramework, string hostFramework, bool inIsolation) { return new RunnerInfo { + Batch = batch, RunnerFramework = runnerFramework, TargetFramework = hostFramework, InIsolationValue = inIsolation ? AcceptanceTestBase.InIsolation : null }; } - private MSTestInfo GetMSTestInfo(string msTestVersion) + private DllInfo GetMSTestInfo(string msTestVersion) { var depsXml = GetDependenciesXml(); @@ -301,7 +321,14 @@ private MSTestInfo GetMSTestInfo(string msTestVersion) var slash = Path.DirectorySeparatorChar; var versionSpecificBinPath = $"{slash}bin{slash}MSTest{msTestVersion}-{version}{slash}"; - return new MSTestInfo(msTestVersion, version, versionSpecificBinPath); + return new DllInfo + { + Name = "MSTest", + PropertyName = "MSTest", + VersionType = msTestVersion, + Version = version, + Path = versionSpecificBinPath, + }; } private static VSTestConsoleInfo GetVSTestConsoleInfo(string vstestConsoleVersion, RunnerInfo runnerInfo) @@ -335,7 +362,12 @@ private static VSTestConsoleInfo GetVSTestConsoleInfo(string vstestConsoleVersio vstestConsolePath = vstestConsolePath.Replace("netcoreapp2.1", "netcoreapp2.0"); } - return new VSTestConsoleInfo(vstestConsoleVersion, version, vstestConsolePath); + return new VSTestConsoleInfo + { + VersionType = vstestConsoleVersion, + Version = version, + Path = vstestConsolePath, + }; } private static NetTestSdkInfo GetNetTestSdkInfo(string testhostVersionType) @@ -358,7 +390,12 @@ private static NetTestSdkInfo GetNetTestSdkInfo(string testhostVersionType) var slash = Path.DirectorySeparatorChar; var versionSpecificBinPath = $"{slash}bin{slash}NETTestSdk{testhostVersionType}-{version}{slash}"; - return new NetTestSdkInfo(testhostVersionType, version, versionSpecificBinPath); + return new NetTestSdkInfo + { + VersionType = testhostVersionType, + Version = version, + Path = versionSpecificBinPath + }; } diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/Features.cs b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/Features.cs index e82389e404..b966b9daa3 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/Features.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/Features.cs @@ -7,13 +7,13 @@ namespace Microsoft.TestPlatform.AcceptanceTests; public static class Features { - public const string ATTACH_DEBUGGER = nameof(ATTACH_DEBUGGER); + public const string ATTACH_DEBUGGER_FLOW = nameof(ATTACH_DEBUGGER_FLOW); public const string MSTEST_IFRAMEWORK_HANDLE_99 = nameof(MSTEST_IFRAMEWORK_HANDLE_99); public static Dictionary TestPlatformFeatures { get; } = new Dictionary { - [ATTACH_DEBUGGER] = new(version: "v16.7.0-preview-20200519-01", issue: "https://github.com/microsoft/vstest/pull/2325"), + [ATTACH_DEBUGGER_FLOW] = new(version: "v16.7.0-preview-20200519-01", issue: "https://github.com/microsoft/vstest/pull/2325"), }; public static Dictionary AdapterFeatures { get; internal set; } = new Dictionary diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/MSTestCompatibilityDataSource.cs b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/MSTestCompatibilityDataSource.cs index 30875f6886..33c53f8a5d 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/MSTestCompatibilityDataSource.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/MSTestCompatibilityDataSource.cs @@ -34,7 +34,7 @@ public MSTestCompatibilityDataSource( } public bool DebugVSTestConsole { get; set; } - public bool DebugTesthost { get; set; } + public bool DebugTestHost { get; set; } public bool DebugDataCollector { get; set; } public bool NoDefaultBreakpoints { get; set; } = true; @@ -43,8 +43,11 @@ public MSTestCompatibilityDataSource( /// public bool InProcess { get; set; } - public string? BeforeFeature { get; set; } - public string? AfterFeature { get; set; } + public string? BeforeRunnerFeature { get; set; } + public string? AfterRunnerFeature { get; set; } + + public string? BeforeTestHostFeature { get; set; } + public string? AfterTestHostFeature { get; set; } public string? BeforeAdapterFeature { get; set; } public string? AfterAdapterFeature { get; set; } @@ -57,14 +60,18 @@ public override void CreateData(MethodInfo methodInfo) _builder.WithOlderConfigurations = false; _builder.WithInProcess = InProcess; - _builder.BeforeFeature = BeforeFeature; - _builder.AfterFeature = AfterFeature; + _builder.BeforeRunnerFeature = BeforeRunnerFeature; + _builder.AfterRunnerFeature = AfterRunnerFeature; + + _builder.BeforeTestHostFeature = BeforeTestHostFeature; + _builder.AfterTestHostFeature = AfterTestHostFeature; + _builder.BeforeAdapterFeature = BeforeAdapterFeature; _builder.AfterAdapterFeature = AfterAdapterFeature; _builder.DebugDataCollector = DebugDataCollector; _builder.DebugVSTestConsole = DebugVSTestConsole; - _builder.DebugTesthost = DebugTesthost; + _builder.DebugTestHost = DebugTestHost; _builder.NoDefaultBreakpoints = NoDefaultBreakpoints; var data = _builder.CreateData(); diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/NetCoreRunner.cs b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/NetCoreRunner.cs index 60d49baf1a..71db23bdee 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/NetCoreRunner.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/NetCoreRunner.cs @@ -35,7 +35,7 @@ public NetCoreRunner(string targetFrameworks = AcceptanceTestBase.NETFX452_NET50 } public bool DebugVSTestConsole { get; set; } - public bool DebugTesthost { get; set; } + public bool DebugTestHost { get; set; } public bool DebugDataCollector { get; set; } public bool NoDefaultBreakpoints { get; set; } = true; @@ -56,7 +56,7 @@ public IEnumerable GetData(MethodInfo methodInfo) runnerInfo.DebugInfo = new DebugInfo { DebugVSTestConsole = DebugVSTestConsole, - DebugTesthost = DebugTesthost, + DebugTestHost = DebugTestHost, DebugDataCollector = DebugDataCollector, NoDefaultBreakpoints = NoDefaultBreakpoints, }; diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/NetCoreTargetFrameworkDataSource.cs b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/NetCoreTargetFrameworkDataSource.cs index e4d646a8b8..56ff01036b 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/NetCoreTargetFrameworkDataSource.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/NetCoreTargetFrameworkDataSource.cs @@ -49,7 +49,7 @@ public NetCoreTargetFrameworkDataSource( } public bool DebugVSTestConsole { get; set; } - public bool DebugTesthost { get; set; } + public bool DebugTestHost { get; set; } public bool DebugDataCollector { get; set; } public bool NoDefaultBreakpoints { get; set; } = true; @@ -64,7 +64,7 @@ private void AddRunnerDataRow(List dataRows, string runnerFramework, s runnerInfo.DebugInfo = new DebugInfo { DebugDataCollector = DebugDataCollector, - DebugTesthost = DebugTesthost, + DebugTestHost = DebugTestHost, DebugVSTestConsole = DebugVSTestConsole, NoDefaultBreakpoints = NoDefaultBreakpoints, }; diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/NetFrameworkRunner.cs b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/NetFrameworkRunner.cs index c7be429345..0f1faa65f5 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/NetFrameworkRunner.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/NetFrameworkRunner.cs @@ -32,7 +32,7 @@ public NetFrameworkRunner(string targetFrameworks = AcceptanceTestBase.NETFX452_ } public bool DebugVSTestConsole { get; set; } - public bool DebugTesthost { get; set; } + public bool DebugTestHost { get; set; } public bool DebugDataCollector { get; set; } public bool NoDefaultBreakpoints { get; set; } = true; @@ -58,7 +58,7 @@ public IEnumerable GetData(MethodInfo methodInfo) runnerInfo.DebugInfo = new DebugInfo { DebugVSTestConsole = DebugVSTestConsole, - DebugTesthost = DebugTesthost, + DebugTestHost = DebugTestHost, DebugDataCollector = DebugDataCollector, NoDefaultBreakpoints = NoDefaultBreakpoints, }; diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/NetFullTargetFrameworkDataSource.cs b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/NetFullTargetFrameworkDataSource.cs index 468aded7f7..b809da7799 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/NetFullTargetFrameworkDataSource.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/NetFullTargetFrameworkDataSource.cs @@ -43,7 +43,7 @@ public NetFullTargetFrameworkDataSource(bool inIsolation = true, bool inProcess } public bool DebugVSTestConsole { get; set; } - public bool DebugTesthost { get; set; } + public bool DebugTestHost { get; set; } public bool DebugDataCollector { get; set; } public bool NoDefaultBreakpoints { get; set; } = true; @@ -62,7 +62,7 @@ public IEnumerable GetData(MethodInfo methodInfo) runnerInfo.DebugInfo = new DebugInfo { DebugVSTestConsole = DebugVSTestConsole, - DebugTesthost = DebugTesthost, + DebugTestHost = DebugTestHost, DebugDataCollector = DebugDataCollector, NoDefaultBreakpoints = NoDefaultBreakpoints, }; @@ -82,7 +82,7 @@ public IEnumerable GetData(MethodInfo methodInfo) runnerInfo.DebugInfo = new DebugInfo { DebugVSTestConsole = DebugVSTestConsole, - DebugTesthost = DebugTesthost, + DebugTestHost = DebugTestHost, DebugDataCollector = DebugDataCollector, NoDefaultBreakpoints = NoDefaultBreakpoints, }; @@ -100,7 +100,7 @@ public IEnumerable GetData(MethodInfo methodInfo) runnerInfo.DebugInfo = new DebugInfo { DebugVSTestConsole = DebugVSTestConsole, - DebugTesthost = DebugTesthost, + DebugTestHost = DebugTestHost, DebugDataCollector = DebugDataCollector, NoDefaultBreakpoints = NoDefaultBreakpoints, }; diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/TranslationLayerCompatibilityDataSource.cs b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/RunnerCompatibilityDataSource.cs similarity index 86% rename from test/Microsoft.TestPlatform.AcceptanceTests/Extension/TranslationLayerCompatibilityDataSource.cs rename to test/Microsoft.TestPlatform.AcceptanceTests/Extension/RunnerCompatibilityDataSource.cs index d55cfe0c62..3b7b9d10ca 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/TranslationLayerCompatibilityDataSource.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/RunnerCompatibilityDataSource.cs @@ -38,7 +38,7 @@ public RunnerCompatibilityDataSource( } public bool DebugVSTestConsole { get; set; } - public bool DebugTesthost { get; set; } + public bool DebugTestHost { get; set; } public bool DebugDataCollector { get; set; } public bool NoDefaultBreakpoints { get; set; } = true; @@ -49,6 +49,10 @@ public RunnerCompatibilityDataSource( public string? BeforeFeature { get; set; } public string? AfterFeature { get; set; } + + //public string? BeforeTestHostFeature { get; set; } + //public string? AfterTestHostFeature { get; set; } + public string? BeforeAdapterFeature { get; set; } public string? AfterAdapterFeature { get; set; } @@ -60,14 +64,18 @@ public override void CreateData(MethodInfo methodInfo) _builder.WithOlderConfigurations = false; _builder.WithInProcess = InProcess; - _builder.BeforeFeature = BeforeFeature; - _builder.AfterFeature = AfterFeature; + _builder.BeforeRunnerFeature = BeforeFeature; + _builder.AfterRunnerFeature = AfterFeature; + + //_builder.BeforeTestHostFeature = BeforeTestHostFeature; + //_builder.AfterTestHostFeature = AfterTestHostFeature; + _builder.BeforeAdapterFeature = BeforeAdapterFeature; _builder.AfterAdapterFeature = AfterAdapterFeature; _builder.DebugDataCollector = DebugDataCollector; _builder.DebugVSTestConsole = DebugVSTestConsole; - _builder.DebugTesthost = DebugTesthost; + _builder.DebugTestHost = DebugTestHost; _builder.NoDefaultBreakpoints = NoDefaultBreakpoints; var data = _builder.CreateData(); diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/RunnnerInfo.cs b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/RunnnerInfo.cs index bb30619479..88f1404005 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/RunnnerInfo.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/RunnnerInfo.cs @@ -2,7 +2,7 @@ // Licensed under the MIT license. See LICENSE file in the project root for full license information. using System; -using System.Collections.Generic; +using System.Linq; using Microsoft.TestPlatform.TestUtilities; @@ -18,16 +18,14 @@ namespace Microsoft.TestPlatform.AcceptanceTests; public class RunnerInfo { public string? RunnerFramework { get; set; } - public VSTestConsoleInfo? VSTestConsoleInfo { get; set; } - - public string? TargetFramework { get; set; } public string? InIsolationValue { get; set; } - public DebugInfo? DebugInfo { get; set; } + public NetTestSdkInfo? TestHostInfo { get; set; } + public DllInfo? AdapterInfo { get; set; } - public List DllInfos { get; set; } = new(); + public string? Batch { get; set; } /// /// Is running via .NET "Core" vstest.console? @@ -49,5 +47,17 @@ public class RunnerInfo /// public bool IsNetFrameworkTarget => TargetFramework!.StartsWith("net4", StringComparison.InvariantCultureIgnoreCase); - public override string ToString() => $"Runner = {RunnerFramework}, TargetFramework = {TargetFramework}, {(string.IsNullOrEmpty(InIsolationValue) ? "InProcess" : "InIsolation")}, {VSTestConsoleInfo}, {string.Join(",", DllInfos)}"; + public override string ToString() + { + return string.Join(", ", new[] + { + Batch != null ? $"{Batch}" : null, + $"Runner = {RunnerFramework}", + $"TargetFramework = {TargetFramework}", + string.IsNullOrEmpty(InIsolationValue) ? "InProcess" : "InIsolation", + VSTestConsoleInfo == null ? null : VSTestConsoleInfo.ToString(), + TestHostInfo == null ? null : string.Join(",", TestHostInfo), + AdapterInfo == null ? null : string.Join(",", AdapterInfo) + }.Where(s => s != null)); + } } diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/TestPlatformCompatibilityDataSource.cs b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/TestPlatformCompatibilityDataSource.cs index b607811116..5f117fff0b 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/TestPlatformCompatibilityDataSource.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/TestPlatformCompatibilityDataSource.cs @@ -42,7 +42,7 @@ public TestPlatformCompatibilityDataSource( } public bool DebugVSTestConsole { get; set; } - public bool DebugTesthost { get; set; } + public bool DebugTestHost { get; set; } public bool DebugDataCollector { get; set; } public bool NoDefaultBreakpoints { get; set; } = true; @@ -59,9 +59,12 @@ public TestPlatformCompatibilityDataSource( public bool WithOlderConfigurations { get; set; } = true; + public string? BeforeRunnerFeature { get; set; } + public string? AfterRunnerFeature { get; set; } + + public string? BeforeTestHostFeature { get; set; } + public string? AfterTestHostFeature { get; set; } - public string? BeforeFeature { get; set; } - public string? AfterFeature { get; set; } public string? BeforeAdapterFeature { get; set; } public string? AfterAdapterFeature { get; set; } @@ -73,14 +76,18 @@ public override void CreateData(MethodInfo methodInfo) _builder.WithOlderConfigurations = WithOlderConfigurations; _builder.WithInProcess = WithInProcess; - _builder.BeforeFeature = BeforeFeature; - _builder.AfterFeature = AfterFeature; + _builder.BeforeRunnerFeature = BeforeRunnerFeature; + _builder.AfterRunnerFeature = AfterRunnerFeature; + + _builder.BeforeTestHostFeature = BeforeTestHostFeature; + _builder.AfterTestHostFeature = AfterTestHostFeature; + _builder.BeforeAdapterFeature = BeforeAdapterFeature; _builder.AfterAdapterFeature = AfterAdapterFeature; _builder.DebugDataCollector = DebugDataCollector; _builder.DebugVSTestConsole = DebugVSTestConsole; - _builder.DebugTesthost = DebugTesthost; + _builder.DebugTestHost = DebugTestHost; _builder.NoDefaultBreakpoints = NoDefaultBreakpoints; var data = _builder.CreateData(); diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/TesthostCompatibilityDataSource.cs b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/TesthostCompatibilityDataSource.cs index e32c32aa87..4bcbba2f93 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/TesthostCompatibilityDataSource.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/TesthostCompatibilityDataSource.cs @@ -10,11 +10,11 @@ namespace Microsoft.TestPlatform.AcceptanceTests; /// /// When that adds up to no configuration exception is thrown. /// -public class HostCompatibilityDataSource : TestDataSource +public class TestHostCompatibilityDataSource : TestDataSource { private readonly CompatibilityRowsBuilder _builder; - public HostCompatibilityDataSource( + public TestHostCompatibilityDataSource( string runnerFrameworks = AcceptanceTestBase.DEFAULT_HOST_NETFX_AND_NET, string hostFrameworks = AcceptanceTestBase.DEFAULT_HOST_NETFX_AND_NET, string hostVersions = AcceptanceTestBase.LATEST_TO_LEGACY) @@ -37,14 +37,12 @@ public HostCompatibilityDataSource( } public bool DebugVSTestConsole { get; set; } - public bool DebugTesthost { get; set; } + public bool DebugTestHost { get; set; } public bool DebugDataCollector { get; set; } public bool NoDefaultBreakpoints { get; set; } = true; public string? BeforeFeature { get; set; } public string? AfterFeature { get; set; } - public string? BeforeAdapterFeature { get; set; } - public string? AfterAdapterFeature { get; set; } public override void CreateData(MethodInfo methodInfo) { @@ -54,14 +52,12 @@ public override void CreateData(MethodInfo methodInfo) _builder.WithOlderConfigurations = false; _builder.WithInProcess = false; - _builder.BeforeFeature = BeforeFeature; - _builder.AfterFeature = AfterFeature; - _builder.BeforeAdapterFeature = BeforeAdapterFeature; - _builder.AfterAdapterFeature = AfterAdapterFeature; + _builder.BeforeTestHostFeature = BeforeFeature; + _builder.AfterTestHostFeature = AfterFeature; _builder.DebugDataCollector = DebugDataCollector; _builder.DebugVSTestConsole = DebugVSTestConsole; - _builder.DebugTesthost = DebugTesthost; + _builder.DebugTestHost = DebugTestHost; _builder.NoDefaultBreakpoints = NoDefaultBreakpoints; var data = _builder.CreateData(); diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/FilePatternParserTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/FilePatternParserTests.cs index 3d8f36218b..4e31163543 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/FilePatternParserTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/FilePatternParserTests.cs @@ -20,7 +20,7 @@ public void WildCardPatternShouldCorrectlyWorkOnFiles(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - var testAssembly = GetSampleTestAssembly(); + var testAssembly = GetSampleTestDll(); testAssembly = testAssembly.Replace("SimpleTestProject.dll", "*TestProj*.dll"); var arguments = PrepareArguments( @@ -40,7 +40,7 @@ public void WildCardPatternShouldCorrectlyWorkOnArbitraryDepthDirectories(Runner { SetTestEnvironment(_testEnvironment, runnerInfo); - var testAssembly = GetSampleTestAssembly(); + var testAssembly = GetSampleTestDll(); // Add one more directory to the temp path, so we can substitute it with ** // and copy then whole directory there. @@ -66,7 +66,7 @@ public void WildCardPatternShouldCorrectlyWorkForRelativeAssemblyPath(RunnerInfo { SetTestEnvironment(_testEnvironment, runnerInfo); - var testAssembly = GetSampleTestAssembly(); + var testAssembly = GetSampleTestDll(); testAssembly = testAssembly.Replace("SimpleTestProject.dll", "*TestProj*.dll"); var wildCardIndex = testAssembly.IndexOfAny(new char[] { '*' }); diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/FrameworkTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/FrameworkTests.cs index 394db87a80..9a00c24cb8 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/FrameworkTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/FrameworkTests.cs @@ -19,7 +19,7 @@ public void FrameworkArgumentShouldWork(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - var arguments = PrepareArguments(GetSampleTestAssembly(), string.Empty, string.Empty, string.Empty, resultsDirectory: TempDirectory.Path); + var arguments = PrepareArguments(GetSampleTestDll(), string.Empty, string.Empty, string.Empty, resultsDirectory: TempDirectory.Path); arguments = string.Concat(arguments, " ", $"/Framework:{FrameworkArgValue}"); InvokeVsTest(arguments); @@ -33,7 +33,7 @@ public void FrameworkShortNameArgumentShouldWork(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - var arguments = PrepareArguments(GetSampleTestAssembly(), string.Empty, string.Empty, string.Empty, resultsDirectory: TempDirectory.Path); + var arguments = PrepareArguments(GetSampleTestDll(), string.Empty, string.Empty, string.Empty, resultsDirectory: TempDirectory.Path); arguments = string.Concat(arguments, " ", $"/Framework:{_testEnvironment.TargetFramework}"); InvokeVsTest(arguments); @@ -49,7 +49,7 @@ public void OnWrongFrameworkPassedTestRunShouldNotRun(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - var arguments = PrepareArguments(GetSampleTestAssembly(), string.Empty, string.Empty, string.Empty, resultsDirectory: TempDirectory.Path); + var arguments = PrepareArguments(GetSampleTestDll(), string.Empty, string.Empty, string.Empty, resultsDirectory: TempDirectory.Path); if (runnerInfo.TargetFramework.Contains("netcore")) { arguments = string.Concat(arguments, " ", "/Framework:Framework45"); @@ -80,7 +80,7 @@ public void RunSpecificTestsShouldWorkWithFrameworkInCompatibleWarning(RunnerInf { SetTestEnvironment(_testEnvironment, runnerInfo); - var arguments = PrepareArguments(GetSampleTestAssembly(), string.Empty, string.Empty, string.Empty, resultsDirectory: TempDirectory.Path); + var arguments = PrepareArguments(GetSampleTestDll(), string.Empty, string.Empty, string.Empty, resultsDirectory: TempDirectory.Path); arguments = string.Concat(arguments, " ", "/tests:PassingTest"); arguments = string.Concat(arguments, " ", "/Framework:Framework40"); diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/LoggerTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/LoggerTests.cs index e9e30d8fdc..8e79cc7589 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/LoggerTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/LoggerTests.cs @@ -23,12 +23,12 @@ public void TrxLoggerWithFriendlyNameShouldProperlyOverwriteFile(RunnerInfo runn { SetTestEnvironment(_testEnvironment, runnerInfo); - var arguments = PrepareArguments(GetSampleTestAssembly(), GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, TempDirectory.Path); + var arguments = PrepareArguments(GetSampleTestDll(), GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, TempDirectory.Path); var trxFileName = "TestResults.trx"; arguments = string.Concat(arguments, $" /logger:\"trx;LogFileName={trxFileName}\""); InvokeVsTest(arguments); - arguments = PrepareArguments(GetSampleTestAssembly(), GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, TempDirectory.Path); + arguments = PrepareArguments(GetSampleTestDll(), GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, TempDirectory.Path); arguments = string.Concat(arguments, $" /logger:\"trx;LogFileName={trxFileName}\""); arguments = string.Concat(arguments, " /testcasefilter:Name~Pass"); InvokeVsTest(arguments); @@ -44,12 +44,12 @@ public void HtmlLoggerWithFriendlyNameShouldProperlyOverwriteFile(RunnerInfo run { SetTestEnvironment(_testEnvironment, runnerInfo); - var arguments = PrepareArguments(GetSampleTestAssembly(), GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, TempDirectory.Path); + var arguments = PrepareArguments(GetSampleTestDll(), GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, TempDirectory.Path); var htmlFileName = "TestResults.html"; arguments = string.Concat(arguments, $" /logger:\"html;LogFileName={htmlFileName}\""); InvokeVsTest(arguments); - arguments = PrepareArguments(GetSampleTestAssembly(), GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, TempDirectory.Path); + arguments = PrepareArguments(GetSampleTestDll(), GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, TempDirectory.Path); arguments = string.Concat(arguments, $" /logger:\"html;LogFileName={htmlFileName}\""); arguments = string.Concat(arguments, " /testcasefilter:Name~Pass"); InvokeVsTest(arguments); @@ -64,12 +64,12 @@ public void TrxLoggerWithExecutorUriShouldProperlyOverwriteFile(RunnerInfo runne { SetTestEnvironment(_testEnvironment, runnerInfo); - var arguments = PrepareArguments(GetSampleTestAssembly(), GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, TempDirectory.Path); + var arguments = PrepareArguments(GetSampleTestDll(), GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, TempDirectory.Path); var trxFileName = "TestResults.trx"; arguments = string.Concat(arguments, $" /logger:\"logger://Microsoft/TestPlatform/TrxLogger/v1;LogFileName={trxFileName}\""); InvokeVsTest(arguments); - arguments = PrepareArguments(GetSampleTestAssembly(), GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, TempDirectory.Path); + arguments = PrepareArguments(GetSampleTestDll(), GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, TempDirectory.Path); arguments = string.Concat(arguments, $" /logger:\"logger://Microsoft/TestPlatform/TrxLogger/v1;LogFileName={trxFileName}\""); arguments = string.Concat(arguments, " /testcasefilter:Name~Pass"); InvokeVsTest(arguments); @@ -86,11 +86,11 @@ public void TrxLoggerWithLogFilePrefixShouldGenerateMultipleTrx(RunnerInfo runne SetTestEnvironment(_testEnvironment, runnerInfo); var trxFileNamePattern = "TestResults"; - var arguments = PrepareArguments(GetSampleTestAssembly(), GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, TempDirectory.Path); + var arguments = PrepareArguments(GetSampleTestDll(), GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, TempDirectory.Path); arguments = string.Concat(arguments, $" /logger:\"logger://Microsoft/TestPlatform/TrxLogger/v1;LogFilePrefix={trxFileNamePattern}\""); InvokeVsTest(arguments); - arguments = PrepareArguments(GetSampleTestAssembly(), GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, TempDirectory.Path); + arguments = PrepareArguments(GetSampleTestDll(), GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, TempDirectory.Path); arguments = string.Concat(arguments, $" /logger:\"logger://Microsoft/TestPlatform/TrxLogger/v1;LogFilePrefix={trxFileNamePattern}\""); arguments = string.Concat(arguments, " /testcasefilter:Name~Pass"); InvokeVsTest(arguments); @@ -105,12 +105,12 @@ public void HtmlLoggerWithExecutorUriShouldProperlyOverwriteFile(RunnerInfo runn { SetTestEnvironment(_testEnvironment, runnerInfo); - var arguments = PrepareArguments(GetSampleTestAssembly(), GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, TempDirectory.Path); + var arguments = PrepareArguments(GetSampleTestDll(), GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, TempDirectory.Path); var htmlFileName = "TestResults.html"; arguments = string.Concat(arguments, $" /logger:\"logger://Microsoft/TestPlatform/htmlLogger/v1;LogFileName{htmlFileName}\""); InvokeVsTest(arguments); - arguments = PrepareArguments(GetSampleTestAssembly(), GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, TempDirectory.Path); + arguments = PrepareArguments(GetSampleTestDll(), GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, TempDirectory.Path); arguments = string.Concat(arguments, $" /logger:\"logger://Microsoft/TestPlatform/htmlLogger/v1;LogFileName={htmlFileName}\""); arguments = string.Concat(arguments, " /testcasefilter:Name~Pass"); InvokeVsTest(arguments); @@ -126,7 +126,7 @@ public void TrxLoggerResultSummaryOutcomeValueShouldBeFailedIfNoTestsExecutedAnd { SetTestEnvironment(_testEnvironment, runnerInfo); - var arguments = PrepareArguments(GetSampleTestAssembly(), GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue); + var arguments = PrepareArguments(GetSampleTestDll(), GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue); var trxFilePath = Path.Combine(TempDirectory.Path, "TrxLogger.trx"); arguments = string.Concat(arguments, $" /logger:\"trx;LogFileName={trxFilePath}\""); @@ -149,7 +149,7 @@ public void TrxLoggerResultSummaryOutcomeValueShouldNotChangeIfNoTestsExecutedAn { SetTestEnvironment(_testEnvironment, runnerInfo); - var arguments = PrepareArguments(GetSampleTestAssembly(), GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue); + var arguments = PrepareArguments(GetSampleTestDll(), GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue); var trxFilePath = Path.Combine(TempDirectory.Path, "TrxLogger.trx"); arguments = string.Concat(arguments, $" /logger:\"trx;LogFileName={trxFilePath}\""); diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/PlatformTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/PlatformTests.cs index b149fa2994..289363a269 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/PlatformTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/PlatformTests.cs @@ -45,7 +45,7 @@ private void RunTestExecutionWithPlatform(string platformArg, string testhostPro { var arguments = PrepareArguments( - GetSampleTestAssembly(), + GetSampleTestDll(), GetTestAdapterPath(), string.Empty, FrameworkArgValue, _testEnvironment.InIsolationValue, resultsDirectory: TempDirectory.Path); diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/PortableNugetPackageTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/PortableNugetPackageTests.cs index d5f227fccf..be24f8ff38 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/PortableNugetPackageTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/PortableNugetPackageTests.cs @@ -58,7 +58,7 @@ public void DiscoverAllTests(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - InvokeVsTestForDiscovery(GetSampleTestAssembly(), GetTestAdapterPath(), string.Empty, FrameworkArgValue); + InvokeVsTestForDiscovery(GetSampleTestDll(), GetTestAdapterPath(), string.Empty, FrameworkArgValue); var listOfTests = new[] { "SampleUnitTestProject.UnitTest1.PassingTest", "SampleUnitTestProject.UnitTest1.FailingTest", "SampleUnitTestProject.UnitTest1.SkippingTest" }; ValidateDiscoveredTests(listOfTests); diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/ProcessesInteractionTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/ProcessesInteractionTests.cs index e82ddc9520..13cd916a56 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/ProcessesInteractionTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/ProcessesInteractionTests.cs @@ -22,7 +22,7 @@ public void WhenTestHostProcessExitsBecauseTheTargetedRuntimeIsNoFoundThenTheMes // Arrange SetTestEnvironment(_testEnvironment, runnerInfo); const string testAssetProjectName = "SimpleTestProjectMessedUpTargetFramework"; - var assemblyPath = GetAssetFullPath(testAssetProjectName + ".dll", Core21TargetFramework); + var assemblyPath = GetTestDllForFramework(testAssetProjectName + ".dll", Core21TargetFramework); UpdateRuntimeConfigJsonWithInvalidFramework(assemblyPath, testAssetProjectName); // Act diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/ResultsDirectoryTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/ResultsDirectoryTests.cs index a7b02265a7..4b6d60ca87 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/ResultsDirectoryTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/ResultsDirectoryTests.cs @@ -20,7 +20,7 @@ public void TrxFileShouldBeCreatedInResultsDirectory(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - var arguments = PrepareArguments(GetSampleTestAssembly(), GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue); + var arguments = PrepareArguments(GetSampleTestDll(), GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue); var trxFileName = "TestResults.trx"; var trxFilePath = Path.Combine(TempDirectory.Path, trxFileName); arguments = string.Concat(arguments, $" /logger:\"trx;LogFileName={trxFileName}\""); @@ -41,7 +41,7 @@ public void ResultsDirectoryRelativePathShouldWork(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - var arguments = PrepareArguments(GetSampleTestAssembly(), GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue); + var arguments = PrepareArguments(GetSampleTestDll(), GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue); var trxFileName = "TestResults.trx"; var relativeDirectory = @"relative\directory"; var resultsDirectory = Path.Combine(Directory.GetCurrentDirectory(), relativeDirectory); diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/RunsettingsTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/RunsettingsTests.cs index 3d5dbcbea7..d1c8084875 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/RunsettingsTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/RunsettingsTests.cs @@ -246,7 +246,7 @@ public void RunSettingsWithInvalidValueShouldLogError(RunnerInfo runnerInfo) }; var runsettingsFilePath = GetRunsettingsFilePath(runConfigurationDictionary, TempDirectory); var arguments = PrepareArguments( - GetSampleTestAssembly(), + GetSampleTestDll(), string.Empty, runsettingsFilePath, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: TempDirectory.Path); @@ -267,7 +267,7 @@ public void TestAdapterPathFromRunSettings(RunnerInfo runnerInfo) }; var runsettingsFilePath = GetRunsettingsFilePath(runConfigurationDictionary, TempDirectory); var arguments = PrepareArguments( - GetSampleTestAssembly(), + GetSampleTestDll(), string.Empty, runsettingsFilePath, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: TempDirectory.Path); @@ -284,7 +284,7 @@ public void LegacySettingsWithPlatform(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - var testAssemblyPath = GetAssetFullPath("LegacySettingsUnitTestProject.dll"); + var testAssemblyPath = GetTestDll("LegacySettingsUnitTestProject.dll"); _ = Path.GetDirectoryName(testAssemblyPath); var runsettingsXml = @" @@ -315,7 +315,7 @@ public void LegacySettingsWithScripts(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - var testAssemblyPath = GetAssetFullPath("LegacySettingsUnitTestProject.dll"); + var testAssemblyPath = GetTestDll("LegacySettingsUnitTestProject.dll"); // Create the script files var guid = Guid.NewGuid(); @@ -361,7 +361,7 @@ public void LegacySettingsWithDeploymentItem(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - var testAssemblyPath = GetAssetFullPath("LegacySettingsUnitTestProject.dll"); + var testAssemblyPath = GetTestDll("LegacySettingsUnitTestProject.dll"); var testAssemblyDirectory = Path.GetDirectoryName(testAssemblyPath); var deploymentItem = Path.Combine(testAssemblyDirectory, "Deployment", "DeploymentFile.xml"); @@ -397,7 +397,7 @@ public void LegacySettingsTestTimeout(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - var testAssemblyPath = GetAssetFullPath("LegacySettingsUnitTestProject.dll"); + var testAssemblyPath = GetTestDll("LegacySettingsUnitTestProject.dll"); var runsettingsXml = @" true @@ -424,7 +424,7 @@ public void LegacySettingsAssemblyResolution(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - var testAssemblyPath = GetAssetFullPath("LegacySettingsUnitTestProject.dll"); + var testAssemblyPath = GetTestDll("LegacySettingsUnitTestProject.dll"); var runsettingsFormat = @" true @@ -466,7 +466,7 @@ public void EnvironmentVariablesSettingsShouldSetEnvironmentVariables(RunnerInfo { SetTestEnvironment(_testEnvironment, runnerInfo); - var testAssemblyPath = GetAssetFullPath("EnvironmentVariablesTestProject.dll"); + var testAssemblyPath = GetTestDll("EnvironmentVariablesTestProject.dll"); var runsettingsXml = @" diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/TelemetryTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/TelemetryTests.cs index fdf49097ce..1b519c1d9f 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/TelemetryTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/TelemetryTests.cs @@ -49,7 +49,7 @@ private void RunTests(RunnerInfo runnerInfo) return; } - var assemblyPaths = GetAssetFullPath("SimpleTestProject2.dll"); + var assemblyPaths = GetTestDll("SimpleTestProject2.dll"); var env = new Dictionary { @@ -70,7 +70,7 @@ private void DiscoverTests(RunnerInfo runnerInfo) return; } - var assemblyPaths = GetAssetFullPath("SimpleTestProject2.dll"); + var assemblyPaths = GetTestDll("SimpleTestProject2.dll"); var env = new Dictionary { diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/TestCaseFilterTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/TestCaseFilterTests.cs index c4363cf825..d235068166 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/TestCaseFilterTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/TestCaseFilterTests.cs @@ -21,7 +21,7 @@ public void RunSelectedTestsWithAndOperatorTrait(RunnerInfo runnerInfo) SetTestEnvironment(_testEnvironment, runnerInfo); var arguments = PrepareArguments( - GetSampleTestAssembly(), + GetSampleTestDll(), GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: TempDirectory.Path); @@ -38,7 +38,7 @@ public void RunSelectedTestsWithCategoryTraitInMixCase(RunnerInfo runnerInfo) SetTestEnvironment(_testEnvironment, runnerInfo); var arguments = PrepareArguments( - GetSampleTestAssembly(), + GetSampleTestDll(), GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: TempDirectory.Path); @@ -55,7 +55,7 @@ public void RunSelectedTestsWithClassNameTrait(RunnerInfo runnerInfo) SetTestEnvironment(_testEnvironment, runnerInfo); var arguments = PrepareArguments( - GetSampleTestAssembly(), + GetSampleTestDll(), GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: TempDirectory.Path); @@ -72,7 +72,7 @@ public void RunSelectedTestsWithFullyQualifiedNameTrait(RunnerInfo runnerInfo) SetTestEnvironment(_testEnvironment, runnerInfo); var arguments = PrepareArguments( - GetSampleTestAssembly(), + GetSampleTestDll(), GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: TempDirectory.Path); @@ -91,7 +91,7 @@ public void RunSelectedTestsWithNameTrait(RunnerInfo runnerInfo) SetTestEnvironment(_testEnvironment, runnerInfo); var arguments = PrepareArguments( - GetSampleTestAssembly(), + GetSampleTestDll(), GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: TempDirectory.Path); @@ -108,7 +108,7 @@ public void RunSelectedTestsWithOrOperatorTrait(RunnerInfo runnerInfo) SetTestEnvironment(_testEnvironment, runnerInfo); var arguments = PrepareArguments( - GetSampleTestAssembly(), + GetSampleTestDll(), GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: TempDirectory.Path); @@ -125,7 +125,7 @@ public void RunSelectedTestsWithPriorityTrait(RunnerInfo runnerInfo) SetTestEnvironment(_testEnvironment, runnerInfo); var arguments = PrepareArguments( - GetSampleTestAssembly(), + GetSampleTestDll(), GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: TempDirectory.Path); diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/CodeCoverageTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/CodeCoverageTests.cs index c77faab8ab..a5f5efb6eb 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/CodeCoverageTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/CodeCoverageTests.cs @@ -437,7 +437,7 @@ public async Task EndSessionShouldEnsureVstestConsoleProcessDies(RunnerInfo runn private IList GetTestAssemblies() { - return GetProjects().Select(p => GetAssetFullPath(p)).ToList(); + return GetProjects().Select(p => GetTestDll(p)).ToList(); } private IList GetProjects() diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/CustomTestHostTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/CustomTestHostTests.cs index 996adfec65..d3730b7378 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/CustomTestHostTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/CustomTestHostTests.cs @@ -1,7 +1,6 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT license. See LICENSE file in the project root for full license information. -using System.Collections.Generic; using System.Diagnostics; using System.Linq; using System.Threading; @@ -24,7 +23,6 @@ namespace Microsoft.TestPlatform.AcceptanceTests.TranslationLayerTests; public class CustomTestHostTests : AcceptanceTestBase { private IVsTestConsoleWrapper _vstestConsoleWrapper; - private RunEventHandler _runEventHandler; [TestCleanup] public void Cleanup() @@ -33,109 +31,120 @@ public void Cleanup() } [TestMethod] - [RunnerCompatibilityDataSource(BeforeFeature = Features.ATTACH_DEBUGGER)] + [RunnerCompatibilityDataSource(BeforeFeature = Features.ATTACH_DEBUGGER_FLOW)] + [TestHostCompatibilityDataSource(BeforeFeature = Features.ATTACH_DEBUGGER_FLOW)] public void RunTestsWithCustomTestHostLauncherLaunchesTheProcessUsingTheProvidedLauncher(RunnerInfo runnerInfo) { + // Pins the existing functionality. + + // Arrange SetTestEnvironment(_testEnvironment, runnerInfo); + _vstestConsoleWrapper = GetVsTestConsoleWrapper(); - _runEventHandler = new RunEventHandler(); + var runEventHandler = new RunEventHandler(); + // Act var customTestHostLauncher = new TestHostLauncherV1(); - _vstestConsoleWrapper.RunTestsWithCustomTestHost(GetTestAssemblies(), GetDefaultRunSettings(), _runEventHandler, customTestHostLauncher); + _vstestConsoleWrapper.RunTestsWithCustomTestHost(GetTestDlls("MSTestProject1.dll", "MSTestProject2.dll"), GetRunSettingsWithCurrentTargetFramework(), runEventHandler, customTestHostLauncher); // Assert + EnsureTestsRunWithoutErrors(runEventHandler, passed: 2, failed: 2, skipped: 2); + + // Ensure we tried to launch testhost process. customTestHostLauncher.Should().BeAssignableTo(); customTestHostLauncher.LaunchProcessProcessId.Should().NotBeNull("we should launch some real process and save the pid of it"); - - _runEventHandler.TestResults.Should().HaveCount(6); - _runEventHandler.TestResults.Count(t => t.Outcome == TestOutcome.Passed).Should().Be(2); - _runEventHandler.TestResults.Count(t => t.Outcome == TestOutcome.Failed).Should().Be(2); - _runEventHandler.TestResults.Count(t => t.Outcome == TestOutcome.Skipped).Should().Be(2); } [TestMethod] - [RunnerCompatibilityDataSource(BeforeFeature = Features.ATTACH_DEBUGGER)] + // [RunnerCompatibilityDataSource(BeforeFeature = Features.ATTACH_DEBUGGER_FLOW)] + [TestHostCompatibilityDataSource("net451", "netcoreapp2.1", "LegacyStable", BeforeFeature = Features.ATTACH_DEBUGGER_FLOW, DebugVSTestConsole = true)] + [Ignore("This is not working for any testhost prior 16.7.0 where the change was introduced. The launch testhost flow was replaced with AttachDebugger in runner, and the new callback to AttachDebugger happens in testhost." + + "But any testhost prior 16.7.0 where the change was introduced does not know to call back AttachDebugger, and the call never happens.")] + // You can confirm that the functionality broke between runner and testhost, past this point by using newer runners, against older testhosts. + // [TestPlatformCompatibilityDataSource(AfterRunnerFeature = Features.ATTACH_DEBUGGER_FLOW, BeforeTestHostFeature = Features.ATTACH_DEBUGGER_FLOW)] public void RunTestsWithCustomTestHostLauncherLaunchesTheProcessUsingTheProvidedLauncherWhenITestHostLauncher2IsProvided(RunnerInfo runnerInfo) { + // Ensures compatibility with testhost and runners that were created before 16.3.0. It makes sure that even if user provides + // an implementation of the ITestHostLauncher2 interface, then testhost expecting ITestHostLauncher still works correctly. + + // Arrange SetTestEnvironment(_testEnvironment, runnerInfo); _vstestConsoleWrapper = GetVsTestConsoleWrapper(); - _runEventHandler = new RunEventHandler(); + var runEventHandler = new RunEventHandler(); + // Act var customTestHostLauncher = new TestHostLauncherV2(); - _vstestConsoleWrapper.RunTestsWithCustomTestHost(GetTestAssemblies(), GetDefaultRunSettings(), _runEventHandler, customTestHostLauncher); + _vstestConsoleWrapper.RunTestsWithCustomTestHost(GetTestDlls("MSTestProject1.dll", "MSTestProject2.dll"), GetRunSettingsWithCurrentTargetFramework(), runEventHandler, customTestHostLauncher); // Assert + EnsureTestsRunWithoutErrors(runEventHandler, passed: 2, failed: 2, skipped: 2); + customTestHostLauncher.Should().BeAssignableTo(); customTestHostLauncher.LaunchProcessProcessId.Should().NotBeNull("we should launch some real process and save the pid of it"); customTestHostLauncher.AttachDebuggerProcessId.Should().BeNull("we should not be asked to attach to a debugger, that flow is not used when vstest.console does not support it yet, even when it is given ITestHostLauncher2"); - - _runEventHandler.TestResults.Should().HaveCount(6); - _runEventHandler.TestResults.Count(t => t.Outcome == TestOutcome.Passed).Should().Be(2); - _runEventHandler.TestResults.Count(t => t.Outcome == TestOutcome.Failed).Should().Be(2); - _runEventHandler.TestResults.Count(t => t.Outcome == TestOutcome.Skipped).Should().Be(2); } [TestMethod] - [RunnerCompatibilityDataSource(AfterFeature = Features.ATTACH_DEBUGGER)] + [RunnerCompatibilityDataSource(AfterFeature = Features.ATTACH_DEBUGGER_FLOW)] + // [TestHostCompatibilityDataSource(AfterFeature = Features.ATTACH_DEBUGGER_FLOW)] + [TestHostCompatibilityDataSource()] public void RunTestsWithCustomTestHostLauncherAttachesToDebuggerUsingTheProvidedLauncher(RunnerInfo runnerInfo) { + + // Arrange SetTestEnvironment(_testEnvironment, runnerInfo); _vstestConsoleWrapper = GetVsTestConsoleWrapper(); - _runEventHandler = new RunEventHandler(); + var runEventHandler = new RunEventHandler(); + // Act var customTestHostLauncher = new TestHostLauncherV2(); - _vstestConsoleWrapper.RunTestsWithCustomTestHost(GetTestAssemblies(), GetDefaultRunSettings(), _runEventHandler, customTestHostLauncher); + _vstestConsoleWrapper.RunTestsWithCustomTestHost(GetTestDlls("MSTestProject1.dll", "MSTestProject2.dll"), GetRunSettingsWithCurrentTargetFramework(), runEventHandler, customTestHostLauncher); // Assert + EnsureTestsRunWithoutErrors(runEventHandler, passed: 2, failed: 2, skipped: 2); + customTestHostLauncher.Should().BeAssignableTo(); customTestHostLauncher.AttachDebuggerProcessId.Should().NotBeNull("we should be asked to attach a debugger to some process and save the pid of the process"); customTestHostLauncher.LaunchProcessProcessId.Should().BeNull("we should not be asked to launch some real process, that flow is not used when vstest.console supports it and is given ITestHostLauncher2"); - - _runEventHandler.TestResults.Should().HaveCount(6); - _runEventHandler.TestResults.Count(t => t.Outcome == TestOutcome.Passed).Should().Be(2); - _runEventHandler.TestResults.Count(t => t.Outcome == TestOutcome.Failed).Should().Be(2); - _runEventHandler.TestResults.Count(t => t.Outcome == TestOutcome.Skipped).Should().Be(2); } [TestMethod] - //[Ignore("This is not working. The compatibility code only checks the protocol version (in handler), which is dictated by testhost. " - // + "It sees 6 but does not realize that the provided CustomTesthostLauncher is not supporting the new feature, it ends up calling back to EditoAttachDebugger" + - // "in translation layer, and that just silently skips the call.")] - [RunnerCompatibilityDataSource("net451", "net451", "Latest", AfterFeature = Features.ATTACH_DEBUGGER, DebugVSTestConsole = true, DebugTesthost=true)] + [Ignore("This is not working. The compatibility code only checks the protocol version (in handler), which is dictated by testhost. " + + "It sees 6 but does not realize that the provided CustomTesthostLauncher is not supporting the new feature, it ends up calling back to EditoAttachDebugger" + + "in translation layer, and that just silently skips the call.")] + [RunnerCompatibilityDataSource(AfterFeature = Features.ATTACH_DEBUGGER_FLOW)] + [TestHostCompatibilityDataSource(AfterFeature = Features.ATTACH_DEBUGGER_FLOW)] public void RunTestsWithCustomTestHostLauncherUsesLaunchWhenGivenAnOutdatedITestHostLauncher(RunnerInfo runnerInfo) { + // Arrange SetTestEnvironment(_testEnvironment, runnerInfo); _vstestConsoleWrapper = GetVsTestConsoleWrapper(); - _runEventHandler = new RunEventHandler(); + var runEventHandler = new RunEventHandler(); + // Act var customTestHostLauncher = new TestHostLauncherV1(); - _vstestConsoleWrapper.RunTestsWithCustomTestHost(GetTestAssemblies(), GetDefaultRunSettings(), _runEventHandler, customTestHostLauncher); + _vstestConsoleWrapper.RunTestsWithCustomTestHost(GetTestDlls("MSTestProject1.dll", "MSTestProject2.dll"), GetRunSettingsWithCurrentTargetFramework(), runEventHandler, customTestHostLauncher); // Assert + EnsureTestsRunWithoutErrors(runEventHandler, passed: 2, failed: 2, skipped: 2); + customTestHostLauncher.Should().NotBeAssignableTo(); customTestHostLauncher.LaunchProcessProcessId.Should().NotBeNull("we should launch some real process and save the pid of it"); - - _runEventHandler.TestResults.Should().HaveCount(6); - _runEventHandler.TestResults.Count(t => t.Outcome == TestOutcome.Passed).Should().Be(2); - _runEventHandler.TestResults.Count(t => t.Outcome == TestOutcome.Failed).Should().Be(2); - _runEventHandler.TestResults.Count(t => t.Outcome == TestOutcome.Skipped).Should().Be(2); } - private IList GetTestAssemblies() + private static void EnsureTestsRunWithoutErrors(RunEventHandler runEventHandler, int passed, int failed, int skipped) { - var testAssemblies = new List - { - GetAssetFullPath("SimpleTestProject.dll"), - GetAssetFullPath("SimpleTestProject2.dll") - }; - - return testAssemblies; + runEventHandler.Errors.Should().BeEmpty(); + runEventHandler.TestResults.Should().HaveCount(passed + failed + skipped); + runEventHandler.TestResults.Count(t => t.Outcome == TestOutcome.Passed).Should().Be(passed); + runEventHandler.TestResults.Count(t => t.Outcome == TestOutcome.Failed).Should().Be(failed); + runEventHandler.TestResults.Count(t => t.Outcome == TestOutcome.Skipped).Should().Be(skipped); } /// /// The custom test host launcher implementing ITestHostLauncher. /// - public class TestHostLauncherV1 : ITestHostLauncher + private class TestHostLauncherV1 : ITestHostLauncher { public int? LaunchProcessProcessId { get; private set; } @@ -168,9 +177,9 @@ public int LaunchTestHost(TestProcessStartInfo defaultTestHostStartInfo, Cancell } /// - /// The custom test host launcher implementing ITestHostLauncher2. + /// The custom test host launcher implementing ITestHostLauncher2, and through that also ITestHostLauncher. /// - public class TestHostLauncherV2 : TestHostLauncherV1, ITestHostLauncher2 + private class TestHostLauncherV2 : TestHostLauncherV1, ITestHostLauncher2 { public int? AttachDebuggerProcessId { get; private set; } diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/DataCollectorAttachmentProcessor.cs b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/DataCollectorAttachmentProcessor.cs index 16708093ba..4a38f260e7 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/DataCollectorAttachmentProcessor.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/DataCollectorAttachmentProcessor.cs @@ -124,5 +124,5 @@ private static void CreateDataCollectionRunSettingsFile(string destinationRunset } private IList GetTestAssemblies() - => new List { "SimpleTestProject.dll", "SimpleTestProject2.dll" }.Select(p => GetAssetFullPath(p)).ToList(); + => new List { "SimpleTestProject.dll", "SimpleTestProject2.dll" }.Select(p => GetTestDll(p)).ToList(); } diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/DifferentTestFrameworkSimpleTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/DifferentTestFrameworkSimpleTests.cs index b08763da05..af2456f1d8 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/DifferentTestFrameworkSimpleTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/DifferentTestFrameworkSimpleTests.cs @@ -47,12 +47,12 @@ public void RunTestsWithNunitAdapter(RunnerInfo runnerInfo) var sources = new List { - GetAssetFullPath("NUTestProject.dll") + GetTestDll("NUTestProject.dll") }; _vstestConsoleWrapper.RunTests( sources, - GetDefaultRunSettings(), + GetRunSettingsWithCurrentTargetFramework(), _runEventHandler); var testCase = @@ -94,7 +94,7 @@ public void RunTestsWithXunitAdapter(RunnerInfo runnerInfo) _vstestConsoleWrapper.RunTests( sources, - GetDefaultRunSettings(), + GetRunSettingsWithCurrentTargetFramework(), _runEventHandler); var testCase = @@ -134,7 +134,7 @@ public void RunTestsWithChutzpahAdapter(RunnerInfo runnerInfo) _vstestConsoleWrapper.RunTests( sources, - GetDefaultRunSettings(), + GetRunSettingsWithCurrentTargetFramework(), _runEventHandler); var testCase = diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/DiscoverTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/DiscoverTests.cs index 38ec639a3f..b09ce2cc0f 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/DiscoverTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/DiscoverTests.cs @@ -50,7 +50,7 @@ public void DiscoverTestsUsingDiscoveryEventHandler1(RunnerInfo runnerInfo) _discoveryEventHandler2 = new DiscoveryEventHandler2(); var vstestConsoleWrapper = GetVsTestConsoleWrapper(); - vstestConsoleWrapper.DiscoverTests(GetTestAssemblies(), GetDefaultRunSettings(), _discoveryEventHandler); + vstestConsoleWrapper.DiscoverTests(GetTestAssemblies(), GetRunSettingsWithCurrentTargetFramework(), _discoveryEventHandler); // Assert. Assert.AreEqual(6, _discoveryEventHandler.DiscoveredTestCases.Count); @@ -69,7 +69,7 @@ public void DiscoverTestsUsingDiscoveryEventHandler2AndTelemetryOptedOut(RunnerI var vstestConsoleWrapper = GetVsTestConsoleWrapper(); vstestConsoleWrapper.DiscoverTests( GetTestAssemblies(), - GetDefaultRunSettings(), + GetRunSettingsWithCurrentTargetFramework(), new TestPlatformOptions() { CollectMetrics = false }, _discoveryEventHandler2); @@ -86,7 +86,7 @@ public void DiscoverTestsUsingDiscoveryEventHandler2AndTelemetryOptedIn(RunnerIn SetTestEnvironment(_testEnvironment, runnerInfo); Setup(); - _vstestConsoleWrapper.DiscoverTests(GetTestAssemblies(), GetDefaultRunSettings(), new TestPlatformOptions() { CollectMetrics = true }, _discoveryEventHandler2); + _vstestConsoleWrapper.DiscoverTests(GetTestAssemblies(), GetRunSettingsWithCurrentTargetFramework(), new TestPlatformOptions() { CollectMetrics = true }, _discoveryEventHandler2); // Assert. Assert.AreEqual(6, _discoveryEventHandler2.DiscoveredTestCases.Count); @@ -164,7 +164,7 @@ public void DisoverTestUsingEventHandler2ShouldContainAllSourcesAsFullyDiscovere _vstestConsoleWrapper.DiscoverTests( GetTestAssemblies(), - GetDefaultRunSettings(), + GetRunSettingsWithCurrentTargetFramework(), null, eventHandler2); @@ -182,7 +182,7 @@ public void DiscoverTestsUsingSourceNavigation(RunnerInfo runnerInfo) _vstestConsoleWrapper.DiscoverTests( GetTestAssemblies(), - GetDefaultRunSettings(), + GetRunSettingsWithCurrentTargetFramework(), _discoveryEventHandler); // Assert. @@ -209,9 +209,9 @@ public void CancelTestDiscovery(RunnerInfo runnerInfo) // Setup var testAssemblies = new List { - GetAssetFullPath("DiscoveryTestProject.dll"), - GetAssetFullPath("SimpleTestProject.dll"), - GetAssetFullPath("SimpleTestProject2.dll") + GetTestDll("DiscoveryTestProject.dll"), + GetTestDll("SimpleTestProject.dll"), + GetTestDll("SimpleTestProject2.dll") }; SetTestEnvironment(_testEnvironment, runnerInfo); @@ -223,7 +223,7 @@ public void CancelTestDiscovery(RunnerInfo runnerInfo) ((IEnumerable testcases) => discoveredTests.AddRange(testcases)); // Act - var discoveryTask = Task.Run(() => _vstestConsoleWrapper.DiscoverTests(testAssemblies, GetDefaultRunSettings(), discoveryEvents.Object)); + var discoveryTask = Task.Run(() => _vstestConsoleWrapper.DiscoverTests(testAssemblies, GetRunSettingsWithCurrentTargetFramework(), discoveryEvents.Object)); Task.Delay(2000).Wait(); _vstestConsoleWrapper.CancelDiscovery(); @@ -238,8 +238,8 @@ private IList GetTestAssemblies() { var testAssemblies = new List { - GetAssetFullPath("SimpleTestProject.dll"), - GetAssetFullPath("SimpleTestProject2.dll") + GetTestDll("SimpleTestProject.dll"), + GetTestDll("SimpleTestProject2.dll") }; return testAssemblies; diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/LiveUnitTestingTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/LiveUnitTestingTests.cs index 0beac2e85c..01b4367468 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/LiveUnitTestingTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/LiveUnitTestingTests.cs @@ -90,8 +90,8 @@ private IList GetTestAssemblies() { var testAssemblies = new List { - GetAssetFullPath("SimpleTestProject.dll"), - GetAssetFullPath("SimpleTestProject2.dll") + GetTestDll("SimpleTestProject.dll"), + GetTestDll("SimpleTestProject2.dll") }; return testAssemblies; diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/RunSelectedTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/RunSelectedTests.cs index 73b07981b6..f2722cc154 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/RunSelectedTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/RunSelectedTests.cs @@ -42,10 +42,10 @@ public void RunSelectedTestsWithoutTestPlatformOptions(RunnerInfo runnerInfo) SetTestEnvironment(_testEnvironment, runnerInfo); Setup(); - _vstestConsoleWrapper.DiscoverTests(GetTestAssemblies(), GetDefaultRunSettings(), _discoveryEventHandler); + _vstestConsoleWrapper.DiscoverTests(GetTestAssemblies(), GetRunSettingsWithCurrentTargetFramework(), _discoveryEventHandler); var testCases = _discoveryEventHandler.DiscoveredTestCases; - _vstestConsoleWrapper.RunTests(testCases, GetDefaultRunSettings(), _runEventHandler); + _vstestConsoleWrapper.RunTests(testCases, GetRunSettingsWithCurrentTargetFramework(), _runEventHandler); // Assert Assert.AreEqual(6, _runEventHandler.TestResults.Count); @@ -62,12 +62,12 @@ public void RunSelectedTestsWithTestPlatformOptions(RunnerInfo runnerInfo) SetTestEnvironment(_testEnvironment, runnerInfo); Setup(); - _vstestConsoleWrapper.DiscoverTests(GetTestAssemblies(), GetDefaultRunSettings(), _discoveryEventHandler); + _vstestConsoleWrapper.DiscoverTests(GetTestAssemblies(), GetRunSettingsWithCurrentTargetFramework(), _discoveryEventHandler); var testCases = _discoveryEventHandler.DiscoveredTestCases; _vstestConsoleWrapper.RunTests( testCases, - GetDefaultRunSettings(), + GetRunSettingsWithCurrentTargetFramework(), new TestPlatformOptions() { CollectMetrics = true }, _runEventHandler); @@ -85,8 +85,8 @@ private IList GetTestAssemblies() { var testAssemblies = new List { - GetAssetFullPath("SimpleTestProject.dll"), - GetAssetFullPath("SimpleTestProject2.dll") + GetTestDll("SimpleTestProject.dll"), + GetTestDll("SimpleTestProject2.dll") }; return testAssemblies; diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/RunTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/RunTests.cs index ffe2449e15..168120af82 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/RunTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/RunTests.cs @@ -47,7 +47,7 @@ public void RunAllTests(RunnerInfo runnerInfo) var vstestConsoleWrapper = GetVsTestConsoleWrapper(); var runEventHandler = new RunEventHandler(); - vstestConsoleWrapper.RunTests(GetTestAssemblies(), GetDefaultRunSettings(), runEventHandler); + vstestConsoleWrapper.RunTests(GetTestAssemblies(), GetRunSettingsWithCurrentTargetFramework(), runEventHandler); // Assert Assert.AreEqual(6, runEventHandler.TestResults.Count); @@ -67,7 +67,7 @@ public void EndSessionShouldEnsureVstestConsoleProcessDies(RunnerInfo runnerInfo SetTestEnvironment(_testEnvironment, runnerInfo); Setup(); - _vstestConsoleWrapper.RunTests(GetTestAssemblies(), GetDefaultRunSettings(), _runEventHandler); + _vstestConsoleWrapper.RunTests(GetTestAssemblies(), GetRunSettingsWithCurrentTargetFramework(), _runEventHandler); _vstestConsoleWrapper?.EndSession(); // Assert @@ -89,7 +89,7 @@ public void RunTestsWithTelemetryOptedIn(RunnerInfo runnerInfo) _vstestConsoleWrapper.RunTests( GetTestAssemblies(), - GetDefaultRunSettings(), + GetRunSettingsWithCurrentTargetFramework(), new TestPlatformOptions() { CollectMetrics = true }, _runEventHandler); @@ -113,7 +113,7 @@ public void RunTestsWithTelemetryOptedOut(RunnerInfo runnerInfo) _vstestConsoleWrapper.RunTests( GetTestAssemblies(), - GetDefaultRunSettings(), + GetRunSettingsWithCurrentTargetFramework(), new TestPlatformOptions() { CollectMetrics = false }, _runEventHandler); @@ -137,11 +137,11 @@ public void RunTestsShouldThrowOnStackOverflowException(RunnerInfo runnerInfo) return; } - var source = new[] { GetAssetFullPath("SimpleTestProject3.dll") }; + var source = new[] { GetTestDll("SimpleTestProject3.dll") }; _vstestConsoleWrapper.RunTests( source, - GetDefaultRunSettings(), + GetRunSettingsWithCurrentTargetFramework(), new TestPlatformOptions() { TestCaseFilter = "ExitWithStackoverFlow" }, _runEventHandler); @@ -162,7 +162,7 @@ public void RunTestsShouldShowProperWarningOnNoTestsForTestCaseFilter(RunnerInfo Setup(); var testAssemblyName = "SimpleTestProject2.dll"; - var source = new List() { GetAssetFullPath(testAssemblyName) }; + var source = new List() { GetTestDll(testAssemblyName) }; var veryLongTestCaseFilter = "FullyQualifiedName=VeryLongTestCaseNameeeeeeeeeeeeee" + @@ -173,7 +173,7 @@ public void RunTestsShouldShowProperWarningOnNoTestsForTestCaseFilter(RunnerInfo _vstestConsoleWrapper.RunTests( source, - GetDefaultRunSettings(), + GetRunSettingsWithCurrentTargetFramework(), new TestPlatformOptions() { TestCaseFilter = veryLongTestCaseFilter }, _runEventHandler); @@ -190,8 +190,8 @@ private IList GetTestAssemblies() { return new List { - GetAssetFullPath("SimpleTestProject.dll"), - GetAssetFullPath("SimpleTestProject2.dll") + GetTestDll("SimpleTestProject.dll"), + GetTestDll("SimpleTestProject2.dll") }; } } diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/RunTestsWithDifferentConfigurationTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/RunTestsWithDifferentConfigurationTests.cs index 4f6e641fb2..de916c635d 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/RunTestsWithDifferentConfigurationTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/RunTestsWithDifferentConfigurationTests.cs @@ -56,7 +56,7 @@ public void RunTestsWithTestAdapterPath(RunnerInfo runnerInfo) _vstestConsoleWrapper.RunTests( GetTestAssemblies(), - GetDefaultRunSettings(), + GetRunSettingsWithCurrentTargetFramework(), _runEventHandler); // Assert @@ -115,7 +115,7 @@ public void RunTestsWithTestSettings(RunnerInfo runnerInfo) var runSettings = $"{FrameworkArgValue}" + testsettingsFile + ""; var sources = new List { - GetAssetFullPath("MstestV1UnitTestProject.dll") + GetTestDll("MstestV1UnitTestProject.dll") }; _vstestConsoleWrapper.RunTests( @@ -140,7 +140,7 @@ public void RunTestsWithX64Source(RunnerInfo runnerInfo) var sources = new List { - GetAssetFullPath("SimpleTestProject3.dll") + GetTestDll("SimpleTestProject3.dll") }; @@ -149,7 +149,7 @@ public void RunTestsWithX64Source(RunnerInfo runnerInfo) _vstestConsoleWrapper.RunTests( sources, - GetDefaultRunSettings(), + GetRunSettingsWithCurrentTargetFramework(), new TestPlatformOptions() { TestCaseFilter = "FullyQualifiedName = SampleUnitTestProject3.UnitTest1.WorkingDirectoryTest" }, _runEventHandler); @@ -163,8 +163,8 @@ private IList GetTestAssemblies() { var testAssemblies = new List { - GetAssetFullPath("SimpleTestProject.dll"), - GetAssetFullPath("SimpleTestProject2.dll") + GetTestDll("SimpleTestProject.dll"), + GetTestDll("SimpleTestProject2.dll") }; return testAssemblies; diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/RunTestsWithFilterTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/RunTestsWithFilterTests.cs index ea4c38ae0f..bfc394614c 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/RunTestsWithFilterTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/RunTestsWithFilterTests.cs @@ -44,11 +44,11 @@ public void RunTestsWithTestCaseFilter(RunnerInfo runnerInfo) _runEventHandler = new RunEventHandler(); var vstestConsoleWrapper = GetVsTestConsoleWrapper(); - var sources = new List { GetAssetFullPath("SimpleTestProject.dll") }; + var sources = new List { GetTestDll("SimpleTestProject.dll") }; vstestConsoleWrapper.RunTests( sources, - GetDefaultRunSettings(), + GetRunSettingsWithCurrentTargetFramework(), new TestPlatformOptions() { TestCaseFilter = "FullyQualifiedName=SampleUnitTestProject.UnitTest1.PassingTest" }, _runEventHandler); @@ -65,11 +65,11 @@ public void RunTestsWithFastFilter(RunnerInfo runnerInfo) SetTestEnvironment(_testEnvironment, runnerInfo); Setup(); - var sources = new List { GetAssetFullPath("SimpleTestProject.dll") }; + var sources = new List { GetTestDll("SimpleTestProject.dll") }; _vstestConsoleWrapper.RunTests( sources, - GetDefaultRunSettings(), + GetRunSettingsWithCurrentTargetFramework(), new TestPlatformOptions() { TestCaseFilter = "FullyQualifiedName=SampleUnitTestProject.UnitTest1.PassingTest | FullyQualifiedName=SampleUnitTestProject.UnitTest1.FailingTest" }, _runEventHandler); diff --git a/test/Microsoft.TestPlatform.ObjectModel.PlatformTests/DiaSessionTests.cs b/test/Microsoft.TestPlatform.ObjectModel.PlatformTests/DiaSessionTests.cs index 4925635fe3..c7880824e9 100644 --- a/test/Microsoft.TestPlatform.ObjectModel.PlatformTests/DiaSessionTests.cs +++ b/test/Microsoft.TestPlatform.ObjectModel.PlatformTests/DiaSessionTests.cs @@ -34,7 +34,7 @@ public static string GetAndSetTargetFrameWork(IntegrationTestEnvironment testEnv public void GetNavigationDataShouldReturnCorrectFileNameAndLineNumber() { var currentTargetFrameWork = GetAndSetTargetFrameWork(_testEnvironment); - var assemblyPath = GetAssetFullPath("SimpleClassLibrary.dll"); + var assemblyPath = GetTestDll("SimpleClassLibrary.dll"); var diaSession = new DiaSession(assemblyPath); DiaNavigationData diaNavigationData = diaSession.GetNavigationData("SimpleClassLibrary.Class1", "PassingTest"); @@ -52,7 +52,7 @@ public void GetNavigationDataShouldReturnCorrectFileNameAndLineNumber() public void GetNavigationDataShouldReturnCorrectDataForAsyncMethod() { var currentTargetFrameWork = GetAndSetTargetFrameWork(_testEnvironment); - var assemblyPath = GetAssetFullPath("SimpleClassLibrary.dll"); + var assemblyPath = GetTestDll("SimpleClassLibrary.dll"); var diaSession = new DiaSession(assemblyPath); DiaNavigationData diaNavigationData = diaSession.GetNavigationData("SimpleClassLibrary.Class1+d__1", "MoveNext"); @@ -70,7 +70,7 @@ public void GetNavigationDataShouldReturnCorrectDataForAsyncMethod() public void GetNavigationDataShouldReturnCorrectDataForOverLoadedMethod() { var currentTargetFrameWork = GetAndSetTargetFrameWork(_testEnvironment); - var assemblyPath = GetAssetFullPath("SimpleClassLibrary.dll"); + var assemblyPath = GetTestDll("SimpleClassLibrary.dll"); var diaSession = new DiaSession(assemblyPath); DiaNavigationData diaNavigationData = diaSession.GetNavigationData("SimpleClassLibrary.Class1", "OverLoadedMethod"); @@ -89,7 +89,7 @@ public void GetNavigationDataShouldReturnCorrectDataForOverLoadedMethod() public void GetNavigationDataShouldReturnNullForNotExistMethodNameOrNotExistTypeName() { var currentTargetFrameWork = GetAndSetTargetFrameWork(_testEnvironment); - var assemblyPath = GetAssetFullPath("SimpleClassLibrary.dll"); + var assemblyPath = GetTestDll("SimpleClassLibrary.dll"); var diaSession = new DiaSession(assemblyPath); @@ -108,7 +108,7 @@ public void GetNavigationDataShouldReturnNullForNotExistMethodNameOrNotExistType public void DiaSessionPerfTest() { var currentTargetFrameWork = GetAndSetTargetFrameWork(_testEnvironment); - var assemblyPath = GetAssetFullPath("SimpleClassLibrary.dll"); + var assemblyPath = GetTestDll("SimpleClassLibrary.dll"); var watch = Stopwatch.StartNew(); var diaSession = new DiaSession(assemblyPath); diff --git a/test/Microsoft.TestPlatform.PerformanceTests/PerformanceTests.cs b/test/Microsoft.TestPlatform.PerformanceTests/PerformanceTests.cs index 7db4ae1e30..f87afc7f58 100644 --- a/test/Microsoft.TestPlatform.PerformanceTests/PerformanceTests.cs +++ b/test/Microsoft.TestPlatform.PerformanceTests/PerformanceTests.cs @@ -18,7 +18,7 @@ public class PerformanceTests : PerformanceTestBase [TestMethod] public void ExecutionPerformanceTest() { - RunExecutionPerformanceTests(GetSampleTestAssembly(), GetTestAdapterPath(), string.Empty); + RunExecutionPerformanceTests(GetSampleTestDll(), GetTestAdapterPath(), string.Empty); ValidateSummaryStatus(1, 1, 1); ValidatePassedTests("SampleUnitTestProject.UnitTest1.PassingTest"); @@ -35,7 +35,7 @@ public void ExecutionPerformanceTest() [TestMethod] public void DiscoveryPerformanceTest() { - RunDiscoveryPerformanceTests(GetSampleTestAssembly(), GetTestAdapterPath(), string.Empty); + RunDiscoveryPerformanceTests(GetSampleTestDll(), GetTestAdapterPath(), string.Empty); ValidateDiscoveredTests( "SampleUnitTestProject.UnitTest1.PassingTest", @@ -52,7 +52,7 @@ public void DiscoveryPerformanceTest() [TestMethod] public void VsTestConsolePerformanceTest() { - RunExecutionPerformanceTests(GetSampleTestAssembly(), GetTestAdapterPath(), string.Empty); + RunExecutionPerformanceTests(GetSampleTestDll(), GetTestAdapterPath(), string.Empty); ValidateSummaryStatus(1, 1, 1); ValidatePassedTests("SampleUnitTestProject.UnitTest1.PassingTest"); @@ -69,7 +69,7 @@ public void VsTestConsolePerformanceTest() [TestMethod] public void TestHostPerformanceTest() { - RunExecutionPerformanceTests(GetSampleTestAssembly(), GetTestAdapterPath(), string.Empty); + RunExecutionPerformanceTests(GetSampleTestDll(), GetTestAdapterPath(), string.Empty); ValidateSummaryStatus(1, 1, 1); ValidatePassedTests("SampleUnitTestProject.UnitTest1.PassingTest"); @@ -86,7 +86,7 @@ public void TestHostPerformanceTest() [TestMethod] public void MsTestV2AdapterPerformanceTest() { - RunExecutionPerformanceTests(GetSampleTestAssembly(), GetTestAdapterPath(), string.Empty); + RunExecutionPerformanceTests(GetSampleTestDll(), GetTestAdapterPath(), string.Empty); ValidateSummaryStatus(1, 1, 1); ValidatePassedTests("SampleUnitTestProject.UnitTest1.PassingTest"); diff --git a/test/Microsoft.TestPlatform.SmokeTests/DiscoveryTests.cs b/test/Microsoft.TestPlatform.SmokeTests/DiscoveryTests.cs index c74c041202..83a3a2b71d 100644 --- a/test/Microsoft.TestPlatform.SmokeTests/DiscoveryTests.cs +++ b/test/Microsoft.TestPlatform.SmokeTests/DiscoveryTests.cs @@ -15,7 +15,7 @@ public class DiscoveryTests : IntegrationTestBase [TestMethod] public void DiscoverAllTests() { - InvokeVsTestForDiscovery(GetSampleTestAssembly(), GetTestAdapterPath(), string.Empty, ".NETFramework,Version=v4.5.1"); + InvokeVsTestForDiscovery(GetSampleTestDll(), GetTestAdapterPath(), string.Empty, ".NETFramework,Version=v4.5.1"); var listOfTests = new string[] { "SampleUnitTestProject.UnitTest1.PassingTest", "SampleUnitTestProject.UnitTest1.FailingTest", "SampleUnitTestProject.UnitTest1.SkippingTest" }; ValidateDiscoveredTests(listOfTests); } diff --git a/test/Microsoft.TestPlatform.SmokeTests/ExecutionTests.cs b/test/Microsoft.TestPlatform.SmokeTests/ExecutionTests.cs index a12a33d98b..b74d83b531 100644 --- a/test/Microsoft.TestPlatform.SmokeTests/ExecutionTests.cs +++ b/test/Microsoft.TestPlatform.SmokeTests/ExecutionTests.cs @@ -15,7 +15,7 @@ public class ExecutionTests : IntegrationTestBase [TestMethod] public void RunAllTestExecution() { - InvokeVsTestForExecution(GetSampleTestAssembly(), GetTestAdapterPath(), ".NETFramework,Version=v4.5.1"); + InvokeVsTestForExecution(GetSampleTestDll(), GetTestAdapterPath(), ".NETFramework,Version=v4.5.1"); ValidateSummaryStatus(1, 1, 1); ValidatePassedTests("SampleUnitTestProject.UnitTest1.PassingTest"); ValidateFailedTests("SampleUnitTestProject.UnitTest1.FailingTest"); @@ -26,7 +26,7 @@ public void RunAllTestExecution() public void RunSelectedTests() { using var resultsDir = new TempDirectory(); - var arguments = PrepareArguments(GetSampleTestAssembly(), GetTestAdapterPath(), string.Empty, ".NETFramework,Version=v4.5.1", resultsDirectory: resultsDir.Path); + var arguments = PrepareArguments(GetSampleTestDll(), GetTestAdapterPath(), string.Empty, ".NETFramework,Version=v4.5.1", resultsDirectory: resultsDir.Path); arguments = string.Concat(arguments, " /Tests:PassingTest"); InvokeVsTest(arguments); ValidateSummaryStatus(1, 0, 0); diff --git a/test/Microsoft.TestPlatform.TestUtilities/DebugInfo.cs b/test/Microsoft.TestPlatform.TestUtilities/DebugInfo.cs index 3869ccb2ea..995855ca1d 100644 --- a/test/Microsoft.TestPlatform.TestUtilities/DebugInfo.cs +++ b/test/Microsoft.TestPlatform.TestUtilities/DebugInfo.cs @@ -6,10 +6,13 @@ namespace Microsoft.TestPlatform.TestUtilities; [Serializable] -public class DebugInfo +// For data source to serialize correctly to enable splitting testcases to one per test in VS, +// this must be serializable. This is sealed because the exact type must be used, not any child type. +// Otherwise it works, but silently does not split the test cases anymore. +public sealed class DebugInfo { public bool DebugVSTestConsole { get; set; } - public bool DebugTesthost { get; set; } + public bool DebugTestHost { get; set; } public bool DebugDataCollector { get; set; } public bool NoDefaultBreakpoints { get; set; } = true; } diff --git a/test/Microsoft.TestPlatform.TestUtilities/DllInfo.cs b/test/Microsoft.TestPlatform.TestUtilities/DllInfo.cs index 8299661238..b5d0590b65 100644 --- a/test/Microsoft.TestPlatform.TestUtilities/DllInfo.cs +++ b/test/Microsoft.TestPlatform.TestUtilities/DllInfo.cs @@ -5,25 +5,18 @@ namespace Microsoft.TestPlatform.TestUtilities; +// For data source to serialize correctly to enable splitting testcases to one per test in VS, +// this must be serializable. This is NOT sealed because we need this for adapters and testSdk. +// But be aware that the exact type must be used, not any child type for the data on data source object (RunnerInfo). +// Otherwise it works, but silently does not split the test cases anymore. [Serializable] -public abstract class DllInfo +public class DllInfo { - protected DllInfo(string name, string propertyName, string versionType, string? version, string path) - { - Name = name; - PropertyName = propertyName; - VersionType = versionType; - // Version can be null when we fail to find the respective propertin in TestPlatform.Dependencies.props - // when that happens we throw when we try to update the path. - Version = version; - Path = path; - } - - public string Name { get; } - public string PropertyName { get; } - public string VersionType { get; } - public string? Version { get; } - public string Path { get; } + public string? Name { get; set; } + public string? PropertyName { get; set; } + public string? VersionType { get; set; } + public string? Version { get; set; } + public string? Path { get; set; } public override string ToString() => $" {Name} = {Version} [{VersionType}]"; diff --git a/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestBase.cs b/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestBase.cs index dbafe748fb..16d5d9e0bc 100644 --- a/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestBase.cs +++ b/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestBase.cs @@ -224,7 +224,7 @@ public void InvokeVsTestForExecution(string testAssembly, { var arguments = PrepareArguments(testAssembly, testAdapterPath, runSettings, framework, _testEnvironment.InIsolationValue, resultsDirectory: TempDirectory.Path); - if (_testEnvironment.DebugInfo.DebugVSTestConsole || _testEnvironment.DebugInfo.DebugTesthost || _testEnvironment.DebugInfo.DebugDataCollector) + if (_testEnvironment.DebugInfo.DebugVSTestConsole || _testEnvironment.DebugInfo.DebugTestHost || _testEnvironment.DebugInfo.DebugDataCollector) { environmentVariables ??= new Dictionary(); @@ -233,7 +233,7 @@ public void InvokeVsTestForExecution(string testAssembly, environmentVariables.Add("VSTEST_RUNNER_DEBUG_ATTACHVS", "1"); } - if (_testEnvironment.DebugInfo.DebugTesthost) + if (_testEnvironment.DebugInfo.DebugTestHost) { environmentVariables.Add("VSTEST_HOST_DEBUG_ATTACHVS", "1"); } @@ -522,27 +522,27 @@ public void ValidateFullyQualifiedDiscoveredTests(string filePath, params string } } - protected string GetSampleTestAssembly() + protected string GetSampleTestDll() { - return GetAssetFullPath("SimpleTestProject.dll"); + return GetTestDll("SimpleTestProject.dll"); } - protected string GetAssetFullPath(string assetName) + protected string GetTestDll(string assetName) { return _testEnvironment.GetTestAsset(assetName); } - protected string GetAssetFullPath(string assetName, string targetFramework) + protected string GetTestDllForFramework(string assetName, string targetFramework) { return _testEnvironment.GetTestAsset(assetName, targetFramework); } - protected List GetAssetFullPath(DllInfo dllInfo, params string[] assetNames) + protected List GetTestDlls(params string[] assetNames) { var assets = new List(); foreach (var assetName in assetNames) { - assets.Add(dllInfo.UpdatePath(GetAssetFullPath(assetName))); + assets.Add(GetTestDll(assetName)); } return assets; @@ -674,7 +674,7 @@ public IVsTestConsoleWrapper GetVsTestConsoleWrapper() Console.WriteLine($"Console runner path: {consoleRunnerPath}"); VsTestConsoleWrapper vstestConsoleWrapper; - if (_testEnvironment.DebugInfo.DebugVSTestConsole || _testEnvironment.DebugInfo.DebugTesthost || _testEnvironment.DebugInfo.DebugDataCollector) + if (_testEnvironment.DebugInfo.DebugVSTestConsole || _testEnvironment.DebugInfo.DebugTestHost || _testEnvironment.DebugInfo.DebugDataCollector) { var environmentVariables = new Dictionary(); Environment.GetEnvironmentVariables().OfType().ToList().ForEach(e => environmentVariables.Add(e.Key.ToString(), e.Value.ToString())); @@ -684,7 +684,7 @@ public IVsTestConsoleWrapper GetVsTestConsoleWrapper() environmentVariables.Add("VSTEST_RUNNER_DEBUG_ATTACHVS", "1"); } - if (_testEnvironment.DebugInfo.DebugTesthost) + if (_testEnvironment.DebugInfo.DebugTestHost) { environmentVariables.Add("VSTEST_HOST_DEBUG_ATTACHVS", "1"); } @@ -906,24 +906,7 @@ public static void CreateRunSettingsFile(string destinationRunsettingsPath, stri protected string BuildMultipleAssemblyPath(params string[] assetNames) { - var assetFullPaths = new string[assetNames.Length]; - for (var i = 0; i < assetNames.Length; i++) - { - var path = GetAssetFullPath(assetNames[i]); - if (_testEnvironment.DllInfos.Count > 0) - { - foreach (var dllInfo in _testEnvironment.DllInfos) - { - path = dllInfo.UpdatePath(path); - } - - Assert.IsTrue(File.Exists(path), "GetTestAsset: Path not found: \"{0}\". Most likely you need to build using build.cmd -s PrepareAcceptanceTests.", path); - } - - assetFullPaths[i] = path.AddDoubleQuote(); - } - - return string.Join(" ", assetFullPaths); + return string.Join(" ", GetTestDlls(assetNames)); } protected static string GetDiagArg(string rootDir) diff --git a/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestEnvironment.cs b/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestEnvironment.cs index b0af339915..01c3069ee1 100644 --- a/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestEnvironment.cs +++ b/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestEnvironment.cs @@ -4,7 +4,6 @@ using System; using System.Collections.Generic; using System.IO; -using System.Linq.Expressions; using System.Xml; using Microsoft.VisualStudio.TestTools.UnitTesting; @@ -196,7 +195,7 @@ public string RunnerFramework public static bool IsCI { get; } = Environment.GetEnvironmentVariable("TF_BUILD") == "True"; public DebugInfo DebugInfo { get; set; } public VSTestConsoleInfo VSTestConsoleInfo { get; set; } - public List DllInfos { get; set; } + public List DllInfos { get; set; } = new(); /// /// Gets the full path to a test asset. @@ -239,6 +238,15 @@ public string GetTestAsset(string assetName, string targetFramework) targetFramework, assetName); + // Update the path to be taken from the compatibility matrix instead of from the root folder. + if (DllInfos.Count > 0) + { + foreach (var dllInfo in DllInfos) + { + assetPath = dllInfo.UpdatePath(assetPath); + } + } + Assert.IsTrue(File.Exists(assetPath), "GetTestAsset: Path not found: \"{0}\". Most likely you need to build using build.cmd -s PrepareAcceptanceTests.", assetPath); // If you are thinking about wrapping the path in double quotes here, diff --git a/test/Microsoft.TestPlatform.TestUtilities/MSTestInfo.cs b/test/Microsoft.TestPlatform.TestUtilities/MSTestInfo.cs deleted file mode 100644 index 6529493ab0..0000000000 --- a/test/Microsoft.TestPlatform.TestUtilities/MSTestInfo.cs +++ /dev/null @@ -1,15 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT license. See LICENSE file in the project root for full license information. - -using System; - -namespace Microsoft.TestPlatform.TestUtilities; - -[Serializable] -public class MSTestInfo : DllInfo -{ - public MSTestInfo(string versionType, string? version, string path) - : base(name: "MSTest", propertyName: "MSTest", versionType, version, path) - { - } -} diff --git a/test/Microsoft.TestPlatform.TestUtilities/TesthostInfo.cs b/test/Microsoft.TestPlatform.TestUtilities/NetTestSdkInfo.cs similarity index 64% rename from test/Microsoft.TestPlatform.TestUtilities/TesthostInfo.cs rename to test/Microsoft.TestPlatform.TestUtilities/NetTestSdkInfo.cs index a9be09a3d8..c16d2a3f44 100644 --- a/test/Microsoft.TestPlatform.TestUtilities/TesthostInfo.cs +++ b/test/Microsoft.TestPlatform.TestUtilities/NetTestSdkInfo.cs @@ -8,8 +8,9 @@ namespace Microsoft.TestPlatform.TestUtilities; [Serializable] public class NetTestSdkInfo : DllInfo { - public NetTestSdkInfo(string versionType, string? version, string path) - : base(name: "Testhost", propertyName: "VSTestConsole", versionType, version, path) + public NetTestSdkInfo() { + Name = "Testhost"; + PropertyName = "VSTestConsole"; } } diff --git a/test/Microsoft.TestPlatform.TestUtilities/VSTestConsoleInfo.cs b/test/Microsoft.TestPlatform.TestUtilities/VSTestConsoleInfo.cs index 41d441e3b8..e55503afbf 100644 --- a/test/Microsoft.TestPlatform.TestUtilities/VSTestConsoleInfo.cs +++ b/test/Microsoft.TestPlatform.TestUtilities/VSTestConsoleInfo.cs @@ -8,18 +8,9 @@ namespace Microsoft.TestPlatform.TestUtilities; [Serializable] public class VSTestConsoleInfo { - public VSTestConsoleInfo(string versionType, string? version, string path) - { - VersionType = versionType; - // Version can be null when we fail to find the respective propertin in TestPlatform.Dependencies.props - // when that happens we throw when we try to update the path. - Version = version; - Path = path; - } - - public string VersionType { get; } - public string? Version { get; } - public string Path { get; } + public string? VersionType { get; set; } + public string? Version { get; set; } + public string? Path { get; set; } public override string ToString() => $" vstest.console = {Version} [{VersionType}]"; } From 1a45042ba7ea4a0b6b5e0de81f79145eb8cf9dff Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Jare=C5=A1?= Date: Mon, 21 Mar 2022 16:19:17 +0100 Subject: [PATCH 057/112] merge --- .../TranslationLayerTests/DiscoverTests.cs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/DiscoverTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/DiscoverTests.cs index 00a8805769..e7db8258ce 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/DiscoverTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/DiscoverTests.cs @@ -208,8 +208,8 @@ public async Task CancelTestDiscovery(RunnerInfo runnerInfo) // Setup var testAssemblies = new List { - GetAssetFullPath("DiscoveryTestProject.dll"), - GetAssetFullPath("SimpleTestProject.dll"), + GetTestDll("DiscoveryTestProject.dll"), + GetTestDll("SimpleTestProject.dll"), }; SetTestEnvironment(_testEnvironment, runnerInfo); @@ -235,7 +235,7 @@ public async Task CancelTestDiscovery(RunnerInfo runnerInfo) }); // Act - await Task.Run(() => _vstestConsoleWrapper.DiscoverTests(testAssemblies, GetDefaultRunSettings(), discoveryEvents.Object)); + await Task.Run(() => _vstestConsoleWrapper.DiscoverTests(testAssemblies, GetRunSettingsWithCurrentTargetFramework(), discoveryEvents.Object)); // Assert. Assert.IsTrue(isTestCancelled); From 6e841581ee31ec6b505ebd8b1edc19b14d88c97f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Jare=C5=A1?= Date: Mon, 21 Mar 2022 16:31:42 +0100 Subject: [PATCH 058/112] Revert unnecessary renames --- .../AcceptanceTestBase.cs | 2 +- .../AppDomainTests.cs | 2 +- .../ArgumentProcessorTests.cs | 2 +- .../BlameDataCollectorTests.cs | 14 +++++------ .../CodeCoverageTests.cs | 2 +- .../DataCollectionTests.cs | 2 +- .../DeprecateExtensionsPathWarningTests.cs | 2 +- .../DifferentTestFrameworkSimpleTests.cs | 2 +- .../DiscoveryTests.cs | 6 ++--- .../FilePatternParserTests.cs | 6 ++--- .../FrameworkTests.cs | 8 +++---- .../LoggerTests.cs | 24 +++++++++---------- .../PlatformTests.cs | 2 +- .../PortableNugetPackageTests.cs | 2 +- .../ResultsDirectoryTests.cs | 4 ++-- .../RunsettingsTests.cs | 16 ++++++------- .../TelemetryTests.cs | 4 ++-- .../TestCaseFilterTests.cs | 14 +++++------ .../CodeCoverageTests.cs | 2 +- .../CustomTestHostTests.cs | 8 +++---- .../DataCollectorAttachmentProcessor.cs | 2 +- .../DifferentTestFrameworkSimpleTests.cs | 8 +++---- .../TranslationLayerTests/DiscoverTests.cs | 20 ++++++++-------- .../LiveUnitTestingTests.cs | 4 ++-- .../TranslationLayerTests/RunSelectedTests.cs | 12 +++++----- .../TranslationLayerTests/RunTests.cs | 20 ++++++++-------- ...RunTestsWithDifferentConfigurationTests.cs | 12 +++++----- .../RunTestsWithFilterTests.cs | 8 +++---- .../DiaSessionTests.cs | 10 ++++---- .../PerformanceTests.cs | 10 ++++---- .../DiscoveryTests.cs | 2 +- .../ExecutionTests.cs | 4 ++-- .../IntegrationTestBase.cs | 8 +++---- 33 files changed, 122 insertions(+), 122 deletions(-) diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/AcceptanceTestBase.cs b/test/Microsoft.TestPlatform.AcceptanceTests/AcceptanceTestBase.cs index bd7fe5afcd..934b7de8e6 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/AcceptanceTestBase.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/AcceptanceTestBase.cs @@ -124,7 +124,7 @@ public string GetEmptyRunsettings() /// /// Almost empty runsettings, just specifying the target framework from the currently set test environment. /// - public string GetRunSettingsWithCurrentTargetFramework() + public string GetDefaultRunSettings() { return GetRunSettingsWithTargetFramework(FrameworkArgValue); } diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/AppDomainTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/AppDomainTests.cs index a413e8d7d6..17e8f88423 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/AppDomainTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/AppDomainTests.cs @@ -38,7 +38,7 @@ public void RunTestExecutionWithDisableAppDomain(RunnerInfo runnerInfo) var runsettingsFilePath = GetInProcDataCollectionRunsettingsFile(true, TempDirectory); var arguments = PrepareArguments( - GetSampleTestDll(), + GetSampleTestAssembly(), GetTestAdapterPath(), runsettingsFilePath, FrameworkArgValue, diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/ArgumentProcessorTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/ArgumentProcessorTests.cs index 8b9dd95d40..44a4a39dbd 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/ArgumentProcessorTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/ArgumentProcessorTests.cs @@ -40,7 +40,7 @@ public void PassingInvalidArgumentsToVsTestConsoleShouldNotPrintHelpMessage(Runn { SetTestEnvironment(_testEnvironment, runnerInfo); - var arguments = PrepareArguments(GetSampleTestDll(), GetTestAdapterPath(), string.Empty, FrameworkArgValue, resultsDirectory: TempDirectory.Path); + var arguments = PrepareArguments(GetSampleTestAssembly(), GetTestAdapterPath(), string.Empty, FrameworkArgValue, resultsDirectory: TempDirectory.Path); arguments = string.Concat(arguments, " /badArgument"); InvokeVsTest(arguments); diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/BlameDataCollectorTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/BlameDataCollectorTests.cs index 8df3e8bc6a..e9e6af25ee 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/BlameDataCollectorTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/BlameDataCollectorTests.cs @@ -46,7 +46,7 @@ public BlameDataCollectorTests() public void BlameDataCollectorShouldGiveCorrectTestCaseName(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - var assemblyPaths = GetTestDll("BlameUnitTestProject.dll"); + var assemblyPaths = GetAssetFullPath("BlameUnitTestProject.dll"); var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue); arguments = string.Concat(arguments, $" /Blame"); arguments = string.Concat(arguments, $" /ResultsDirectory:{TempDirectory.Path}"); @@ -137,7 +137,7 @@ public void BlameDataCollectorShouldOutputDumpFileWhenNoCrashOccursButCollectAlw public void HangDumpOnTimeout(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - var assemblyPaths = GetTestDll("timeout.dll"); + var assemblyPaths = GetAssetFullPath("timeout.dll"); var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, string.Empty, runnerInfo.InIsolationValue); arguments = string.Concat(arguments, $@" /Blame:""CollectHangDump;HangDumpType=full;TestTimeout=3s"""); @@ -160,7 +160,7 @@ public void HangDumpOnTimeout(RunnerInfo runnerInfo) public void CrashDumpWhenThereIsNoTimeout(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - var assemblyPaths = GetTestDll("timeout.dll"); + var assemblyPaths = GetAssetFullPath("timeout.dll"); var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, string.Empty, runnerInfo.InIsolationValue); arguments = string.Concat(arguments, $@" /Blame:""CollectDump;DumpType=full;CollectAlways=true;CollectHangDump"""); @@ -183,7 +183,7 @@ public void CrashDumpWhenThereIsNoTimeout(RunnerInfo runnerInfo) public void CrashDumpOnExit(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - var assemblyPaths = GetTestDll("timeout.dll"); + var assemblyPaths = GetAssetFullPath("timeout.dll"); var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, string.Empty, runnerInfo.InIsolationValue); arguments = string.Concat(arguments, $@" /Blame:""CollectDump;DumpType=full;CollectAlways=true"""); @@ -204,7 +204,7 @@ public void CrashDumpOnExit(RunnerInfo runnerInfo) public void CrashDumpOnStackOverflow(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - var assemblyPaths = GetTestDll("crash.dll"); + var assemblyPaths = GetAssetFullPath("crash.dll"); var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, string.Empty, runnerInfo.InIsolationValue); arguments = string.Concat(arguments, $@" /Blame:""CollectDump;DumpType=full"""); @@ -225,7 +225,7 @@ public void CrashDumpOnStackOverflow(RunnerInfo runnerInfo) public void CrashDumpChildProcesses(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - var assemblyPaths = GetTestDll("child-crash.dll"); + var assemblyPaths = GetAssetFullPath("child-crash.dll"); var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, string.Empty, runnerInfo.InIsolationValue); arguments = string.Concat(arguments, $@" /Blame:""CollectDump;DumpType=full"""); InvokeVsTest(arguments); @@ -240,7 +240,7 @@ public void CrashDumpChildProcesses(RunnerInfo runnerInfo) public void HangDumpChildProcesses(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - var assemblyPaths = GetTestDll("child-hang.dll"); + var assemblyPaths = GetAssetFullPath("child-hang.dll"); var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, string.Empty, runnerInfo.InIsolationValue); arguments = string.Concat(arguments, $@" /Blame:""CollectHangDump;HangDumpType=full;TestTimeout=15s"""); InvokeVsTest(arguments); diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/CodeCoverageTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/CodeCoverageTests.cs index c7901f2326..b426fc7fd1 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/CodeCoverageTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/CodeCoverageTests.cs @@ -251,7 +251,7 @@ private string CreateArguments( TestParameters testParameters, out string trxFilePath) { - var assemblyPaths = GetTestDll(testParameters.AssemblyName); + var assemblyPaths = GetAssetFullPath(testParameters.AssemblyName); string traceDataCollectorDir = Path.Combine(IntegrationTestEnvironment.TestPlatformRootDirectory, "artifacts", IntegrationTestEnvironment.BuildConfiguration, "Microsoft.CodeCoverage"); diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/DataCollectionTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/DataCollectionTests.cs index 1308f4345a..a600a7bf0c 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/DataCollectionTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/DataCollectionTests.cs @@ -100,7 +100,7 @@ public void DataCollectorAssemblyLoadingShouldNotThrowErrorForFullFramework(Runn { SetTestEnvironment(_testEnvironment, runnerInfo); - var arguments = PrepareArguments(GetTestDll("AppDomainGetAssembliesTestProject.dll"), string.Empty, string.Empty, FrameworkArgValue, resultsDirectory: TempDirectory.Path); + var arguments = PrepareArguments(GetAssetFullPath("AppDomainGetAssembliesTestProject.dll"), string.Empty, string.Empty, FrameworkArgValue, resultsDirectory: TempDirectory.Path); InvokeVsTest(arguments); ValidateSummaryStatus(1, 0, 0); diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/DeprecateExtensionsPathWarningTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/DeprecateExtensionsPathWarningTests.cs index d33857347a..4c10549775 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/DeprecateExtensionsPathWarningTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/DeprecateExtensionsPathWarningTests.cs @@ -61,7 +61,7 @@ public void CopyAdapterToExtensions() [TestMethod] public void VerifyDeprecatedWarningIsThrownWhenAdaptersPickedFromExtensionDirectory() { - var arguments = PrepareArguments(GetSampleTestDll(), null, null, FrameworkArgValue, resultsDirectory: TempDirectory.Path); + var arguments = PrepareArguments(GetSampleTestAssembly(), null, null, FrameworkArgValue, resultsDirectory: TempDirectory.Path); InvokeVsTest(arguments); StdOutputContains("Adapter lookup is being changed, please follow"); diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/DifferentTestFrameworkSimpleTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/DifferentTestFrameworkSimpleTests.cs index 10d59d084b..70de29e59c 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/DifferentTestFrameworkSimpleTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/DifferentTestFrameworkSimpleTests.cs @@ -155,7 +155,7 @@ public void NUnitRunAllTestExecution(RunnerInfo runnerInfo) SetTestEnvironment(_testEnvironment, runnerInfo); var arguments = PrepareArguments( - GetTestDll("NUTestProject.dll"), + GetAssetFullPath("NUTestProject.dll"), GetTestAdapterPath(UnitTestFramework.NUnit), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, TempDirectory.Path); diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/DiscoveryTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/DiscoveryTests.cs index 444cc873ec..fa9bb8627b 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/DiscoveryTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/DiscoveryTests.cs @@ -25,7 +25,7 @@ public void DiscoverAllTests(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - InvokeVsTestForDiscovery(GetSampleTestDll(), GetTestAdapterPath(), string.Empty, FrameworkArgValue); + InvokeVsTestForDiscovery(GetSampleTestAssembly(), GetTestAdapterPath(), string.Empty, FrameworkArgValue); var listOfTests = new[] { "SampleUnitTestProject.UnitTest1.PassingTest", "SampleUnitTestProject.UnitTest1.FailingTest", "SampleUnitTestProject.UnitTest1.SkippingTest" }; ValidateDiscoveredTests(listOfTests); @@ -65,7 +65,7 @@ public void DiscoverFullyQualifiedTests(RunnerInfo runnerInfo) var listOfTests = new[] { "SampleUnitTestProject.UnitTest1.PassingTest", "SampleUnitTestProject.UnitTest1.FailingTest", "SampleUnitTestProject.UnitTest1.SkippingTest" }; - var arguments = PrepareArguments(GetSampleTestDll(), GetTestAdapterPath(), string.Empty, FrameworkArgValue, _testEnvironment.InIsolationValue, resultsDirectory: TempDirectory.Path); + var arguments = PrepareArguments(GetSampleTestAssembly(), GetTestAdapterPath(), string.Empty, FrameworkArgValue, _testEnvironment.InIsolationValue, resultsDirectory: TempDirectory.Path); arguments = string.Concat(arguments, " /ListFullyQualifiedTests", " /ListTestsTargetPath:\"" + dummyFilePath + "\""); InvokeVsTest(arguments); @@ -80,7 +80,7 @@ public void DiscoverTestsShouldShowProperWarningIfNoTestsOnTestCaseFilter(Runner { SetTestEnvironment(_testEnvironment, runnerInfo); - var assetFullPath = GetTestDll("SimpleTestProject2.dll"); + var assetFullPath = GetAssetFullPath("SimpleTestProject2.dll"); var arguments = PrepareArguments(assetFullPath, GetTestAdapterPath(), string.Empty, FrameworkArgValue, _testEnvironment.InIsolationValue, resultsDirectory: TempDirectory.Path); arguments = string.Concat(arguments, " /listtests"); arguments = string.Concat(arguments, " /testcasefilter:NonExistTestCaseName"); diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/FilePatternParserTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/FilePatternParserTests.cs index 4e31163543..3d8f36218b 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/FilePatternParserTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/FilePatternParserTests.cs @@ -20,7 +20,7 @@ public void WildCardPatternShouldCorrectlyWorkOnFiles(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - var testAssembly = GetSampleTestDll(); + var testAssembly = GetSampleTestAssembly(); testAssembly = testAssembly.Replace("SimpleTestProject.dll", "*TestProj*.dll"); var arguments = PrepareArguments( @@ -40,7 +40,7 @@ public void WildCardPatternShouldCorrectlyWorkOnArbitraryDepthDirectories(Runner { SetTestEnvironment(_testEnvironment, runnerInfo); - var testAssembly = GetSampleTestDll(); + var testAssembly = GetSampleTestAssembly(); // Add one more directory to the temp path, so we can substitute it with ** // and copy then whole directory there. @@ -66,7 +66,7 @@ public void WildCardPatternShouldCorrectlyWorkForRelativeAssemblyPath(RunnerInfo { SetTestEnvironment(_testEnvironment, runnerInfo); - var testAssembly = GetSampleTestDll(); + var testAssembly = GetSampleTestAssembly(); testAssembly = testAssembly.Replace("SimpleTestProject.dll", "*TestProj*.dll"); var wildCardIndex = testAssembly.IndexOfAny(new char[] { '*' }); diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/FrameworkTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/FrameworkTests.cs index 9a00c24cb8..394db87a80 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/FrameworkTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/FrameworkTests.cs @@ -19,7 +19,7 @@ public void FrameworkArgumentShouldWork(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - var arguments = PrepareArguments(GetSampleTestDll(), string.Empty, string.Empty, string.Empty, resultsDirectory: TempDirectory.Path); + var arguments = PrepareArguments(GetSampleTestAssembly(), string.Empty, string.Empty, string.Empty, resultsDirectory: TempDirectory.Path); arguments = string.Concat(arguments, " ", $"/Framework:{FrameworkArgValue}"); InvokeVsTest(arguments); @@ -33,7 +33,7 @@ public void FrameworkShortNameArgumentShouldWork(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - var arguments = PrepareArguments(GetSampleTestDll(), string.Empty, string.Empty, string.Empty, resultsDirectory: TempDirectory.Path); + var arguments = PrepareArguments(GetSampleTestAssembly(), string.Empty, string.Empty, string.Empty, resultsDirectory: TempDirectory.Path); arguments = string.Concat(arguments, " ", $"/Framework:{_testEnvironment.TargetFramework}"); InvokeVsTest(arguments); @@ -49,7 +49,7 @@ public void OnWrongFrameworkPassedTestRunShouldNotRun(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - var arguments = PrepareArguments(GetSampleTestDll(), string.Empty, string.Empty, string.Empty, resultsDirectory: TempDirectory.Path); + var arguments = PrepareArguments(GetSampleTestAssembly(), string.Empty, string.Empty, string.Empty, resultsDirectory: TempDirectory.Path); if (runnerInfo.TargetFramework.Contains("netcore")) { arguments = string.Concat(arguments, " ", "/Framework:Framework45"); @@ -80,7 +80,7 @@ public void RunSpecificTestsShouldWorkWithFrameworkInCompatibleWarning(RunnerInf { SetTestEnvironment(_testEnvironment, runnerInfo); - var arguments = PrepareArguments(GetSampleTestDll(), string.Empty, string.Empty, string.Empty, resultsDirectory: TempDirectory.Path); + var arguments = PrepareArguments(GetSampleTestAssembly(), string.Empty, string.Empty, string.Empty, resultsDirectory: TempDirectory.Path); arguments = string.Concat(arguments, " ", "/tests:PassingTest"); arguments = string.Concat(arguments, " ", "/Framework:Framework40"); diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/LoggerTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/LoggerTests.cs index 8e79cc7589..e9e30d8fdc 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/LoggerTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/LoggerTests.cs @@ -23,12 +23,12 @@ public void TrxLoggerWithFriendlyNameShouldProperlyOverwriteFile(RunnerInfo runn { SetTestEnvironment(_testEnvironment, runnerInfo); - var arguments = PrepareArguments(GetSampleTestDll(), GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, TempDirectory.Path); + var arguments = PrepareArguments(GetSampleTestAssembly(), GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, TempDirectory.Path); var trxFileName = "TestResults.trx"; arguments = string.Concat(arguments, $" /logger:\"trx;LogFileName={trxFileName}\""); InvokeVsTest(arguments); - arguments = PrepareArguments(GetSampleTestDll(), GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, TempDirectory.Path); + arguments = PrepareArguments(GetSampleTestAssembly(), GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, TempDirectory.Path); arguments = string.Concat(arguments, $" /logger:\"trx;LogFileName={trxFileName}\""); arguments = string.Concat(arguments, " /testcasefilter:Name~Pass"); InvokeVsTest(arguments); @@ -44,12 +44,12 @@ public void HtmlLoggerWithFriendlyNameShouldProperlyOverwriteFile(RunnerInfo run { SetTestEnvironment(_testEnvironment, runnerInfo); - var arguments = PrepareArguments(GetSampleTestDll(), GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, TempDirectory.Path); + var arguments = PrepareArguments(GetSampleTestAssembly(), GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, TempDirectory.Path); var htmlFileName = "TestResults.html"; arguments = string.Concat(arguments, $" /logger:\"html;LogFileName={htmlFileName}\""); InvokeVsTest(arguments); - arguments = PrepareArguments(GetSampleTestDll(), GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, TempDirectory.Path); + arguments = PrepareArguments(GetSampleTestAssembly(), GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, TempDirectory.Path); arguments = string.Concat(arguments, $" /logger:\"html;LogFileName={htmlFileName}\""); arguments = string.Concat(arguments, " /testcasefilter:Name~Pass"); InvokeVsTest(arguments); @@ -64,12 +64,12 @@ public void TrxLoggerWithExecutorUriShouldProperlyOverwriteFile(RunnerInfo runne { SetTestEnvironment(_testEnvironment, runnerInfo); - var arguments = PrepareArguments(GetSampleTestDll(), GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, TempDirectory.Path); + var arguments = PrepareArguments(GetSampleTestAssembly(), GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, TempDirectory.Path); var trxFileName = "TestResults.trx"; arguments = string.Concat(arguments, $" /logger:\"logger://Microsoft/TestPlatform/TrxLogger/v1;LogFileName={trxFileName}\""); InvokeVsTest(arguments); - arguments = PrepareArguments(GetSampleTestDll(), GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, TempDirectory.Path); + arguments = PrepareArguments(GetSampleTestAssembly(), GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, TempDirectory.Path); arguments = string.Concat(arguments, $" /logger:\"logger://Microsoft/TestPlatform/TrxLogger/v1;LogFileName={trxFileName}\""); arguments = string.Concat(arguments, " /testcasefilter:Name~Pass"); InvokeVsTest(arguments); @@ -86,11 +86,11 @@ public void TrxLoggerWithLogFilePrefixShouldGenerateMultipleTrx(RunnerInfo runne SetTestEnvironment(_testEnvironment, runnerInfo); var trxFileNamePattern = "TestResults"; - var arguments = PrepareArguments(GetSampleTestDll(), GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, TempDirectory.Path); + var arguments = PrepareArguments(GetSampleTestAssembly(), GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, TempDirectory.Path); arguments = string.Concat(arguments, $" /logger:\"logger://Microsoft/TestPlatform/TrxLogger/v1;LogFilePrefix={trxFileNamePattern}\""); InvokeVsTest(arguments); - arguments = PrepareArguments(GetSampleTestDll(), GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, TempDirectory.Path); + arguments = PrepareArguments(GetSampleTestAssembly(), GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, TempDirectory.Path); arguments = string.Concat(arguments, $" /logger:\"logger://Microsoft/TestPlatform/TrxLogger/v1;LogFilePrefix={trxFileNamePattern}\""); arguments = string.Concat(arguments, " /testcasefilter:Name~Pass"); InvokeVsTest(arguments); @@ -105,12 +105,12 @@ public void HtmlLoggerWithExecutorUriShouldProperlyOverwriteFile(RunnerInfo runn { SetTestEnvironment(_testEnvironment, runnerInfo); - var arguments = PrepareArguments(GetSampleTestDll(), GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, TempDirectory.Path); + var arguments = PrepareArguments(GetSampleTestAssembly(), GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, TempDirectory.Path); var htmlFileName = "TestResults.html"; arguments = string.Concat(arguments, $" /logger:\"logger://Microsoft/TestPlatform/htmlLogger/v1;LogFileName{htmlFileName}\""); InvokeVsTest(arguments); - arguments = PrepareArguments(GetSampleTestDll(), GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, TempDirectory.Path); + arguments = PrepareArguments(GetSampleTestAssembly(), GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, TempDirectory.Path); arguments = string.Concat(arguments, $" /logger:\"logger://Microsoft/TestPlatform/htmlLogger/v1;LogFileName={htmlFileName}\""); arguments = string.Concat(arguments, " /testcasefilter:Name~Pass"); InvokeVsTest(arguments); @@ -126,7 +126,7 @@ public void TrxLoggerResultSummaryOutcomeValueShouldBeFailedIfNoTestsExecutedAnd { SetTestEnvironment(_testEnvironment, runnerInfo); - var arguments = PrepareArguments(GetSampleTestDll(), GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue); + var arguments = PrepareArguments(GetSampleTestAssembly(), GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue); var trxFilePath = Path.Combine(TempDirectory.Path, "TrxLogger.trx"); arguments = string.Concat(arguments, $" /logger:\"trx;LogFileName={trxFilePath}\""); @@ -149,7 +149,7 @@ public void TrxLoggerResultSummaryOutcomeValueShouldNotChangeIfNoTestsExecutedAn { SetTestEnvironment(_testEnvironment, runnerInfo); - var arguments = PrepareArguments(GetSampleTestDll(), GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue); + var arguments = PrepareArguments(GetSampleTestAssembly(), GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue); var trxFilePath = Path.Combine(TempDirectory.Path, "TrxLogger.trx"); arguments = string.Concat(arguments, $" /logger:\"trx;LogFileName={trxFilePath}\""); diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/PlatformTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/PlatformTests.cs index 289363a269..b149fa2994 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/PlatformTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/PlatformTests.cs @@ -45,7 +45,7 @@ private void RunTestExecutionWithPlatform(string platformArg, string testhostPro { var arguments = PrepareArguments( - GetSampleTestDll(), + GetSampleTestAssembly(), GetTestAdapterPath(), string.Empty, FrameworkArgValue, _testEnvironment.InIsolationValue, resultsDirectory: TempDirectory.Path); diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/PortableNugetPackageTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/PortableNugetPackageTests.cs index be24f8ff38..d5f227fccf 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/PortableNugetPackageTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/PortableNugetPackageTests.cs @@ -58,7 +58,7 @@ public void DiscoverAllTests(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - InvokeVsTestForDiscovery(GetSampleTestDll(), GetTestAdapterPath(), string.Empty, FrameworkArgValue); + InvokeVsTestForDiscovery(GetSampleTestAssembly(), GetTestAdapterPath(), string.Empty, FrameworkArgValue); var listOfTests = new[] { "SampleUnitTestProject.UnitTest1.PassingTest", "SampleUnitTestProject.UnitTest1.FailingTest", "SampleUnitTestProject.UnitTest1.SkippingTest" }; ValidateDiscoveredTests(listOfTests); diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/ResultsDirectoryTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/ResultsDirectoryTests.cs index 4b6d60ca87..a7b02265a7 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/ResultsDirectoryTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/ResultsDirectoryTests.cs @@ -20,7 +20,7 @@ public void TrxFileShouldBeCreatedInResultsDirectory(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - var arguments = PrepareArguments(GetSampleTestDll(), GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue); + var arguments = PrepareArguments(GetSampleTestAssembly(), GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue); var trxFileName = "TestResults.trx"; var trxFilePath = Path.Combine(TempDirectory.Path, trxFileName); arguments = string.Concat(arguments, $" /logger:\"trx;LogFileName={trxFileName}\""); @@ -41,7 +41,7 @@ public void ResultsDirectoryRelativePathShouldWork(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - var arguments = PrepareArguments(GetSampleTestDll(), GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue); + var arguments = PrepareArguments(GetSampleTestAssembly(), GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue); var trxFileName = "TestResults.trx"; var relativeDirectory = @"relative\directory"; var resultsDirectory = Path.Combine(Directory.GetCurrentDirectory(), relativeDirectory); diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/RunsettingsTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/RunsettingsTests.cs index d1c8084875..3d5dbcbea7 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/RunsettingsTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/RunsettingsTests.cs @@ -246,7 +246,7 @@ public void RunSettingsWithInvalidValueShouldLogError(RunnerInfo runnerInfo) }; var runsettingsFilePath = GetRunsettingsFilePath(runConfigurationDictionary, TempDirectory); var arguments = PrepareArguments( - GetSampleTestDll(), + GetSampleTestAssembly(), string.Empty, runsettingsFilePath, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: TempDirectory.Path); @@ -267,7 +267,7 @@ public void TestAdapterPathFromRunSettings(RunnerInfo runnerInfo) }; var runsettingsFilePath = GetRunsettingsFilePath(runConfigurationDictionary, TempDirectory); var arguments = PrepareArguments( - GetSampleTestDll(), + GetSampleTestAssembly(), string.Empty, runsettingsFilePath, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: TempDirectory.Path); @@ -284,7 +284,7 @@ public void LegacySettingsWithPlatform(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - var testAssemblyPath = GetTestDll("LegacySettingsUnitTestProject.dll"); + var testAssemblyPath = GetAssetFullPath("LegacySettingsUnitTestProject.dll"); _ = Path.GetDirectoryName(testAssemblyPath); var runsettingsXml = @" @@ -315,7 +315,7 @@ public void LegacySettingsWithScripts(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - var testAssemblyPath = GetTestDll("LegacySettingsUnitTestProject.dll"); + var testAssemblyPath = GetAssetFullPath("LegacySettingsUnitTestProject.dll"); // Create the script files var guid = Guid.NewGuid(); @@ -361,7 +361,7 @@ public void LegacySettingsWithDeploymentItem(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - var testAssemblyPath = GetTestDll("LegacySettingsUnitTestProject.dll"); + var testAssemblyPath = GetAssetFullPath("LegacySettingsUnitTestProject.dll"); var testAssemblyDirectory = Path.GetDirectoryName(testAssemblyPath); var deploymentItem = Path.Combine(testAssemblyDirectory, "Deployment", "DeploymentFile.xml"); @@ -397,7 +397,7 @@ public void LegacySettingsTestTimeout(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - var testAssemblyPath = GetTestDll("LegacySettingsUnitTestProject.dll"); + var testAssemblyPath = GetAssetFullPath("LegacySettingsUnitTestProject.dll"); var runsettingsXml = @" true @@ -424,7 +424,7 @@ public void LegacySettingsAssemblyResolution(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - var testAssemblyPath = GetTestDll("LegacySettingsUnitTestProject.dll"); + var testAssemblyPath = GetAssetFullPath("LegacySettingsUnitTestProject.dll"); var runsettingsFormat = @" true @@ -466,7 +466,7 @@ public void EnvironmentVariablesSettingsShouldSetEnvironmentVariables(RunnerInfo { SetTestEnvironment(_testEnvironment, runnerInfo); - var testAssemblyPath = GetTestDll("EnvironmentVariablesTestProject.dll"); + var testAssemblyPath = GetAssetFullPath("EnvironmentVariablesTestProject.dll"); var runsettingsXml = @" diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/TelemetryTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/TelemetryTests.cs index 1b519c1d9f..fdf49097ce 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/TelemetryTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/TelemetryTests.cs @@ -49,7 +49,7 @@ private void RunTests(RunnerInfo runnerInfo) return; } - var assemblyPaths = GetTestDll("SimpleTestProject2.dll"); + var assemblyPaths = GetAssetFullPath("SimpleTestProject2.dll"); var env = new Dictionary { @@ -70,7 +70,7 @@ private void DiscoverTests(RunnerInfo runnerInfo) return; } - var assemblyPaths = GetTestDll("SimpleTestProject2.dll"); + var assemblyPaths = GetAssetFullPath("SimpleTestProject2.dll"); var env = new Dictionary { diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/TestCaseFilterTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/TestCaseFilterTests.cs index d235068166..c4363cf825 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/TestCaseFilterTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/TestCaseFilterTests.cs @@ -21,7 +21,7 @@ public void RunSelectedTestsWithAndOperatorTrait(RunnerInfo runnerInfo) SetTestEnvironment(_testEnvironment, runnerInfo); var arguments = PrepareArguments( - GetSampleTestDll(), + GetSampleTestAssembly(), GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: TempDirectory.Path); @@ -38,7 +38,7 @@ public void RunSelectedTestsWithCategoryTraitInMixCase(RunnerInfo runnerInfo) SetTestEnvironment(_testEnvironment, runnerInfo); var arguments = PrepareArguments( - GetSampleTestDll(), + GetSampleTestAssembly(), GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: TempDirectory.Path); @@ -55,7 +55,7 @@ public void RunSelectedTestsWithClassNameTrait(RunnerInfo runnerInfo) SetTestEnvironment(_testEnvironment, runnerInfo); var arguments = PrepareArguments( - GetSampleTestDll(), + GetSampleTestAssembly(), GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: TempDirectory.Path); @@ -72,7 +72,7 @@ public void RunSelectedTestsWithFullyQualifiedNameTrait(RunnerInfo runnerInfo) SetTestEnvironment(_testEnvironment, runnerInfo); var arguments = PrepareArguments( - GetSampleTestDll(), + GetSampleTestAssembly(), GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: TempDirectory.Path); @@ -91,7 +91,7 @@ public void RunSelectedTestsWithNameTrait(RunnerInfo runnerInfo) SetTestEnvironment(_testEnvironment, runnerInfo); var arguments = PrepareArguments( - GetSampleTestDll(), + GetSampleTestAssembly(), GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: TempDirectory.Path); @@ -108,7 +108,7 @@ public void RunSelectedTestsWithOrOperatorTrait(RunnerInfo runnerInfo) SetTestEnvironment(_testEnvironment, runnerInfo); var arguments = PrepareArguments( - GetSampleTestDll(), + GetSampleTestAssembly(), GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: TempDirectory.Path); @@ -125,7 +125,7 @@ public void RunSelectedTestsWithPriorityTrait(RunnerInfo runnerInfo) SetTestEnvironment(_testEnvironment, runnerInfo); var arguments = PrepareArguments( - GetSampleTestDll(), + GetSampleTestAssembly(), GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: TempDirectory.Path); diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/CodeCoverageTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/CodeCoverageTests.cs index a5f5efb6eb..c77faab8ab 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/CodeCoverageTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/CodeCoverageTests.cs @@ -437,7 +437,7 @@ public async Task EndSessionShouldEnsureVstestConsoleProcessDies(RunnerInfo runn private IList GetTestAssemblies() { - return GetProjects().Select(p => GetTestDll(p)).ToList(); + return GetProjects().Select(p => GetAssetFullPath(p)).ToList(); } private IList GetProjects() diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/CustomTestHostTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/CustomTestHostTests.cs index d3730b7378..f045bf159d 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/CustomTestHostTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/CustomTestHostTests.cs @@ -45,7 +45,7 @@ public void RunTestsWithCustomTestHostLauncherLaunchesTheProcessUsingTheProvided // Act var customTestHostLauncher = new TestHostLauncherV1(); - _vstestConsoleWrapper.RunTestsWithCustomTestHost(GetTestDlls("MSTestProject1.dll", "MSTestProject2.dll"), GetRunSettingsWithCurrentTargetFramework(), runEventHandler, customTestHostLauncher); + _vstestConsoleWrapper.RunTestsWithCustomTestHost(GetTestDlls("MSTestProject1.dll", "MSTestProject2.dll"), GetDefaultRunSettings(), runEventHandler, customTestHostLauncher); // Assert EnsureTestsRunWithoutErrors(runEventHandler, passed: 2, failed: 2, skipped: 2); @@ -74,7 +74,7 @@ public void RunTestsWithCustomTestHostLauncherLaunchesTheProcessUsingTheProvided // Act var customTestHostLauncher = new TestHostLauncherV2(); - _vstestConsoleWrapper.RunTestsWithCustomTestHost(GetTestDlls("MSTestProject1.dll", "MSTestProject2.dll"), GetRunSettingsWithCurrentTargetFramework(), runEventHandler, customTestHostLauncher); + _vstestConsoleWrapper.RunTestsWithCustomTestHost(GetTestDlls("MSTestProject1.dll", "MSTestProject2.dll"), GetDefaultRunSettings(), runEventHandler, customTestHostLauncher); // Assert EnsureTestsRunWithoutErrors(runEventHandler, passed: 2, failed: 2, skipped: 2); @@ -98,7 +98,7 @@ public void RunTestsWithCustomTestHostLauncherAttachesToDebuggerUsingTheProvided // Act var customTestHostLauncher = new TestHostLauncherV2(); - _vstestConsoleWrapper.RunTestsWithCustomTestHost(GetTestDlls("MSTestProject1.dll", "MSTestProject2.dll"), GetRunSettingsWithCurrentTargetFramework(), runEventHandler, customTestHostLauncher); + _vstestConsoleWrapper.RunTestsWithCustomTestHost(GetTestDlls("MSTestProject1.dll", "MSTestProject2.dll"), GetDefaultRunSettings(), runEventHandler, customTestHostLauncher); // Assert EnsureTestsRunWithoutErrors(runEventHandler, passed: 2, failed: 2, skipped: 2); @@ -123,7 +123,7 @@ public void RunTestsWithCustomTestHostLauncherUsesLaunchWhenGivenAnOutdatedITest // Act var customTestHostLauncher = new TestHostLauncherV1(); - _vstestConsoleWrapper.RunTestsWithCustomTestHost(GetTestDlls("MSTestProject1.dll", "MSTestProject2.dll"), GetRunSettingsWithCurrentTargetFramework(), runEventHandler, customTestHostLauncher); + _vstestConsoleWrapper.RunTestsWithCustomTestHost(GetTestDlls("MSTestProject1.dll", "MSTestProject2.dll"), GetDefaultRunSettings(), runEventHandler, customTestHostLauncher); // Assert EnsureTestsRunWithoutErrors(runEventHandler, passed: 2, failed: 2, skipped: 2); diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/DataCollectorAttachmentProcessor.cs b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/DataCollectorAttachmentProcessor.cs index 4a38f260e7..16708093ba 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/DataCollectorAttachmentProcessor.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/DataCollectorAttachmentProcessor.cs @@ -124,5 +124,5 @@ private static void CreateDataCollectionRunSettingsFile(string destinationRunset } private IList GetTestAssemblies() - => new List { "SimpleTestProject.dll", "SimpleTestProject2.dll" }.Select(p => GetTestDll(p)).ToList(); + => new List { "SimpleTestProject.dll", "SimpleTestProject2.dll" }.Select(p => GetAssetFullPath(p)).ToList(); } diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/DifferentTestFrameworkSimpleTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/DifferentTestFrameworkSimpleTests.cs index af2456f1d8..b08763da05 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/DifferentTestFrameworkSimpleTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/DifferentTestFrameworkSimpleTests.cs @@ -47,12 +47,12 @@ public void RunTestsWithNunitAdapter(RunnerInfo runnerInfo) var sources = new List { - GetTestDll("NUTestProject.dll") + GetAssetFullPath("NUTestProject.dll") }; _vstestConsoleWrapper.RunTests( sources, - GetRunSettingsWithCurrentTargetFramework(), + GetDefaultRunSettings(), _runEventHandler); var testCase = @@ -94,7 +94,7 @@ public void RunTestsWithXunitAdapter(RunnerInfo runnerInfo) _vstestConsoleWrapper.RunTests( sources, - GetRunSettingsWithCurrentTargetFramework(), + GetDefaultRunSettings(), _runEventHandler); var testCase = @@ -134,7 +134,7 @@ public void RunTestsWithChutzpahAdapter(RunnerInfo runnerInfo) _vstestConsoleWrapper.RunTests( sources, - GetRunSettingsWithCurrentTargetFramework(), + GetDefaultRunSettings(), _runEventHandler); var testCase = diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/DiscoverTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/DiscoverTests.cs index e7db8258ce..519c8686ce 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/DiscoverTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/DiscoverTests.cs @@ -50,7 +50,7 @@ public void DiscoverTestsUsingDiscoveryEventHandler1(RunnerInfo runnerInfo) _discoveryEventHandler2 = new DiscoveryEventHandler2(); var vstestConsoleWrapper = GetVsTestConsoleWrapper(); - vstestConsoleWrapper.DiscoverTests(GetTestAssemblies(), GetRunSettingsWithCurrentTargetFramework(), _discoveryEventHandler); + vstestConsoleWrapper.DiscoverTests(GetTestAssemblies(), GetDefaultRunSettings(), _discoveryEventHandler); // Assert. Assert.AreEqual(6, _discoveryEventHandler.DiscoveredTestCases.Count); @@ -69,7 +69,7 @@ public void DiscoverTestsUsingDiscoveryEventHandler2AndTelemetryOptedOut(RunnerI var vstestConsoleWrapper = GetVsTestConsoleWrapper(); vstestConsoleWrapper.DiscoverTests( GetTestAssemblies(), - GetRunSettingsWithCurrentTargetFramework(), + GetDefaultRunSettings(), new TestPlatformOptions() { CollectMetrics = false }, _discoveryEventHandler2); @@ -86,7 +86,7 @@ public void DiscoverTestsUsingDiscoveryEventHandler2AndTelemetryOptedIn(RunnerIn SetTestEnvironment(_testEnvironment, runnerInfo); Setup(); - _vstestConsoleWrapper.DiscoverTests(GetTestAssemblies(), GetRunSettingsWithCurrentTargetFramework(), new TestPlatformOptions() { CollectMetrics = true }, _discoveryEventHandler2); + _vstestConsoleWrapper.DiscoverTests(GetTestAssemblies(), GetDefaultRunSettings(), new TestPlatformOptions() { CollectMetrics = true }, _discoveryEventHandler2); // Assert. Assert.AreEqual(6, _discoveryEventHandler2.DiscoveredTestCases.Count); @@ -164,7 +164,7 @@ public void DisoverTestUsingEventHandler2ShouldContainAllSourcesAsFullyDiscovere _vstestConsoleWrapper.DiscoverTests( GetTestAssemblies(), - GetRunSettingsWithCurrentTargetFramework(), + GetDefaultRunSettings(), null, eventHandler2); @@ -182,7 +182,7 @@ public void DiscoverTestsUsingSourceNavigation(RunnerInfo runnerInfo) _vstestConsoleWrapper.DiscoverTests( GetTestAssemblies(), - GetRunSettingsWithCurrentTargetFramework(), + GetDefaultRunSettings(), _discoveryEventHandler); // Assert. @@ -208,8 +208,8 @@ public async Task CancelTestDiscovery(RunnerInfo runnerInfo) // Setup var testAssemblies = new List { - GetTestDll("DiscoveryTestProject.dll"), - GetTestDll("SimpleTestProject.dll"), + GetAssetFullPath("DiscoveryTestProject.dll"), + GetAssetFullPath("SimpleTestProject.dll"), }; SetTestEnvironment(_testEnvironment, runnerInfo); @@ -235,7 +235,7 @@ public async Task CancelTestDiscovery(RunnerInfo runnerInfo) }); // Act - await Task.Run(() => _vstestConsoleWrapper.DiscoverTests(testAssemblies, GetRunSettingsWithCurrentTargetFramework(), discoveryEvents.Object)); + await Task.Run(() => _vstestConsoleWrapper.DiscoverTests(testAssemblies, GetDefaultRunSettings(), discoveryEvents.Object)); // Assert. Assert.IsTrue(isTestCancelled); @@ -247,8 +247,8 @@ private IList GetTestAssemblies() { var testAssemblies = new List { - GetTestDll("SimpleTestProject.dll"), - GetTestDll("SimpleTestProject2.dll") + GetAssetFullPath("SimpleTestProject.dll"), + GetAssetFullPath("SimpleTestProject2.dll") }; return testAssemblies; diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/LiveUnitTestingTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/LiveUnitTestingTests.cs index 01b4367468..0beac2e85c 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/LiveUnitTestingTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/LiveUnitTestingTests.cs @@ -90,8 +90,8 @@ private IList GetTestAssemblies() { var testAssemblies = new List { - GetTestDll("SimpleTestProject.dll"), - GetTestDll("SimpleTestProject2.dll") + GetAssetFullPath("SimpleTestProject.dll"), + GetAssetFullPath("SimpleTestProject2.dll") }; return testAssemblies; diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/RunSelectedTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/RunSelectedTests.cs index f2722cc154..73b07981b6 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/RunSelectedTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/RunSelectedTests.cs @@ -42,10 +42,10 @@ public void RunSelectedTestsWithoutTestPlatformOptions(RunnerInfo runnerInfo) SetTestEnvironment(_testEnvironment, runnerInfo); Setup(); - _vstestConsoleWrapper.DiscoverTests(GetTestAssemblies(), GetRunSettingsWithCurrentTargetFramework(), _discoveryEventHandler); + _vstestConsoleWrapper.DiscoverTests(GetTestAssemblies(), GetDefaultRunSettings(), _discoveryEventHandler); var testCases = _discoveryEventHandler.DiscoveredTestCases; - _vstestConsoleWrapper.RunTests(testCases, GetRunSettingsWithCurrentTargetFramework(), _runEventHandler); + _vstestConsoleWrapper.RunTests(testCases, GetDefaultRunSettings(), _runEventHandler); // Assert Assert.AreEqual(6, _runEventHandler.TestResults.Count); @@ -62,12 +62,12 @@ public void RunSelectedTestsWithTestPlatformOptions(RunnerInfo runnerInfo) SetTestEnvironment(_testEnvironment, runnerInfo); Setup(); - _vstestConsoleWrapper.DiscoverTests(GetTestAssemblies(), GetRunSettingsWithCurrentTargetFramework(), _discoveryEventHandler); + _vstestConsoleWrapper.DiscoverTests(GetTestAssemblies(), GetDefaultRunSettings(), _discoveryEventHandler); var testCases = _discoveryEventHandler.DiscoveredTestCases; _vstestConsoleWrapper.RunTests( testCases, - GetRunSettingsWithCurrentTargetFramework(), + GetDefaultRunSettings(), new TestPlatformOptions() { CollectMetrics = true }, _runEventHandler); @@ -85,8 +85,8 @@ private IList GetTestAssemblies() { var testAssemblies = new List { - GetTestDll("SimpleTestProject.dll"), - GetTestDll("SimpleTestProject2.dll") + GetAssetFullPath("SimpleTestProject.dll"), + GetAssetFullPath("SimpleTestProject2.dll") }; return testAssemblies; diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/RunTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/RunTests.cs index 168120af82..ffe2449e15 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/RunTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/RunTests.cs @@ -47,7 +47,7 @@ public void RunAllTests(RunnerInfo runnerInfo) var vstestConsoleWrapper = GetVsTestConsoleWrapper(); var runEventHandler = new RunEventHandler(); - vstestConsoleWrapper.RunTests(GetTestAssemblies(), GetRunSettingsWithCurrentTargetFramework(), runEventHandler); + vstestConsoleWrapper.RunTests(GetTestAssemblies(), GetDefaultRunSettings(), runEventHandler); // Assert Assert.AreEqual(6, runEventHandler.TestResults.Count); @@ -67,7 +67,7 @@ public void EndSessionShouldEnsureVstestConsoleProcessDies(RunnerInfo runnerInfo SetTestEnvironment(_testEnvironment, runnerInfo); Setup(); - _vstestConsoleWrapper.RunTests(GetTestAssemblies(), GetRunSettingsWithCurrentTargetFramework(), _runEventHandler); + _vstestConsoleWrapper.RunTests(GetTestAssemblies(), GetDefaultRunSettings(), _runEventHandler); _vstestConsoleWrapper?.EndSession(); // Assert @@ -89,7 +89,7 @@ public void RunTestsWithTelemetryOptedIn(RunnerInfo runnerInfo) _vstestConsoleWrapper.RunTests( GetTestAssemblies(), - GetRunSettingsWithCurrentTargetFramework(), + GetDefaultRunSettings(), new TestPlatformOptions() { CollectMetrics = true }, _runEventHandler); @@ -113,7 +113,7 @@ public void RunTestsWithTelemetryOptedOut(RunnerInfo runnerInfo) _vstestConsoleWrapper.RunTests( GetTestAssemblies(), - GetRunSettingsWithCurrentTargetFramework(), + GetDefaultRunSettings(), new TestPlatformOptions() { CollectMetrics = false }, _runEventHandler); @@ -137,11 +137,11 @@ public void RunTestsShouldThrowOnStackOverflowException(RunnerInfo runnerInfo) return; } - var source = new[] { GetTestDll("SimpleTestProject3.dll") }; + var source = new[] { GetAssetFullPath("SimpleTestProject3.dll") }; _vstestConsoleWrapper.RunTests( source, - GetRunSettingsWithCurrentTargetFramework(), + GetDefaultRunSettings(), new TestPlatformOptions() { TestCaseFilter = "ExitWithStackoverFlow" }, _runEventHandler); @@ -162,7 +162,7 @@ public void RunTestsShouldShowProperWarningOnNoTestsForTestCaseFilter(RunnerInfo Setup(); var testAssemblyName = "SimpleTestProject2.dll"; - var source = new List() { GetTestDll(testAssemblyName) }; + var source = new List() { GetAssetFullPath(testAssemblyName) }; var veryLongTestCaseFilter = "FullyQualifiedName=VeryLongTestCaseNameeeeeeeeeeeeee" + @@ -173,7 +173,7 @@ public void RunTestsShouldShowProperWarningOnNoTestsForTestCaseFilter(RunnerInfo _vstestConsoleWrapper.RunTests( source, - GetRunSettingsWithCurrentTargetFramework(), + GetDefaultRunSettings(), new TestPlatformOptions() { TestCaseFilter = veryLongTestCaseFilter }, _runEventHandler); @@ -190,8 +190,8 @@ private IList GetTestAssemblies() { return new List { - GetTestDll("SimpleTestProject.dll"), - GetTestDll("SimpleTestProject2.dll") + GetAssetFullPath("SimpleTestProject.dll"), + GetAssetFullPath("SimpleTestProject2.dll") }; } } diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/RunTestsWithDifferentConfigurationTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/RunTestsWithDifferentConfigurationTests.cs index de916c635d..4f6e641fb2 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/RunTestsWithDifferentConfigurationTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/RunTestsWithDifferentConfigurationTests.cs @@ -56,7 +56,7 @@ public void RunTestsWithTestAdapterPath(RunnerInfo runnerInfo) _vstestConsoleWrapper.RunTests( GetTestAssemblies(), - GetRunSettingsWithCurrentTargetFramework(), + GetDefaultRunSettings(), _runEventHandler); // Assert @@ -115,7 +115,7 @@ public void RunTestsWithTestSettings(RunnerInfo runnerInfo) var runSettings = $"{FrameworkArgValue}" + testsettingsFile + ""; var sources = new List { - GetTestDll("MstestV1UnitTestProject.dll") + GetAssetFullPath("MstestV1UnitTestProject.dll") }; _vstestConsoleWrapper.RunTests( @@ -140,7 +140,7 @@ public void RunTestsWithX64Source(RunnerInfo runnerInfo) var sources = new List { - GetTestDll("SimpleTestProject3.dll") + GetAssetFullPath("SimpleTestProject3.dll") }; @@ -149,7 +149,7 @@ public void RunTestsWithX64Source(RunnerInfo runnerInfo) _vstestConsoleWrapper.RunTests( sources, - GetRunSettingsWithCurrentTargetFramework(), + GetDefaultRunSettings(), new TestPlatformOptions() { TestCaseFilter = "FullyQualifiedName = SampleUnitTestProject3.UnitTest1.WorkingDirectoryTest" }, _runEventHandler); @@ -163,8 +163,8 @@ private IList GetTestAssemblies() { var testAssemblies = new List { - GetTestDll("SimpleTestProject.dll"), - GetTestDll("SimpleTestProject2.dll") + GetAssetFullPath("SimpleTestProject.dll"), + GetAssetFullPath("SimpleTestProject2.dll") }; return testAssemblies; diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/RunTestsWithFilterTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/RunTestsWithFilterTests.cs index bfc394614c..ea4c38ae0f 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/RunTestsWithFilterTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/RunTestsWithFilterTests.cs @@ -44,11 +44,11 @@ public void RunTestsWithTestCaseFilter(RunnerInfo runnerInfo) _runEventHandler = new RunEventHandler(); var vstestConsoleWrapper = GetVsTestConsoleWrapper(); - var sources = new List { GetTestDll("SimpleTestProject.dll") }; + var sources = new List { GetAssetFullPath("SimpleTestProject.dll") }; vstestConsoleWrapper.RunTests( sources, - GetRunSettingsWithCurrentTargetFramework(), + GetDefaultRunSettings(), new TestPlatformOptions() { TestCaseFilter = "FullyQualifiedName=SampleUnitTestProject.UnitTest1.PassingTest" }, _runEventHandler); @@ -65,11 +65,11 @@ public void RunTestsWithFastFilter(RunnerInfo runnerInfo) SetTestEnvironment(_testEnvironment, runnerInfo); Setup(); - var sources = new List { GetTestDll("SimpleTestProject.dll") }; + var sources = new List { GetAssetFullPath("SimpleTestProject.dll") }; _vstestConsoleWrapper.RunTests( sources, - GetRunSettingsWithCurrentTargetFramework(), + GetDefaultRunSettings(), new TestPlatformOptions() { TestCaseFilter = "FullyQualifiedName=SampleUnitTestProject.UnitTest1.PassingTest | FullyQualifiedName=SampleUnitTestProject.UnitTest1.FailingTest" }, _runEventHandler); diff --git a/test/Microsoft.TestPlatform.ObjectModel.PlatformTests/DiaSessionTests.cs b/test/Microsoft.TestPlatform.ObjectModel.PlatformTests/DiaSessionTests.cs index c7880824e9..4925635fe3 100644 --- a/test/Microsoft.TestPlatform.ObjectModel.PlatformTests/DiaSessionTests.cs +++ b/test/Microsoft.TestPlatform.ObjectModel.PlatformTests/DiaSessionTests.cs @@ -34,7 +34,7 @@ public static string GetAndSetTargetFrameWork(IntegrationTestEnvironment testEnv public void GetNavigationDataShouldReturnCorrectFileNameAndLineNumber() { var currentTargetFrameWork = GetAndSetTargetFrameWork(_testEnvironment); - var assemblyPath = GetTestDll("SimpleClassLibrary.dll"); + var assemblyPath = GetAssetFullPath("SimpleClassLibrary.dll"); var diaSession = new DiaSession(assemblyPath); DiaNavigationData diaNavigationData = diaSession.GetNavigationData("SimpleClassLibrary.Class1", "PassingTest"); @@ -52,7 +52,7 @@ public void GetNavigationDataShouldReturnCorrectFileNameAndLineNumber() public void GetNavigationDataShouldReturnCorrectDataForAsyncMethod() { var currentTargetFrameWork = GetAndSetTargetFrameWork(_testEnvironment); - var assemblyPath = GetTestDll("SimpleClassLibrary.dll"); + var assemblyPath = GetAssetFullPath("SimpleClassLibrary.dll"); var diaSession = new DiaSession(assemblyPath); DiaNavigationData diaNavigationData = diaSession.GetNavigationData("SimpleClassLibrary.Class1+d__1", "MoveNext"); @@ -70,7 +70,7 @@ public void GetNavigationDataShouldReturnCorrectDataForAsyncMethod() public void GetNavigationDataShouldReturnCorrectDataForOverLoadedMethod() { var currentTargetFrameWork = GetAndSetTargetFrameWork(_testEnvironment); - var assemblyPath = GetTestDll("SimpleClassLibrary.dll"); + var assemblyPath = GetAssetFullPath("SimpleClassLibrary.dll"); var diaSession = new DiaSession(assemblyPath); DiaNavigationData diaNavigationData = diaSession.GetNavigationData("SimpleClassLibrary.Class1", "OverLoadedMethod"); @@ -89,7 +89,7 @@ public void GetNavigationDataShouldReturnCorrectDataForOverLoadedMethod() public void GetNavigationDataShouldReturnNullForNotExistMethodNameOrNotExistTypeName() { var currentTargetFrameWork = GetAndSetTargetFrameWork(_testEnvironment); - var assemblyPath = GetTestDll("SimpleClassLibrary.dll"); + var assemblyPath = GetAssetFullPath("SimpleClassLibrary.dll"); var diaSession = new DiaSession(assemblyPath); @@ -108,7 +108,7 @@ public void GetNavigationDataShouldReturnNullForNotExistMethodNameOrNotExistType public void DiaSessionPerfTest() { var currentTargetFrameWork = GetAndSetTargetFrameWork(_testEnvironment); - var assemblyPath = GetTestDll("SimpleClassLibrary.dll"); + var assemblyPath = GetAssetFullPath("SimpleClassLibrary.dll"); var watch = Stopwatch.StartNew(); var diaSession = new DiaSession(assemblyPath); diff --git a/test/Microsoft.TestPlatform.PerformanceTests/PerformanceTests.cs b/test/Microsoft.TestPlatform.PerformanceTests/PerformanceTests.cs index f87afc7f58..7db4ae1e30 100644 --- a/test/Microsoft.TestPlatform.PerformanceTests/PerformanceTests.cs +++ b/test/Microsoft.TestPlatform.PerformanceTests/PerformanceTests.cs @@ -18,7 +18,7 @@ public class PerformanceTests : PerformanceTestBase [TestMethod] public void ExecutionPerformanceTest() { - RunExecutionPerformanceTests(GetSampleTestDll(), GetTestAdapterPath(), string.Empty); + RunExecutionPerformanceTests(GetSampleTestAssembly(), GetTestAdapterPath(), string.Empty); ValidateSummaryStatus(1, 1, 1); ValidatePassedTests("SampleUnitTestProject.UnitTest1.PassingTest"); @@ -35,7 +35,7 @@ public void ExecutionPerformanceTest() [TestMethod] public void DiscoveryPerformanceTest() { - RunDiscoveryPerformanceTests(GetSampleTestDll(), GetTestAdapterPath(), string.Empty); + RunDiscoveryPerformanceTests(GetSampleTestAssembly(), GetTestAdapterPath(), string.Empty); ValidateDiscoveredTests( "SampleUnitTestProject.UnitTest1.PassingTest", @@ -52,7 +52,7 @@ public void DiscoveryPerformanceTest() [TestMethod] public void VsTestConsolePerformanceTest() { - RunExecutionPerformanceTests(GetSampleTestDll(), GetTestAdapterPath(), string.Empty); + RunExecutionPerformanceTests(GetSampleTestAssembly(), GetTestAdapterPath(), string.Empty); ValidateSummaryStatus(1, 1, 1); ValidatePassedTests("SampleUnitTestProject.UnitTest1.PassingTest"); @@ -69,7 +69,7 @@ public void VsTestConsolePerformanceTest() [TestMethod] public void TestHostPerformanceTest() { - RunExecutionPerformanceTests(GetSampleTestDll(), GetTestAdapterPath(), string.Empty); + RunExecutionPerformanceTests(GetSampleTestAssembly(), GetTestAdapterPath(), string.Empty); ValidateSummaryStatus(1, 1, 1); ValidatePassedTests("SampleUnitTestProject.UnitTest1.PassingTest"); @@ -86,7 +86,7 @@ public void TestHostPerformanceTest() [TestMethod] public void MsTestV2AdapterPerformanceTest() { - RunExecutionPerformanceTests(GetSampleTestDll(), GetTestAdapterPath(), string.Empty); + RunExecutionPerformanceTests(GetSampleTestAssembly(), GetTestAdapterPath(), string.Empty); ValidateSummaryStatus(1, 1, 1); ValidatePassedTests("SampleUnitTestProject.UnitTest1.PassingTest"); diff --git a/test/Microsoft.TestPlatform.SmokeTests/DiscoveryTests.cs b/test/Microsoft.TestPlatform.SmokeTests/DiscoveryTests.cs index 83a3a2b71d..c74c041202 100644 --- a/test/Microsoft.TestPlatform.SmokeTests/DiscoveryTests.cs +++ b/test/Microsoft.TestPlatform.SmokeTests/DiscoveryTests.cs @@ -15,7 +15,7 @@ public class DiscoveryTests : IntegrationTestBase [TestMethod] public void DiscoverAllTests() { - InvokeVsTestForDiscovery(GetSampleTestDll(), GetTestAdapterPath(), string.Empty, ".NETFramework,Version=v4.5.1"); + InvokeVsTestForDiscovery(GetSampleTestAssembly(), GetTestAdapterPath(), string.Empty, ".NETFramework,Version=v4.5.1"); var listOfTests = new string[] { "SampleUnitTestProject.UnitTest1.PassingTest", "SampleUnitTestProject.UnitTest1.FailingTest", "SampleUnitTestProject.UnitTest1.SkippingTest" }; ValidateDiscoveredTests(listOfTests); } diff --git a/test/Microsoft.TestPlatform.SmokeTests/ExecutionTests.cs b/test/Microsoft.TestPlatform.SmokeTests/ExecutionTests.cs index b74d83b531..a12a33d98b 100644 --- a/test/Microsoft.TestPlatform.SmokeTests/ExecutionTests.cs +++ b/test/Microsoft.TestPlatform.SmokeTests/ExecutionTests.cs @@ -15,7 +15,7 @@ public class ExecutionTests : IntegrationTestBase [TestMethod] public void RunAllTestExecution() { - InvokeVsTestForExecution(GetSampleTestDll(), GetTestAdapterPath(), ".NETFramework,Version=v4.5.1"); + InvokeVsTestForExecution(GetSampleTestAssembly(), GetTestAdapterPath(), ".NETFramework,Version=v4.5.1"); ValidateSummaryStatus(1, 1, 1); ValidatePassedTests("SampleUnitTestProject.UnitTest1.PassingTest"); ValidateFailedTests("SampleUnitTestProject.UnitTest1.FailingTest"); @@ -26,7 +26,7 @@ public void RunAllTestExecution() public void RunSelectedTests() { using var resultsDir = new TempDirectory(); - var arguments = PrepareArguments(GetSampleTestDll(), GetTestAdapterPath(), string.Empty, ".NETFramework,Version=v4.5.1", resultsDirectory: resultsDir.Path); + var arguments = PrepareArguments(GetSampleTestAssembly(), GetTestAdapterPath(), string.Empty, ".NETFramework,Version=v4.5.1", resultsDirectory: resultsDir.Path); arguments = string.Concat(arguments, " /Tests:PassingTest"); InvokeVsTest(arguments); ValidateSummaryStatus(1, 0, 0); diff --git a/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestBase.cs b/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestBase.cs index e68e8fec05..a3084ede30 100644 --- a/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestBase.cs +++ b/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestBase.cs @@ -522,12 +522,12 @@ public void ValidateFullyQualifiedDiscoveredTests(string filePath, params string } } - protected string GetSampleTestDll() + protected string GetSampleTestAssembly() { - return GetTestDll("SimpleTestProject.dll"); + return GetAssetFullPath("SimpleTestProject.dll"); } - protected string GetTestDll(string assetName) + protected string GetAssetFullPath(string assetName) { return _testEnvironment.GetTestAsset(assetName); } @@ -542,7 +542,7 @@ protected List GetTestDlls(params string[] assetNames) var assets = new List(); foreach (var assetName in assetNames) { - assets.Add(GetTestDll(assetName)); + assets.Add(GetAssetFullPath(assetName)); } return assets; From 2b3e039f525713af91722d128b8f587bf35d960c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Jare=C5=A1?= Date: Mon, 21 Mar 2022 16:48:05 +0100 Subject: [PATCH 059/112] Ignore strong naming in tests --- playground/TestPlatform.Playground/Program.cs | 3 +-- .../TestPlatform.Playground/TestPlatform.Playground.csproj | 2 +- scripts/build/TestPlatform.Settings.targets | 3 ++- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/playground/TestPlatform.Playground/Program.cs b/playground/TestPlatform.Playground/Program.cs index 5d46ab9328..2b1fcb0a0c 100644 --- a/playground/TestPlatform.Playground/Program.cs +++ b/playground/TestPlatform.Playground/Program.cs @@ -55,8 +55,7 @@ static void Main(string[] args) "; var sources = new[] { - @"C:\p\vstest\test\TestAssets\MSTestProject1\bin\NETTestSdkLatest-17.2.0-dev\MSTestLatestPreview-2.2.9-preview-20220210-07\Debug\net451\MSTestProject1.dll", - @"C:\p\vstest\test\TestAssets\MSTestProject2\bin\NETTestSdkLatest-17.2.0-dev\MSTestLatestPreview-2.2.9-preview-20220210-07\Debug\net451\MSTestProject2.dll" + Path.Combine(playground, "MSTest1", "bin", "Debug", "net472", "MSTest1.dll") }; var options = new TestPlatformOptions(); diff --git a/playground/TestPlatform.Playground/TestPlatform.Playground.csproj b/playground/TestPlatform.Playground/TestPlatform.Playground.csproj index 7bdbeb915e..77ccfd0b7e 100644 --- a/playground/TestPlatform.Playground/TestPlatform.Playground.csproj +++ b/playground/TestPlatform.Playground/TestPlatform.Playground.csproj @@ -38,7 +38,7 @@ - + diff --git a/scripts/build/TestPlatform.Settings.targets b/scripts/build/TestPlatform.Settings.targets index b297c84ab9..3c1e0d7ef9 100644 --- a/scripts/build/TestPlatform.Settings.targets +++ b/scripts/build/TestPlatform.Settings.targets @@ -77,7 +77,8 @@ false - $(NoWarn);MSB3270 + + $(NoWarn);MSB3270;CS8002 From 598739127683b0613022a8dba4c7f0d256e44dbe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Jare=C5=A1?= Date: Mon, 21 Mar 2022 17:17:10 +0100 Subject: [PATCH 060/112] Add tags --- .../Microsoft.TestPlatform.AcceptanceTests/ExecutionTests.cs | 4 ++++ .../Extension/MSTestCompatibilityDataSource.cs | 2 -- .../TranslationLayerTests/CustomTestHostTests.cs | 5 ++++- .../TranslationLayerTests/DiscoverTests.cs | 2 ++ 4 files changed, 10 insertions(+), 3 deletions(-) diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/ExecutionTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/ExecutionTests.cs index 60b60036ed..c9676ae547 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/ExecutionTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/ExecutionTests.cs @@ -18,6 +18,7 @@ public class ExecutionTests : AcceptanceTestBase { // TODO: It looks like the first 3 tests would be useful to multiply by all 3 test frameworks, should we make the test even more generic, or duplicate them? [TestMethod] + [TestCategory("Windows-Review")] [MSTestCompatibilityDataSource(InProcess = true)] public void RunMultipleTestAssemblies(RunnerInfo runnerInfo) { @@ -34,6 +35,7 @@ public void RunMultipleTestAssemblies(RunnerInfo runnerInfo) } [TestMethod] + [TestCategory("Windows-Review")] [TestPlatformCompatibilityDataSource()] //[TestPlatformCompatibilityDataSource(BeforeFeature = Features.ATTACH_DEBUGGER, AfterAdapterFeature = Features.MSTEST_IFRAMEWORK_HANDLE_99)] @@ -52,6 +54,7 @@ public void RunTestsFromMultipleMSTestAssemblies(RunnerInfo runnerInfo) } [TestMethod] + [TestCategory("Windows-Review")] [TestHostCompatibilityDataSource] public void RunMultipleMSTestAssembliesOnVstestConsoleAndTesthostCombinations(RunnerInfo runnerInfo) { @@ -67,6 +70,7 @@ public void RunMultipleMSTestAssembliesOnVstestConsoleAndTesthostCombinations(Ru [TestMethod] + [TestCategory("Windows-Review")] [RunnerCompatibilityDataSource] public void RunMultipleMSTestAssembliesOnVstestConsoleAndTesthostCombinations2(RunnerInfo runnerInfo) { diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/MSTestCompatibilityDataSource.cs b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/MSTestCompatibilityDataSource.cs index 33c53f8a5d..f6d20b577c 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/MSTestCompatibilityDataSource.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/MSTestCompatibilityDataSource.cs @@ -3,10 +3,8 @@ using System.Reflection; - namespace Microsoft.TestPlatform.AcceptanceTests; - public class MSTestCompatibilityDataSource : TestDataSource { private readonly CompatibilityRowsBuilder _builder; diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/CustomTestHostTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/CustomTestHostTests.cs index f045bf159d..7783efc554 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/CustomTestHostTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/CustomTestHostTests.cs @@ -31,6 +31,7 @@ public void Cleanup() } [TestMethod] + [TestCategory("Windows-Review")] [RunnerCompatibilityDataSource(BeforeFeature = Features.ATTACH_DEBUGGER_FLOW)] [TestHostCompatibilityDataSource(BeforeFeature = Features.ATTACH_DEBUGGER_FLOW)] public void RunTestsWithCustomTestHostLauncherLaunchesTheProcessUsingTheProvidedLauncher(RunnerInfo runnerInfo) @@ -56,6 +57,7 @@ public void RunTestsWithCustomTestHostLauncherLaunchesTheProcessUsingTheProvided } [TestMethod] + [TestCategory("Windows-Review")] // [RunnerCompatibilityDataSource(BeforeFeature = Features.ATTACH_DEBUGGER_FLOW)] [TestHostCompatibilityDataSource("net451", "netcoreapp2.1", "LegacyStable", BeforeFeature = Features.ATTACH_DEBUGGER_FLOW, DebugVSTestConsole = true)] [Ignore("This is not working for any testhost prior 16.7.0 where the change was introduced. The launch testhost flow was replaced with AttachDebugger in runner, and the new callback to AttachDebugger happens in testhost." @@ -85,9 +87,9 @@ public void RunTestsWithCustomTestHostLauncherLaunchesTheProcessUsingTheProvided } [TestMethod] + [TestCategory("Windows-Review")] [RunnerCompatibilityDataSource(AfterFeature = Features.ATTACH_DEBUGGER_FLOW)] // [TestHostCompatibilityDataSource(AfterFeature = Features.ATTACH_DEBUGGER_FLOW)] - [TestHostCompatibilityDataSource()] public void RunTestsWithCustomTestHostLauncherAttachesToDebuggerUsingTheProvidedLauncher(RunnerInfo runnerInfo) { @@ -109,6 +111,7 @@ public void RunTestsWithCustomTestHostLauncherAttachesToDebuggerUsingTheProvided } [TestMethod] + [TestCategory("Windows-Review")] [Ignore("This is not working. The compatibility code only checks the protocol version (in handler), which is dictated by testhost. " + "It sees 6 but does not realize that the provided CustomTesthostLauncher is not supporting the new feature, it ends up calling back to EditoAttachDebugger" + "in translation layer, and that just silently skips the call.")] diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/DiscoverTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/DiscoverTests.cs index 519c8686ce..9280597fc6 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/DiscoverTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/DiscoverTests.cs @@ -40,6 +40,7 @@ public void Cleanup() } [TestMethod] + [TestCategory("Windows-Review")] [RunnerCompatibilityDataSource] public void DiscoverTestsUsingDiscoveryEventHandler1(RunnerInfo runnerInfo) { @@ -57,6 +58,7 @@ public void DiscoverTestsUsingDiscoveryEventHandler1(RunnerInfo runnerInfo) } [TestMethod] + [TestCategory("Windows-Review")] [RunnerCompatibilityDataSource()] public void DiscoverTestsUsingDiscoveryEventHandler2AndTelemetryOptedOut(RunnerInfo runnerInfo) { From 0e4ce5a3d63e731d448abf16f96bbd10866b276f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Jare=C5=A1?= Date: Mon, 21 Mar 2022 17:59:18 +0100 Subject: [PATCH 061/112] Fix console path --- .../TranslationLayerTests/CustomTestHostTests.cs | 5 ++++- .../IntegrationTestBase.cs | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/CustomTestHostTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/CustomTestHostTests.cs index 7783efc554..ac6b33001a 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/CustomTestHostTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/CustomTestHostTests.cs @@ -33,7 +33,10 @@ public void Cleanup() [TestMethod] [TestCategory("Windows-Review")] [RunnerCompatibilityDataSource(BeforeFeature = Features.ATTACH_DEBUGGER_FLOW)] - [TestHostCompatibilityDataSource(BeforeFeature = Features.ATTACH_DEBUGGER_FLOW)] + // This does not work with testhosts that are earlier than when the feature was introduced, + // when latest runner is used, because the latest runner does not downgrade the messages when + // older testhost launcher is used. + // [TestHostCompatibilityDataSource(BeforeFeature = Features.ATTACH_DEBUGGER_FLOW)] public void RunTestsWithCustomTestHostLauncherLaunchesTheProcessUsingTheProvidedLauncher(RunnerInfo runnerInfo) { // Pins the existing functionality. diff --git a/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestBase.cs b/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestBase.cs index a3084ede30..0b93cfcfae 100644 --- a/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestBase.cs +++ b/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestBase.cs @@ -605,7 +605,7 @@ public virtual string GetConsoleRunnerPath() if (IsDesktopRunner()) { - if (!string.IsNullOrWhiteSpace(_testEnvironment?.VSTestConsoleInfo.Path)) + if (!string.IsNullOrWhiteSpace(_testEnvironment.VSTestConsoleInfo?.Path)) { consoleRunnerPath = _testEnvironment.VSTestConsoleInfo.Path; } From 969c2b3fd8efeeee3516840741073f7aab1ff4ea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Jare=C5=A1?= Date: Tue, 22 Mar 2022 11:53:38 +0100 Subject: [PATCH 062/112] fix --- .../TestPluginDiscoverer.cs | 2 +- .../DotnetTestTests.cs | 2 +- .../DifferentTestFrameworkSimpleTests.cs | 16 +++--- .../BannedSymbols.txt | 3 ++ .../IntegrationTestBase.cs | 54 ++++++++++--------- ...icrosoft.TestPlatform.TestUtilities.csproj | 10 ++++ .../TempDirectory.cs | 39 +++++++++++++- 7 files changed, 91 insertions(+), 35 deletions(-) create mode 100644 test/Microsoft.TestPlatform.TestUtilities/BannedSymbols.txt diff --git a/src/Microsoft.TestPlatform.Common/ExtensionFramework/TestPluginDiscoverer.cs b/src/Microsoft.TestPlatform.Common/ExtensionFramework/TestPluginDiscoverer.cs index e6fb7315ca..4401f39cec 100644 --- a/src/Microsoft.TestPlatform.Common/ExtensionFramework/TestPluginDiscoverer.cs +++ b/src/Microsoft.TestPlatform.Common/ExtensionFramework/TestPluginDiscoverer.cs @@ -190,7 +190,7 @@ private void GetTestExtensionsFromAssembly(Assembly ass if (e.Types?.Length > 0) { - types.AddRange(e.Types.Where(type => type.GetTypeInfo().IsClass && !type.GetTypeInfo().IsAbstract)); + types.AddRange(e.Types.Where(type => type != null && type.GetTypeInfo().IsClass && !type.GetTypeInfo().IsAbstract)); } if (e.LoaderExceptions != null) diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/DotnetTestTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/DotnetTestTests.cs index afd985065d..cd5515701a 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/DotnetTestTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/DotnetTestTests.cs @@ -31,7 +31,7 @@ public void RunDotnetTestWithCsproj(RunnerInfo runnerInfo) [TestMethod] // patched dotnet is not published on non-windows systems [TestCategory("Windows-Review")] - [NetCoreTargetFrameworkDataSource] + [NetCoreTargetFrameworkDataSource(useDesktopRunner: false)] public void RunDotnetTestWithDll(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/DifferentTestFrameworkSimpleTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/DifferentTestFrameworkSimpleTests.cs index b08763da05..cfae534f62 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/DifferentTestFrameworkSimpleTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/DifferentTestFrameworkSimpleTests.cs @@ -124,21 +124,23 @@ public void RunTestsWithChutzpahAdapter(RunnerInfo runnerInfo) SetTestEnvironment(_testEnvironment, runnerInfo); Setup(); - var sources = new List - { - Path.Combine(_testEnvironment.TestAssetsPath, "test.js") - }; + var jsSource = Path.Combine(_testEnvironment.TestAssetsPath, "test.js"); + + // Chuzpah adapter creates _Chutzpah temp files, to give data back from the runner. + // But when cleaning up it deletes all the _Chutzpah files, not just the one it owns, + // so when we run in parallel, the slower process will never find it's own file, because it was already deleted: + // https://github.com/mmanela/chutzpah/issues/812 + var jsInTemp = TempDirectory.CopyFile(jsSource); var testAdapterPath = Directory.EnumerateFiles(GetTestAdapterPath(UnitTestFramework.Chutzpah), "*.TestAdapter.dll").ToList(); _vstestConsoleWrapper.InitializeExtensions(new List() { testAdapterPath.FirstOrDefault() }); _vstestConsoleWrapper.RunTests( - sources, + new[] { jsInTemp }, GetDefaultRunSettings(), _runEventHandler); - var testCase = - _runEventHandler.TestResults.Where(tr => tr.TestCase.DisplayName.Equals("TestMethod1")); + var testCase = _runEventHandler.TestResults.Where(tr => tr.TestCase.DisplayName.Equals("TestMethod1")); // Assert Assert.AreEqual(2, _runEventHandler.TestResults.Count); diff --git a/test/Microsoft.TestPlatform.TestUtilities/BannedSymbols.txt b/test/Microsoft.TestPlatform.TestUtilities/BannedSymbols.txt new file mode 100644 index 0000000000..c8eaef56e1 --- /dev/null +++ b/test/Microsoft.TestPlatform.TestUtilities/BannedSymbols.txt @@ -0,0 +1,3 @@ +M:System.IO.Path.GetTempPath(); Use 'IntegrationTestBase.GetTempPath()' instead +M:System.Environment.SetEnvironmentVariable(System.String,System.String); Use one of the overload accepting a dictionary of environment variables instead +M:System.Environment.SetEnvironmentVariable(System.String,System.String,System.EnvironmentVariableTarget); Use one of the overload accepting a dictionary of environment variables instead \ No newline at end of file diff --git a/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestBase.cs b/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestBase.cs index 0b93cfcfae..6e835eedd0 100644 --- a/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestBase.cs +++ b/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestBase.cs @@ -189,24 +189,14 @@ public void InvokeVsTest(string arguments, Dictionary environmen public void InvokeDotnetTest(string arguments, Dictionary environmentVariables = null) { var vstestConsolePath = GetDotnetRunnerPath(); - var env = "VSTEST_CONSOLE_PATH"; - var originalVstestConsolePath = Environment.GetEnvironmentVariable(env); - - try - { - Environment.SetEnvironmentVariable(env, vstestConsolePath); - if (arguments.Contains(".csproj")) - { - arguments = $@"-p:VsTestConsolePath=""{vstestConsolePath}"" " + arguments; - } - - ExecutePatchedDotnet("test", arguments, out _standardTestOutput, out _standardTestError, out _runnerExitCode, environmentVariables); - FormatStandardOutCome(); - } - finally + + if (arguments.Contains(".csproj")) { - Environment.SetEnvironmentVariable(env, originalVstestConsolePath); + arguments = $@"-p:VsTestConsolePath=""{vstestConsolePath}"" " + arguments; } + + ExecutePatchedDotnet("test", arguments, out _standardTestOutput, out _standardTestError, out _runnerExitCode, environmentVariables); + FormatStandardOutCome(); } /// @@ -223,33 +213,44 @@ public void InvokeVsTestForExecution(string testAssembly, Dictionary environmentVariables = null) { var arguments = PrepareArguments(testAssembly, testAdapterPath, runSettings, framework, _testEnvironment.InIsolationValue, resultsDirectory: TempDirectory.Path); + InvokeVsTest(arguments, AddDebugEnvVariables(environmentVariables)); + } - if (_testEnvironment.DebugInfo.DebugVSTestConsole || _testEnvironment.DebugInfo.DebugTestHost || _testEnvironment.DebugInfo.DebugDataCollector) + private Dictionary AddDebugEnvVariables(Dictionary environmentVariables) + { + var debugVariables = new Dictionary(); + if (environmentVariables != null) { - environmentVariables ??= new Dictionary(); + foreach (var pair in environmentVariables) + { + debugVariables.Add(pair.Key, pair.Value); + } + } + if (_testEnvironment.DebugInfo != null) + { if (_testEnvironment.DebugInfo.DebugVSTestConsole) { - environmentVariables.Add("VSTEST_RUNNER_DEBUG_ATTACHVS", "1"); + debugVariables.Add("VSTEST_RUNNER_DEBUG_ATTACHVS", "1"); } if (_testEnvironment.DebugInfo.DebugTestHost) { - environmentVariables.Add("VSTEST_HOST_DEBUG_ATTACHVS", "1"); + debugVariables.Add("VSTEST_HOST_DEBUG_ATTACHVS", "1"); } if (_testEnvironment.DebugInfo.DebugDataCollector) { - environmentVariables.Add("VSTEST_DATACOLLECTOR_DEBUG_ATTACHVS", "1"); + debugVariables.Add("VSTEST_DATACOLLECTOR_DEBUG_ATTACHVS", "1"); } if (_testEnvironment.DebugInfo.NoDefaultBreakpoints) { - environmentVariables.Add("VSTEST_DEBUG_NOBP", "1"); + debugVariables.Add("VSTEST_DEBUG_NOBP", "1"); } } - InvokeVsTest(arguments, environmentVariables); + return debugVariables; } /// @@ -674,7 +675,10 @@ public IVsTestConsoleWrapper GetVsTestConsoleWrapper() Console.WriteLine($"Console runner path: {consoleRunnerPath}"); VsTestConsoleWrapper vstestConsoleWrapper; - if (_testEnvironment.DebugInfo.DebugVSTestConsole || _testEnvironment.DebugInfo.DebugTestHost || _testEnvironment.DebugInfo.DebugDataCollector) + if (_testEnvironment.DebugInfo != null + && (_testEnvironment.DebugInfo.DebugVSTestConsole + || _testEnvironment.DebugInfo.DebugTestHost + || _testEnvironment.DebugInfo.DebugDataCollector)) { var environmentVariables = new Dictionary(); Environment.GetEnvironmentVariables().OfType().ToList().ForEach(e => environmentVariables.Add(e.Key.ToString(), e.Value.ToString())); @@ -908,7 +912,7 @@ public static void CreateRunSettingsFile(string destinationRunsettingsPath, stri protected string BuildMultipleAssemblyPath(params string[] assetNames) { - return string.Join(" ", GetTestDlls(assetNames)); + return @$"""{string.Join(@""" """, GetTestDlls(assetNames))}"""; } protected static string GetDiagArg(string rootDir) diff --git a/test/Microsoft.TestPlatform.TestUtilities/Microsoft.TestPlatform.TestUtilities.csproj b/test/Microsoft.TestPlatform.TestUtilities/Microsoft.TestPlatform.TestUtilities.csproj index baea232289..527a9a44b3 100644 --- a/test/Microsoft.TestPlatform.TestUtilities/Microsoft.TestPlatform.TestUtilities.csproj +++ b/test/Microsoft.TestPlatform.TestUtilities/Microsoft.TestPlatform.TestUtilities.csproj @@ -10,6 +10,12 @@ netcoreapp3.1 false + + + + + + @@ -22,6 +28,10 @@ + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + diff --git a/test/Microsoft.TestPlatform.TestUtilities/TempDirectory.cs b/test/Microsoft.TestPlatform.TestUtilities/TempDirectory.cs index d005672347..f7353ace6c 100644 --- a/test/Microsoft.TestPlatform.TestUtilities/TempDirectory.cs +++ b/test/Microsoft.TestPlatform.TestUtilities/TempDirectory.cs @@ -2,6 +2,7 @@ // Licensed under the MIT license. See LICENSE file in the project root for full license information. using System; +using System.Collections.Generic; using System.IO; using IO = System.IO; @@ -51,6 +52,36 @@ public void CopyDirectory(DirectoryInfo source, DirectoryInfo target) } } + /// + /// Copy given files into the TempDirectory and return the updated paths that are pointing to TempDirectory. + /// + /// + /// + public string[] CopyFile(params string[] filePaths) + { + var paths = new List(filePaths.Length); + foreach (var filePath in filePaths) + { + var destination = IO.Path.Combine(Path, IO.Path.GetFileName(filePath)); + File.Copy(filePath, destination); + paths.Add(destination); + } + + return paths.ToArray(); + } + + /// + /// Copy given file into TempDirectory and return the updated path. + /// + /// + /// + public string CopyFile(string filePath) + { + var destination = IO.Path.Combine(Path, IO.Path.GetFileName(filePath)); + File.Copy(filePath, destination); + return destination; + } + /// /// Creates an unique temporary directory. /// @@ -71,7 +102,13 @@ private static string GetTempPath() // AGENT_TEMPDIRECTORY is AzureDevops variable, which is set to path // that is cleaned up after every job. This is preferable to use over // just the normal TEMP, because that is not cleaned up for every run. - return Environment.GetEnvironmentVariable("AGENT_TEMPDIRECTORY") ?? IO.Path.GetTempPath(); + // + // System.IO.Path.GetTempPath is banned from the rest of the code. This is the only + // place we we are allowed to use it. All other methods should use our GetTempPath (this method). +#pragma warning disable RS0030 // Do not used banned APIs + return Environment.GetEnvironmentVariable("AGENT_TEMPDIRECTORY") + ?? IO.Path.GetTempPath(); +#pragma warning restore RS0030 // Do not used banned APIs } public static void TryRemoveDirectory(string directory) From 9894fa311d1d7764fbe542c667cfab32c2e4601a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Jare=C5=A1?= Date: Tue, 22 Mar 2022 14:59:07 +0100 Subject: [PATCH 063/112] CI cleanup always --- .../IntegrationTestBase.cs | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestBase.cs b/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestBase.cs index e3aebd8afe..bb5d07d65e 100644 --- a/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestBase.cs +++ b/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestBase.cs @@ -82,8 +82,11 @@ public IntegrationTestBase() [TestCleanup] public void TempDirectoryCleanup() { - // Delete the directory only when the test succeeded, so we can look at results and logs of failed tests. - if (TestContext.CurrentTestOutcome == UnitTestOutcome.Passed) + // In CI always delete the results, because we have limited disk space there. + // + // Locally delete the directory only when the test succeeded, so we can look + // at results and logs of failed tests. + if (IsCI || TestContext.CurrentTestOutcome == UnitTestOutcome.Passed) { TempDirectory.Dispose(); } From 5af74378ecfa71928a1b6b55ddbbbbfbd6477514 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Jare=C5=A1?= Date: Wed, 23 Mar 2022 13:14:27 +0100 Subject: [PATCH 064/112] Print free disk space --- .../IntegrationTestBase.cs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestBase.cs b/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestBase.cs index bb5d07d65e..39b9099afe 100644 --- a/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestBase.cs +++ b/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestBase.cs @@ -62,6 +62,10 @@ public IntegrationTestBase() _testEnvironment = new IntegrationTestEnvironment(); BuildConfiguration = IntegrationTestEnvironment.BuildConfiguration; TempDirectory = new TempDirectory(); + + var drive = new DriveInfo(Directory.GetDirectoryRoot(TempDirectory.Path)); + Console.WriteLine($"Available space for TEMP: {drive.Name} {drive.AvailableFreeSpace / (1024 * 1024)} MB"); + IsCI = IntegrationTestEnvironment.IsCI; } From 81c0b1b97ba6548964e2332e942ee85ce7fdb7ff Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Jare=C5=A1?= Date: Wed, 23 Mar 2022 14:26:47 +0100 Subject: [PATCH 065/112] Whitespace and so on --- .../ExecutionTests.cs | 8 ++------ .../Extension/CompatibilityRowsBuilder.cs | 19 ++++++++----------- .../Extension/Features.cs | 14 +++++++------- .../Extension/NetCoreRunner.cs | 2 +- .../RunnerCompatibilityDataSource.cs | 1 - .../Extension/RunnnerInfo.cs | 2 +- .../DebugInfo.cs | 2 -- .../NetTestSdkInfo.cs | 2 +- .../VSTestConsoleInfo.cs | 2 +- 9 files changed, 21 insertions(+), 31 deletions(-) diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/ExecutionTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/ExecutionTests.cs index c9676ae547..5f5fc4ead8 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/ExecutionTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/ExecutionTests.cs @@ -38,7 +38,6 @@ public void RunMultipleTestAssemblies(RunnerInfo runnerInfo) [TestCategory("Windows-Review")] [TestPlatformCompatibilityDataSource()] //[TestPlatformCompatibilityDataSource(BeforeFeature = Features.ATTACH_DEBUGGER, AfterAdapterFeature = Features.MSTEST_IFRAMEWORK_HANDLE_99)] - public void RunTestsFromMultipleMSTestAssemblies(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); @@ -68,7 +67,6 @@ public void RunMultipleMSTestAssembliesOnVstestConsoleAndTesthostCombinations(Ru ExitCodeEquals(1); // failing tests } - [TestMethod] [TestCategory("Windows-Review")] [RunnerCompatibilityDataSource] @@ -84,7 +82,6 @@ public void RunMultipleMSTestAssembliesOnVstestConsoleAndTesthostCombinations2(R ExitCodeEquals(1); // failing tests } - // TODO: This one mixes different frameworks, I can make it work, but it is worth it? We are going to test // the two respective versions together (e.g. latest xunit and latest mstest), but does using two different test // frameworks have any added value over using 2 mstest dlls? @@ -115,13 +112,12 @@ public void RunMultipleTestAssembliesWithoutTestAdapterPath(RunnerInfo runnerInf public void RunMultipleTestAssembliesInParallel(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - using var tempDir = new TempDirectory(); var assemblyPaths = BuildMultipleAssemblyPath("SimpleTestProject.dll", "SimpleTestProject2.dll"); var arguments = PrepareArguments(assemblyPaths, testAdapterPath: null, runSettings: null, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: tempDir.Path); arguments = string.Concat(arguments, " /Parallel"); arguments = string.Concat(arguments, " /Platform:x86"); - arguments += GetDiagArg(tempDir.Path); + arguments += GetDiagArg(TempDirectory.Path); // for the desktop we will run testhost.x86 in two copies, but for core // we will run a combination of testhost.x86 and dotnet, where the dotnet will be @@ -134,7 +130,7 @@ public void RunMultipleTestAssembliesInParallel(RunnerInfo runnerInfo) InvokeVsTest(arguments); - AssertExpectedNumberOfHostProcesses(expectedNumOfProcessCreated, tempDir.Path, testHostProcessNames); + AssertExpectedNumberOfHostProcesses(expectedNumOfProcessCreated, TempDirectory.Path, testHostProcessNames); ValidateSummaryStatus(2, 2, 2); ExitCodeEquals(1); // failing tests } diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/CompatibilityRowsBuilder.cs b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/CompatibilityRowsBuilder.cs index dccbd03525..3a4184bc9c 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/CompatibilityRowsBuilder.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/CompatibilityRowsBuilder.cs @@ -24,11 +24,11 @@ public class CompatibilityRowsBuilder private readonly string[] _hostVersions; public CompatibilityRowsBuilder(string runnerFrameworks = AcceptanceTestBase.DEFAULT_HOST_NETFX_AND_NET, - string runnerVersions = AcceptanceTestBase.LATEST_TO_LEGACY, - string hostFrameworks = AcceptanceTestBase.DEFAULT_HOST_NETFX_AND_NET, - string hostVersions = AcceptanceTestBase.LATEST_TO_LEGACY, - string adapterVersions = AcceptanceTestBase.LATESTPREVIEW_TO_LEGACY, - string adapters = AcceptanceTestBase.MSTEST) + string runnerVersions = AcceptanceTestBase.LATEST_TO_LEGACY, + string hostFrameworks = AcceptanceTestBase.DEFAULT_HOST_NETFX_AND_NET, + string hostVersions = AcceptanceTestBase.LATEST_TO_LEGACY, + string adapterVersions = AcceptanceTestBase.LATESTPREVIEW_TO_LEGACY, + string adapters = AcceptanceTestBase.MSTEST) { _runnerFrameworks = runnerFrameworks.Split(';'); _runnerVersions = runnerVersions.Split(';'); @@ -197,7 +197,6 @@ private void AddOlderConfigurations(List dataRows) } } - private void AddEveryVersionOfAdapter(List dataRows) { var runnerVersion = _runnerVersions[0]; @@ -270,7 +269,7 @@ private void AddEveryVersionOfRunner(List dataRows) } private void AddRow(List dataRows, string batch, -string runnerVersion, string runnerFramework, string hostVersion, string hostFramework, string adapter, string adapterVersion, bool inIsolation) + string runnerVersion, string runnerFramework, string hostVersion, string hostFramework, string adapter, string adapterVersion, bool inIsolation) { RunnerInfo runnerInfo = GetRunnerInfo(batch, runnerFramework, hostFramework, inIsolation); runnerInfo.DebugInfo = GetDebugInfo(); @@ -298,7 +297,6 @@ private DebugInfo GetDebugInfo() }; } - private RunnerInfo GetRunnerInfo(string batch, string runnerFramework, string hostFramework, bool inIsolation) { return new RunnerInfo @@ -339,7 +337,7 @@ private static VSTestConsoleInfo GetVSTestConsoleInfo(string vstestConsoleVersio // same as other versions, we just need to grab the version from a different property. var propertyName = vstestConsoleVersion == AcceptanceTestBase.LATEST - ? $"NETTestSdkVersion" + ? "NETTestSdkVersion" : $"VSTestConsole{vstestConsoleVersion}Version"; var packageName = runnerInfo.IsNetFrameworkRunner @@ -378,7 +376,7 @@ private static NetTestSdkInfo GetNetTestSdkInfo(string testhostVersionType) // same as other versions, we just need to grab the version from a different property. var propertyName = testhostVersionType == AcceptanceTestBase.LATEST - ? $"NETTestSdkVersion" + ? "NETTestSdkVersion" : $"VSTestConsole{testhostVersionType}Version"; // It is okay when node is null, we check that Version has value when we update paths by using TesthostInfo, and throw. @@ -398,7 +396,6 @@ private static NetTestSdkInfo GetNetTestSdkInfo(string testhostVersionType) }; } - private static XmlDocument GetDependenciesXml() { if (s_depsXml != null) diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/Features.cs b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/Features.cs index b966b9daa3..0be9ac104c 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/Features.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/Features.cs @@ -2,22 +2,22 @@ // Licensed under the MIT license. See LICENSE file in the project root for full license information. using System.Collections.Generic; +using System.Collections.Immutable; namespace Microsoft.TestPlatform.AcceptanceTests; public static class Features { public const string ATTACH_DEBUGGER_FLOW = nameof(ATTACH_DEBUGGER_FLOW); - public const string MSTEST_IFRAMEWORK_HANDLE_99 = nameof(MSTEST_IFRAMEWORK_HANDLE_99); + public const string MSTEST_EXAMPLE_FEATURE = nameof(MSTEST_EXAMPLE_FEATURE); - - public static Dictionary TestPlatformFeatures { get; } = new Dictionary + public static IImmutableDictionary TestPlatformFeatures { get; } = new Dictionary { [ATTACH_DEBUGGER_FLOW] = new(version: "v16.7.0-preview-20200519-01", issue: "https://github.com/microsoft/vstest/pull/2325"), - }; + }.ToImmutableDictionary(); - public static Dictionary AdapterFeatures { get; internal set; } = new Dictionary + public static IImmutableDictionary AdapterFeatures { get; internal set; } = new Dictionary { - [MSTEST_IFRAMEWORK_HANDLE_99] = new("2.2.8", issue: "idk"), - }; + [MSTEST_EXAMPLE_FEATURE] = new("2.2.8", issue: "This feature does not actually exist."), + }.ToImmutableDictionary(); } diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/NetCoreRunner.cs b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/NetCoreRunner.cs index 71db23bdee..202264d0c7 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/NetCoreRunner.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/NetCoreRunner.cs @@ -51,7 +51,7 @@ public IEnumerable GetData(MethodInfo methodInfo) { RunnerFramework = IntegrationTestBase.CoreRunnerFramework, TargetFramework = fmw, - InIsolationValue = null + InIsolationValue = null, }; runnerInfo.DebugInfo = new DebugInfo { diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/RunnerCompatibilityDataSource.cs b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/RunnerCompatibilityDataSource.cs index 3b7b9d10ca..357dcd1070 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/RunnerCompatibilityDataSource.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/RunnerCompatibilityDataSource.cs @@ -5,7 +5,6 @@ namespace Microsoft.TestPlatform.AcceptanceTests; - /// /// A data source that provides every version of runner. /// diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/RunnnerInfo.cs b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/RunnnerInfo.cs index 88f1404005..6e1fefc769 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/RunnnerInfo.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/RunnnerInfo.cs @@ -14,7 +14,7 @@ namespace Microsoft.TestPlatform.AcceptanceTests; /// /// /// Supported value = /InIsolation. -[Serializable] +[Serializable] // Type should be serializable to allow the tree-view behavior of test discovery in Test Explorer public class RunnerInfo { public string? RunnerFramework { get; set; } diff --git a/test/Microsoft.TestPlatform.TestUtilities/DebugInfo.cs b/test/Microsoft.TestPlatform.TestUtilities/DebugInfo.cs index 995855ca1d..f9ac35d770 100644 --- a/test/Microsoft.TestPlatform.TestUtilities/DebugInfo.cs +++ b/test/Microsoft.TestPlatform.TestUtilities/DebugInfo.cs @@ -16,5 +16,3 @@ public sealed class DebugInfo public bool DebugDataCollector { get; set; } public bool NoDefaultBreakpoints { get; set; } = true; } - - diff --git a/test/Microsoft.TestPlatform.TestUtilities/NetTestSdkInfo.cs b/test/Microsoft.TestPlatform.TestUtilities/NetTestSdkInfo.cs index c16d2a3f44..00deda8823 100644 --- a/test/Microsoft.TestPlatform.TestUtilities/NetTestSdkInfo.cs +++ b/test/Microsoft.TestPlatform.TestUtilities/NetTestSdkInfo.cs @@ -5,7 +5,7 @@ namespace Microsoft.TestPlatform.TestUtilities; -[Serializable] +[Serializable] // Type should be serializable to allow the tree-view behavior of test discovery in Test Explorer public class NetTestSdkInfo : DllInfo { public NetTestSdkInfo() diff --git a/test/Microsoft.TestPlatform.TestUtilities/VSTestConsoleInfo.cs b/test/Microsoft.TestPlatform.TestUtilities/VSTestConsoleInfo.cs index e55503afbf..3ba8deabc9 100644 --- a/test/Microsoft.TestPlatform.TestUtilities/VSTestConsoleInfo.cs +++ b/test/Microsoft.TestPlatform.TestUtilities/VSTestConsoleInfo.cs @@ -5,7 +5,7 @@ namespace Microsoft.TestPlatform.TestUtilities; -[Serializable] +[Serializable] // Type should be serializable to allow the tree-view behavior of test discovery in Test Explorer public class VSTestConsoleInfo { public string? VersionType { get; set; } From eff69def0b3da3fe50f593c25152f4803a306e0e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Jare=C5=A1?= Date: Wed, 23 Mar 2022 14:29:46 +0100 Subject: [PATCH 066/112] Missing temp dir --- test/Microsoft.TestPlatform.AcceptanceTests/ExecutionTests.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/ExecutionTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/ExecutionTests.cs index 5f5fc4ead8..ead92079dc 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/ExecutionTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/ExecutionTests.cs @@ -114,7 +114,7 @@ public void RunMultipleTestAssembliesInParallel(RunnerInfo runnerInfo) SetTestEnvironment(_testEnvironment, runnerInfo); var assemblyPaths = BuildMultipleAssemblyPath("SimpleTestProject.dll", "SimpleTestProject2.dll"); - var arguments = PrepareArguments(assemblyPaths, testAdapterPath: null, runSettings: null, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: tempDir.Path); + var arguments = PrepareArguments(assemblyPaths, testAdapterPath: null, runSettings: null, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: TempDirectory.Path); arguments = string.Concat(arguments, " /Parallel"); arguments = string.Concat(arguments, " /Platform:x86"); arguments += GetDiagArg(TempDirectory.Path); From 5f3f5c2836adfcf03180623c42b1e3db4e84381b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Jare=C5=A1?= Date: Wed, 23 Mar 2022 15:29:04 +0100 Subject: [PATCH 067/112] Fix env variables --- .../TranslationLayerTests/CodeCoverageTests.cs | 1 + .../IntegrationTestBase.cs | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/CodeCoverageTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/CodeCoverageTests.cs index c77faab8ab..0b56578e3f 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/CodeCoverageTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/CodeCoverageTests.cs @@ -183,6 +183,7 @@ await _vstestConsoleWrapper.ProcessTestRunAttachmentsAsync( [NetCoreTargetFrameworkDataSource] public async Task TestRunWithCodeCoverageAndAttachmentsProcessingNoMetrics(RunnerInfo runnerInfo) { + // System.Environment.SetEnvironmentVariable("VSTEST_RUNNER_DEBUG_ATTACHVS", "1"); // arrange SetTestEnvironment(_testEnvironment, runnerInfo); Setup(); diff --git a/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestBase.cs b/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestBase.cs index 39b9099afe..013cad3096 100644 --- a/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestBase.cs +++ b/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestBase.cs @@ -696,14 +696,14 @@ public IVsTestConsoleWrapper GetVsTestConsoleWrapper() Environment.GetEnvironmentVariables().OfType().ToList().ForEach(e => environmentVariables.Add(e.Key.ToString(), e.Value.ToString())); foreach (var pair in debugEnvironmentVariables) { - environmentVariables.Add(pair.Key, pair.Value); + environmentVariables[pair.Key] = pair.Value; } } if (environmentVariables.Count > 0) { // This clears all variables, so we copy all environment variables, and add the debug ones to them. - vstestConsoleWrapper = new VsTestConsoleWrapper(consoleRunnerPath, dotnetPath, new ConsoleParameters() { LogFilePath = logFilePath, EnvironmentVariables = debugEnvironmentVariables }); + vstestConsoleWrapper = new VsTestConsoleWrapper(consoleRunnerPath, dotnetPath, new ConsoleParameters() { LogFilePath = logFilePath, EnvironmentVariables = environmentVariables }); } else { From a1a461c6a1b7f660dcb2cbbb8a9da0f13d242a46 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Jare=C5=A1?= Date: Wed, 23 Mar 2022 16:04:58 +0100 Subject: [PATCH 068/112] Fixes I feel like I did before --- .../TranslationLayerTests/RunTests.cs | 1 + .../TranslationLayerTests/RunTestsWithFilterTests.cs | 1 + .../Microsoft.TestPlatform.TestUtilities/IntegrationTestBase.cs | 2 +- 3 files changed, 3 insertions(+), 1 deletion(-) diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/RunTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/RunTests.cs index ffe2449e15..62830b4d29 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/RunTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/RunTests.cs @@ -40,6 +40,7 @@ public void Cleanup() } [TestMethod] + [TestCategory("Windows-Review")] [RunnerCompatibilityDataSource] public void RunAllTests(RunnerInfo runnerInfo) { diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/RunTestsWithFilterTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/RunTestsWithFilterTests.cs index ea4c38ae0f..78039d80d3 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/RunTestsWithFilterTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/RunTestsWithFilterTests.cs @@ -35,6 +35,7 @@ public void Cleanup() } [TestMethod] + [TestCategory("Windows-Review")] [RunnerCompatibilityDataSource] public void RunTestsWithTestCaseFilter(RunnerInfo runnerInfo) { diff --git a/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestBase.cs b/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestBase.cs index 013cad3096..3bf607b9a1 100644 --- a/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestBase.cs +++ b/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestBase.cs @@ -911,7 +911,7 @@ public static void CreateRunSettingsFile(string destinationRunsettingsPath, stri protected string BuildMultipleAssemblyPath(params string[] assetNames) { - return string.Join(" ", GetTestDlls(assetNames)); + return $"\"{string.Join("\" \"", GetTestDlls(assetNames))}\""; } protected static string GetDiagArg(string rootDir) From 5a19d194cf193adf5e91e17c77e2054d2e4da5ee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Jare=C5=A1?= Date: Wed, 23 Mar 2022 16:37:09 +0100 Subject: [PATCH 069/112] Touch --- test/Microsoft.TestPlatform.AcceptanceTests/ExecutionTests.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/ExecutionTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/ExecutionTests.cs index ead92079dc..f3b7645b4c 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/ExecutionTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/ExecutionTests.cs @@ -16,7 +16,7 @@ namespace Microsoft.TestPlatform.AcceptanceTests; [TestClass] public class ExecutionTests : AcceptanceTestBase { - // TODO: It looks like the first 3 tests would be useful to multiply by all 3 test frameworks, should we make the test even more generic, or duplicate them? + //TODO: It looks like the first 3 tests would be useful to multiply by all 3 test frameworks, should we make the test even more generic, or duplicate them? [TestMethod] [TestCategory("Windows-Review")] [MSTestCompatibilityDataSource(InProcess = true)] From 0075b0bf901f37282b45d30898f72c6ca696ceb6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Jare=C5=A1?= Date: Thu, 24 Mar 2022 13:59:24 +0100 Subject: [PATCH 070/112] Remove comment, and revert back to using TempDirectory --- .../ExecutionTests.cs | 41 +++++++------------ 1 file changed, 14 insertions(+), 27 deletions(-) diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/ExecutionTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/ExecutionTests.cs index f3b7645b4c..bf79c780ae 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/ExecutionTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/ExecutionTests.cs @@ -37,7 +37,6 @@ public void RunMultipleTestAssemblies(RunnerInfo runnerInfo) [TestMethod] [TestCategory("Windows-Review")] [TestPlatformCompatibilityDataSource()] - //[TestPlatformCompatibilityDataSource(BeforeFeature = Features.ATTACH_DEBUGGER, AfterAdapterFeature = Features.MSTEST_IFRAMEWORK_HANDLE_99)] public void RunTestsFromMultipleMSTestAssemblies(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); @@ -141,11 +140,10 @@ public void RunMultipleTestAssembliesInParallel(RunnerInfo runnerInfo) public void TestSessionTimeOutTests(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - using var tempDir = new TempDirectory(); var assemblyPaths = BuildMultipleAssemblyPath("SimpleTestProject3.dll"); - var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: tempDir.Path); + var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: TempDirectory.Path); arguments = string.Concat(arguments, " /TestCaseFilter:TestSessionTimeoutTest"); // set TestSessionTimeOut = 7 sec @@ -163,10 +161,9 @@ public void TestSessionTimeOutTests(RunnerInfo runnerInfo) public void TestPlatformShouldBeCompatibleWithOldTestHost(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - using var tempDir = new TempDirectory(); var assemblyPaths = BuildMultipleAssemblyPath("SampleProjectWithOldTestHost.dll"); - var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: tempDir.Path); + var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: TempDirectory.Path); InvokeVsTest(arguments); @@ -180,10 +177,9 @@ public void TestPlatformShouldBeCompatibleWithOldTestHost(RunnerInfo runnerInfo) public void WorkingDirectoryIsSourceDirectory(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - using var tempDir = new TempDirectory(); var assemblyPaths = BuildMultipleAssemblyPath("SimpleTestProject3.dll"); - var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: tempDir.Path); + var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: TempDirectory.Path); arguments = string.Concat(arguments, " /tests:WorkingDirectoryTest"); InvokeVsTest(arguments); @@ -198,7 +194,6 @@ public void WorkingDirectoryIsSourceDirectory(RunnerInfo runnerInfo) public void StackOverflowExceptionShouldBeLoggedToConsoleAndDiagLogFile(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - using var tempDir = new TempDirectory(); if (IntegrationTestEnvironment.BuildConfiguration.Equals("release", StringComparison.OrdinalIgnoreCase)) { @@ -207,11 +202,11 @@ public void StackOverflowExceptionShouldBeLoggedToConsoleAndDiagLogFile(RunnerIn return; } - var diagLogFilePath = Path.Combine(tempDir.Path, $"std_error_log_{Guid.NewGuid()}.txt"); + var diagLogFilePath = Path.Combine(TempDirectory.Path, $"std_error_log_{Guid.NewGuid()}.txt"); File.Delete(diagLogFilePath); var assemblyPaths = BuildMultipleAssemblyPath("SimpleTestProject3.dll"); - var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: tempDir.Path); + var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: TempDirectory.Path); arguments = string.Concat(arguments, " /testcasefilter:ExitWithStackoverFlow"); arguments = string.Concat(arguments, $" /diag:{diagLogFilePath}"); @@ -235,14 +230,13 @@ public void StackOverflowExceptionShouldBeLoggedToConsoleAndDiagLogFile(RunnerIn public void UnhandleExceptionExceptionShouldBeLoggedToDiagLogFile(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - using var tempDir = new TempDirectory(); - var diagLogFilePath = Path.Combine(tempDir.Path, $"std_error_log_{Guid.NewGuid()}.txt"); + var diagLogFilePath = Path.Combine(TempDirectory.Path, $"std_error_log_{Guid.NewGuid()}.txt"); File.Delete(diagLogFilePath); var assemblyPaths = BuildMultipleAssemblyPath("SimpleTestProject3.dll"); - var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: tempDir.Path); + var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: TempDirectory.Path); arguments = string.Concat(arguments, " /testcasefilter:ExitwithUnhandleException"); arguments = string.Concat(arguments, $" /diag:{diagLogFilePath}"); @@ -259,12 +253,11 @@ public void UnhandleExceptionExceptionShouldBeLoggedToDiagLogFile(RunnerInfo run public void IncompatibleSourcesWarningShouldBeDisplayedInTheConsole(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - using var tempDir = new TempDirectory(); var expectedWarningContains = @"Following DLL(s) do not match current settings, which are .NETFramework,Version=v4.5.1 framework and X86 platform. SimpleTestProject3.dll is built for Framework .NETFramework,Version=v4.5.1 and Platform X64"; var assemblyPaths = BuildMultipleAssemblyPath("SimpleTestProject3.dll", "SimpleTestProjectx86.dll"); - var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: tempDir.Path); + var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: TempDirectory.Path); arguments = string.Concat(arguments, " /testcasefilter:PassingTestx86"); InvokeVsTest(arguments); @@ -282,12 +275,11 @@ public void IncompatibleSourcesWarningShouldBeDisplayedInTheConsole(RunnerInfo r public void NoIncompatibleSourcesWarningShouldBeDisplayedInTheConsole(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - using var tempDir = new TempDirectory(); var expectedWarningContains = @"Following DLL(s) do not match current settings, which are .NETFramework,Version=v4.5.1 framework and X86 platform. SimpleTestProjectx86 is built for Framework .NETFramework,Version=v4.5.1 and Platform X86"; var assemblyPaths = BuildMultipleAssemblyPath("SimpleTestProjectx86.dll"); - var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: tempDir.Path); + var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: TempDirectory.Path); InvokeVsTest(arguments); @@ -303,12 +295,11 @@ public void NoIncompatibleSourcesWarningShouldBeDisplayedInTheConsole(RunnerInfo public void IncompatibleSourcesWarningShouldBeDisplayedInTheConsoleOnlyWhenRunningIn32BitOS(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - using var tempDir = new TempDirectory(); var expectedWarningContains = @"Following DLL(s) do not match current settings, which are .NETFramework,Version=v4.5.1 framework and X86 platform. SimpleTestProject2.dll is built for Framework .NETFramework,Version=v4.5.1 and Platform X64"; var assemblyPaths = BuildMultipleAssemblyPath("SimpleTestProject2.dll"); - var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: tempDir.Path); + var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: TempDirectory.Path); InvokeVsTest(arguments); @@ -333,11 +324,10 @@ public void IncompatibleSourcesWarningShouldBeDisplayedInTheConsoleOnlyWhenRunni public void ExitCodeShouldReturnOneWhenTreatNoTestsAsErrorParameterSetToTrueAndNoTestMatchesFilter(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - using var tempDir = new TempDirectory(); var assemblyPaths = BuildMultipleAssemblyPath("SimpleTestProject2.dll"); - var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: tempDir.Path); + var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: TempDirectory.Path); // Setting /TestCaseFilter to the test name, which does not exists in the assembly, so we will have 0 tests executed arguments = string.Concat(arguments, " /TestCaseFilter:TestNameThatMatchesNoTestInTheAssembly"); @@ -354,10 +344,9 @@ public void ExitCodeShouldReturnOneWhenTreatNoTestsAsErrorParameterSetToTrueAndN public void ExitCodeShouldReturnZeroWhenTreatNoTestsAsErrorParameterSetToFalseAndNoTestMatchesFilter(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - using var tempDir = new TempDirectory(); var assemblyPaths = BuildMultipleAssemblyPath("SimpleTestProject2.dll"); - var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: tempDir.Path); + var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: TempDirectory.Path); // Setting /TestCaseFilter to the test name, which does not exists in the assembly, so we will have 0 tests executed arguments = string.Concat(arguments, " /TestCaseFilter:TestNameThatMatchesNoTestInTheAssembly"); @@ -374,11 +363,10 @@ public void ExitCodeShouldReturnZeroWhenTreatNoTestsAsErrorParameterSetToFalseAn public void ExitCodeShouldNotDependOnTreatNoTestsAsErrorTrueValueWhenThereAreAnyTestsToRun(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - using var tempDir = new TempDirectory(); var assemblyPaths = BuildMultipleAssemblyPath("SimpleTestProject2.dll"); - var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: tempDir.Path); + var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: TempDirectory.Path); arguments = string.Concat(arguments, " -- RunConfiguration.TreatNoTestsAsError=true"); InvokeVsTest(arguments); @@ -393,10 +381,9 @@ public void ExitCodeShouldNotDependOnTreatNoTestsAsErrorTrueValueWhenThereAreAny public void ExitCodeShouldNotDependOnFailTreatNoTestsAsErrorFalseValueWhenThereAreAnyTestsToRun(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - using var tempDir = new TempDirectory(); var assemblyPaths = BuildMultipleAssemblyPath("SimpleTestProject2.dll"); - var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: tempDir.Path); + var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: TempDirectory.Path); arguments = string.Concat(arguments, " -- RunConfiguration.TreatNoTestsAsError=false"); InvokeVsTest(arguments); From 85085dc2f0c0ff2f51906acb961597da19116b09 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Jare=C5=A1?= Date: Thu, 24 Mar 2022 14:09:39 +0100 Subject: [PATCH 071/112] Fix quoting --- test/Microsoft.TestPlatform.AcceptanceTests/ExecutionTests.cs | 3 ++- .../IntegrationTestBase.cs | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/ExecutionTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/ExecutionTests.cs index bf79c780ae..57dc40b3a8 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/ExecutionTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/ExecutionTests.cs @@ -6,6 +6,7 @@ using Microsoft.TestPlatform.TestUtilities; using Microsoft.VisualStudio.TestTools.UnitTesting; +using Microsoft.VisualStudio.TestPlatform.CoreUtilities.Extensions; using TestPlatform.TestUtilities; @@ -96,7 +97,7 @@ public void RunMultipleTestAssembliesWithoutTestAdapterPath(RunnerInfo runnerInf _testEnvironment.GetTestAsset("XUTestProject.dll", "net46") : _testEnvironment.GetTestAsset("XUTestProject.dll"); - assemblyPaths = string.Concat(assemblyPaths, "\" \"", xunitAssemblyPath); + assemblyPaths = string.Join(" ", assemblyPaths, xunitAssemblyPath.AddDoubleQuote()); InvokeVsTestForExecution(assemblyPaths, testAdapterPath: string.Empty, FrameworkArgValue, string.Empty); ValidateSummaryStatus(2, 2, 1); diff --git a/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestBase.cs b/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestBase.cs index 3bf607b9a1..ae05021e2a 100644 --- a/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestBase.cs +++ b/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestBase.cs @@ -911,7 +911,8 @@ public static void CreateRunSettingsFile(string destinationRunsettingsPath, stri protected string BuildMultipleAssemblyPath(params string[] assetNames) { - return $"\"{string.Join("\" \"", GetTestDlls(assetNames))}\""; + // Double quoted sources sepearated by space. + return string.Join(" ", GetTestDlls(assetNames).Select(a => a.AddDoubleQuote())); } protected static string GetDiagArg(string rootDir) From efc810fb10aa98743cf87ccd94f1f50eb0c22a28 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Jare=C5=A1?= Date: Thu, 24 Mar 2022 14:38:33 +0100 Subject: [PATCH 072/112] Use Nuget.Versioning rather than semver, and revert strong name warning ignore for test projects --- scripts/build/TestPlatform.Settings.targets | 3 +- .../Extension/CompatibilityRowsBuilder.cs | 39 +++++++++---------- ...rosoft.TestPlatform.AcceptanceTests.csproj | 2 +- 3 files changed, 21 insertions(+), 23 deletions(-) diff --git a/scripts/build/TestPlatform.Settings.targets b/scripts/build/TestPlatform.Settings.targets index 3c1e0d7ef9..b297c84ab9 100644 --- a/scripts/build/TestPlatform.Settings.targets +++ b/scripts/build/TestPlatform.Settings.targets @@ -77,8 +77,7 @@ false - - $(NoWarn);MSB3270;CS8002 + $(NoWarn);MSB3270 diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/CompatibilityRowsBuilder.cs b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/CompatibilityRowsBuilder.cs index 3a4184bc9c..157b2a14b0 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/CompatibilityRowsBuilder.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/CompatibilityRowsBuilder.cs @@ -6,11 +6,10 @@ using System.IO; using System.Linq; using System.Xml; +using NuGet.Versioning; using Microsoft.TestPlatform.TestUtilities; -using Semver; - namespace Microsoft.TestPlatform.AcceptanceTests; public class CompatibilityRowsBuilder @@ -79,49 +78,49 @@ public List CreateData() if (WithInProcess) AddInProcess(dataRows); - var minVersion = SemVersion.Parse("0.0.0-alpha.1"); - var maxVersion = SemVersion.Parse("9999.0.0"); - SemVersion? beforeRunnerVersion = maxVersion; - SemVersion? afterRunnerVersion = minVersion; - SemVersion? beforeTestHostVersion = maxVersion; - SemVersion? afterTestHostVersion = minVersion; - SemVersion? beforeAdapterVersion = maxVersion; - SemVersion? afterAdapterVersion = minVersion; + var minVersion = SemanticVersion.Parse("0.0.0-alpha.1"); + var maxVersion = SemanticVersion.Parse("9999.0.0"); + SemanticVersion? beforeRunnerVersion = maxVersion; + SemanticVersion? afterRunnerVersion = minVersion; + SemanticVersion? beforeTestHostVersion = maxVersion; + SemanticVersion? afterTestHostVersion = minVersion; + SemanticVersion? beforeAdapterVersion = maxVersion; + SemanticVersion? afterAdapterVersion = minVersion; if (BeforeRunnerFeature != null) { var feature = Features.TestPlatformFeatures[BeforeRunnerFeature]; - beforeRunnerVersion = SemVersion.Parse(feature.Version.TrimStart('v')); + beforeRunnerVersion = SemanticVersion.Parse(feature.Version.TrimStart('v')); } if (AfterRunnerFeature != null) { var feature = Features.TestPlatformFeatures[AfterRunnerFeature]; - afterRunnerVersion = SemVersion.Parse(feature.Version.TrimStart('v')); + afterRunnerVersion = SemanticVersion.Parse(feature.Version.TrimStart('v')); } if (BeforeTestHostFeature != null) { var feature = Features.TestPlatformFeatures[BeforeTestHostFeature]; - beforeTestHostVersion = SemVersion.Parse(feature.Version.TrimStart('v')); + beforeTestHostVersion = SemanticVersion.Parse(feature.Version.TrimStart('v')); } if (AfterTestHostFeature != null) { var feature = Features.TestPlatformFeatures[AfterTestHostFeature]; - afterTestHostVersion = SemVersion.Parse(feature.Version.TrimStart('v')); + afterTestHostVersion = SemanticVersion.Parse(feature.Version.TrimStart('v')); } if (BeforeAdapterFeature != null) { var feature = Features.TestPlatformFeatures[BeforeAdapterFeature]; - beforeAdapterVersion = SemVersion.Parse(feature.Version.TrimStart('v')); + beforeAdapterVersion = SemanticVersion.Parse(feature.Version.TrimStart('v')); } if (AfterAdapterFeature != null) { var feature = Features.AdapterFeatures[AfterAdapterFeature]; - afterAdapterVersion = SemVersion.Parse(feature.Version.TrimStart('v')); + afterAdapterVersion = SemanticVersion.Parse(feature.Version.TrimStart('v')); } var isWindows = Environment.OSVersion.Platform.ToString().StartsWith("Win"); @@ -133,12 +132,12 @@ public List CreateData() // We probably don't have that need right now, because legacy version is 15.x.x, which is very old, and we are still keeping // compatibility. - Func isInRange = (version, before, after) => version < before && after < version; + Func isInRange = (version, before, after) => version < before && after < version; var rows = dataRows.Where(r => r.VSTestConsoleInfo != null - && isInRange(r.VSTestConsoleInfo.Version, beforeRunnerVersion, afterRunnerVersion) - && r.TestHostInfo != null && isInRange(r.TestHostInfo.Version, beforeTestHostVersion, afterTestHostVersion) - && r.AdapterInfo != null && isInRange(r.AdapterInfo.Version, beforeAdapterVersion, afterAdapterVersion)).ToList(); + && isInRange(SemanticVersion.Parse(r.VSTestConsoleInfo.Version), beforeRunnerVersion, afterRunnerVersion) + && r.TestHostInfo != null && isInRange(SemanticVersion.Parse(r.TestHostInfo.Version), beforeTestHostVersion, afterTestHostVersion) + && r.AdapterInfo != null && isInRange(SemanticVersion.Parse(r.AdapterInfo.Version), beforeAdapterVersion, afterAdapterVersion)).ToList(); // We use ToString to determine which values are unique. Not great solution, but works better than using records. var distinctRows = new Dictionary(); diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/Microsoft.TestPlatform.AcceptanceTests.csproj b/test/Microsoft.TestPlatform.AcceptanceTests/Microsoft.TestPlatform.AcceptanceTests.csproj index c33244d042..ef1e78ee2c 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/Microsoft.TestPlatform.AcceptanceTests.csproj +++ b/test/Microsoft.TestPlatform.AcceptanceTests/Microsoft.TestPlatform.AcceptanceTests.csproj @@ -36,7 +36,7 @@ - + From 02c7d41917ecffb378cb6e103a76fdea88a99c8c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Jare=C5=A1?= Date: Mon, 28 Mar 2022 14:59:40 +0200 Subject: [PATCH 073/112] Fix tests --- test/Microsoft.TestPlatform.AcceptanceTests/ExecutionTests.cs | 2 +- .../TranslationLayerTests/DiscoverTests.cs | 4 ++-- .../TranslationLayerTests/RunTests.cs | 2 +- .../TranslationLayerTests/RunTestsWithFilterTests.cs | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/ExecutionTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/ExecutionTests.cs index 57dc40b3a8..037ec5deda 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/ExecutionTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/ExecutionTests.cs @@ -113,7 +113,7 @@ public void RunMultipleTestAssembliesInParallel(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - var assemblyPaths = BuildMultipleAssemblyPath("SimpleTestProject.dll", "SimpleTestProject2.dll"); + var assemblyPaths = BuildMultipleAssemblyPath("MSTestProject1.dll", "MSTestProject2.dll"); var arguments = PrepareArguments(assemblyPaths, testAdapterPath: null, runSettings: null, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: TempDirectory.Path); arguments = string.Concat(arguments, " /Parallel"); arguments = string.Concat(arguments, " /Platform:x86"); diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/DiscoverTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/DiscoverTests.cs index 9280597fc6..a227871e1b 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/DiscoverTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/DiscoverTests.cs @@ -51,7 +51,7 @@ public void DiscoverTestsUsingDiscoveryEventHandler1(RunnerInfo runnerInfo) _discoveryEventHandler2 = new DiscoveryEventHandler2(); var vstestConsoleWrapper = GetVsTestConsoleWrapper(); - vstestConsoleWrapper.DiscoverTests(GetTestAssemblies(), GetDefaultRunSettings(), _discoveryEventHandler); + vstestConsoleWrapper.DiscoverTests(GetTestDlls("MSTestProject1.dll", "MSTestProject2.dll"), GetDefaultRunSettings(), _discoveryEventHandler); // Assert. Assert.AreEqual(6, _discoveryEventHandler.DiscoveredTestCases.Count); @@ -70,7 +70,7 @@ public void DiscoverTestsUsingDiscoveryEventHandler2AndTelemetryOptedOut(RunnerI var vstestConsoleWrapper = GetVsTestConsoleWrapper(); vstestConsoleWrapper.DiscoverTests( - GetTestAssemblies(), + GetTestDlls("MSTestProject1.dll", "MSTestProject2.dll"), GetDefaultRunSettings(), new TestPlatformOptions() { CollectMetrics = false }, _discoveryEventHandler2); diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/RunTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/RunTests.cs index 62830b4d29..b17f17b57a 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/RunTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/RunTests.cs @@ -48,7 +48,7 @@ public void RunAllTests(RunnerInfo runnerInfo) var vstestConsoleWrapper = GetVsTestConsoleWrapper(); var runEventHandler = new RunEventHandler(); - vstestConsoleWrapper.RunTests(GetTestAssemblies(), GetDefaultRunSettings(), runEventHandler); + vstestConsoleWrapper.RunTests(GetTestDlls("MSTestProject1.dll", "MSTestProject2.dll"), GetDefaultRunSettings(), runEventHandler); // Assert Assert.AreEqual(6, runEventHandler.TestResults.Count); diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/RunTestsWithFilterTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/RunTestsWithFilterTests.cs index 78039d80d3..6c1b41fac1 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/RunTestsWithFilterTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/RunTestsWithFilterTests.cs @@ -45,7 +45,7 @@ public void RunTestsWithTestCaseFilter(RunnerInfo runnerInfo) _runEventHandler = new RunEventHandler(); var vstestConsoleWrapper = GetVsTestConsoleWrapper(); - var sources = new List { GetAssetFullPath("SimpleTestProject.dll") }; + var sources = new List { GetAssetFullPath("MSTestProject1.dll") }; vstestConsoleWrapper.RunTests( sources, From 57be8db00b5f540f81a99c488fd9b1099b072ad0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Jare=C5=A1?= Date: Mon, 28 Mar 2022 16:16:02 +0200 Subject: [PATCH 074/112] Fix test --- .../TranslationLayerTests/RunTestsWithFilterTests.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/RunTestsWithFilterTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/RunTestsWithFilterTests.cs index 78039d80d3..6c1b41fac1 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/RunTestsWithFilterTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/RunTestsWithFilterTests.cs @@ -45,7 +45,7 @@ public void RunTestsWithTestCaseFilter(RunnerInfo runnerInfo) _runEventHandler = new RunEventHandler(); var vstestConsoleWrapper = GetVsTestConsoleWrapper(); - var sources = new List { GetAssetFullPath("SimpleTestProject.dll") }; + var sources = new List { GetAssetFullPath("MSTestProject1.dll") }; vstestConsoleWrapper.RunTests( sources, From ab3093f71aaed62df068d39a9b19508958ab94a0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Jare=C5=A1?= Date: Mon, 28 Mar 2022 16:18:18 +0200 Subject: [PATCH 075/112] Fix --- .../TranslationLayerTests/RunTestsWithFilterTests.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/RunTestsWithFilterTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/RunTestsWithFilterTests.cs index 6c1b41fac1..162ed10bde 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/RunTestsWithFilterTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/RunTestsWithFilterTests.cs @@ -50,7 +50,7 @@ public void RunTestsWithTestCaseFilter(RunnerInfo runnerInfo) vstestConsoleWrapper.RunTests( sources, GetDefaultRunSettings(), - new TestPlatformOptions() { TestCaseFilter = "FullyQualifiedName=SampleUnitTestProject.UnitTest1.PassingTest" }, + new TestPlatformOptions() { TestCaseFilter = "FullyQualifiedName=MSTestProject1.UnitTest1.PassingTest" }, _runEventHandler); // Assert From d33a3f2c69a38bf12f8bf0d96b0420dd1cdd2a7b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Jare=C5=A1?= Date: Mon, 28 Mar 2022 16:56:11 +0200 Subject: [PATCH 076/112] Fix build --- scripts/build.ps1 | 35 +++------------------------- scripts/common.lib.ps1 | 2 +- src/package/external/external.csproj | 2 +- test/TestAssets/TestAssets.sln | 16 ++++++++++++- 4 files changed, 20 insertions(+), 35 deletions(-) diff --git a/scripts/build.ps1 b/scripts/build.ps1 index f9e8cfa78c..6da1ff53d4 100644 --- a/scripts/build.ps1 +++ b/scripts/build.ps1 @@ -198,40 +198,11 @@ function Invoke-TestAssetsBuild { throw "VSTestConsoleVersion for $propertyName is empty." } - $packages = @( - "Microsoft.TestPlatform" - "Microsoft.TestPlatform.CLI", - "Microsoft.TestPlatform.TranslationLayer" - "Microsoft.NET.Test.SDK" - ) - - foreach ($package in $packages) { - $packagePath = "$env:TP_ROOT_DIR\packages\$($package.ToLower())" - $cachePath = "$packagePath\$vstestConsoleVersion" - - if ((Test-Path -Path $cachePath) -and (Get-ChildItem $cachePath)) { - "Package $package $vsTestConsoleVersion is already in nuget cache at $cachePath." - continue - } - - Invoke-Exe $nugetExe -Arguments "install $package -Version $vsTestConsoleVersion -OutputDirectory $packagePath -ConfigFile $nugetConfig" - - # Install puts it in packages/microsoft.testplatform/Microsoft.TestPlatform.17.1.0, - # because we use that as our output folder. And it also caches it in packages/microsoft.testplatform/17.1.0 - # unless the package is from local source, then it does not do that. So we need to rename the folder and remove - # the original one. - if (-not (Test-Path -Path $cachePath) -or -not (Get-ChildItem $cachePath)) { - Rename-Item "$packagePath\$package.$vsTestConsoleVersion" $cachePath - # Nuget locks the locally copied package from time to time. - Start-Sleep -Milliseconds 300 - } - if (Test-Path "$packagePath\$package.$vsTestConsoleVersion") { - Remove-Item -Recurse -Force "$packagePath\$package.$vsTestConsoleVersion" - } - } + # We don't use the results of this build anywhere, we just use them to restore the packages to nuget cache + # because using nuget.exe install errors out in various weird ways. + Invoke-Exe $dotnetExe -Arguments "build $env:TP_ROOT_DIR\test\TestAssets\Tools\Tools.csproj --configuration $TPB_Configuration -v:minimal -p:CIBuild=$TPB_CIBuild -p:LocalizedBuild=$TPB_LocalizedBuild -p:NETTestSdkVersion=$vsTestConsoleVersion" } - # Build with multiple versions of MSTest. The projects are directly in the root. # The folder structure in VS is not echoed in the TestAssets directory. $projects = @( diff --git a/scripts/common.lib.ps1 b/scripts/common.lib.ps1 index 34fd49a1e8..38b98bd805 100644 --- a/scripts/common.lib.ps1 +++ b/scripts/common.lib.ps1 @@ -47,7 +47,7 @@ Write-Verbose "Setup dotnet configuration." $env:DOTNET_SKIP_FIRST_TIME_EXPERIENCE = 1 # Dotnet build doesn't support --packages yet. See https://github.com/dotnet/cli/issues/2712 $env:NUGET_PACKAGES = $env:TP_PACKAGES_DIR -$env:NUGET_EXE_Version = "5.8.1" +$env:NUGET_EXE_Version = "6.0.0" $env:DOTNET_CLI_VERSION = $GlobalJson.tools.dotnet # $env:DOTNET_RUNTIME_VERSION = "LATEST" $env:VSWHERE_VERSION = "2.0.2" diff --git a/src/package/external/external.csproj b/src/package/external/external.csproj index ebead5a9dc..4a0e70027d 100644 --- a/src/package/external/external.csproj +++ b/src/package/external/external.csproj @@ -33,7 +33,7 @@ - + diff --git a/test/TestAssets/TestAssets.sln b/test/TestAssets/TestAssets.sln index 1a05bdef96..496762cf71 100644 --- a/test/TestAssets/TestAssets.sln +++ b/test/TestAssets/TestAssets.sln @@ -98,10 +98,12 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "SimpleTestProjectMessedUpTa EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "MSTestProject2", "MSTestProject2\MSTestProject2.csproj", "{10AA955C-B412-41A8-899F-8609AAE19F61}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "MSTestProject1", "MSTestProject1\MSTestProject1.csproj", "{E166D337-4033-4209-863F-8F77675EAEE8}" +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "MSTestProject1", "MSTestProject1\MSTestProject1.csproj", "{E166D337-4033-4209-863F-8F77675EAEE8}" EndProject Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "basic", "basic", "{2633D125-64A7-456C-AD37-F8A6B56C2403}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Tools", "Tools\Tools.csproj", "{85F9F2A8-D2A5-4EA1-8BE0-06CCE141EC7A}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -664,6 +666,18 @@ Global {E166D337-4033-4209-863F-8F77675EAEE8}.Release|x64.Build.0 = Release|Any CPU {E166D337-4033-4209-863F-8F77675EAEE8}.Release|x86.ActiveCfg = Release|Any CPU {E166D337-4033-4209-863F-8F77675EAEE8}.Release|x86.Build.0 = Release|Any CPU + {85F9F2A8-D2A5-4EA1-8BE0-06CCE141EC7A}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {85F9F2A8-D2A5-4EA1-8BE0-06CCE141EC7A}.Debug|Any CPU.Build.0 = Debug|Any CPU + {85F9F2A8-D2A5-4EA1-8BE0-06CCE141EC7A}.Debug|x64.ActiveCfg = Debug|Any CPU + {85F9F2A8-D2A5-4EA1-8BE0-06CCE141EC7A}.Debug|x64.Build.0 = Debug|Any CPU + {85F9F2A8-D2A5-4EA1-8BE0-06CCE141EC7A}.Debug|x86.ActiveCfg = Debug|Any CPU + {85F9F2A8-D2A5-4EA1-8BE0-06CCE141EC7A}.Debug|x86.Build.0 = Debug|Any CPU + {85F9F2A8-D2A5-4EA1-8BE0-06CCE141EC7A}.Release|Any CPU.ActiveCfg = Release|Any CPU + {85F9F2A8-D2A5-4EA1-8BE0-06CCE141EC7A}.Release|Any CPU.Build.0 = Release|Any CPU + {85F9F2A8-D2A5-4EA1-8BE0-06CCE141EC7A}.Release|x64.ActiveCfg = Release|Any CPU + {85F9F2A8-D2A5-4EA1-8BE0-06CCE141EC7A}.Release|x64.Build.0 = Release|Any CPU + {85F9F2A8-D2A5-4EA1-8BE0-06CCE141EC7A}.Release|x86.ActiveCfg = Release|Any CPU + {85F9F2A8-D2A5-4EA1-8BE0-06CCE141EC7A}.Release|x86.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE From 2ac14cb6b950b89a2e18df16bc904d833c981080 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Jare=C5=A1?= Date: Mon, 28 Mar 2022 17:08:35 +0200 Subject: [PATCH 077/112] Free space --- azure-pipelines.yml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 8df66a8969..0deed3301f 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -8,6 +8,9 @@ jobs: variables: buildConfiguration: 'Release' steps: + - checkout: self + fetchDepth: 1 + clean: true - task: BatchScript@1 displayName: 'Run script build.cmd' inputs: @@ -57,6 +60,9 @@ jobs: variables: buildConfiguration: 'Release' steps: + - checkout: self + fetchDepth: 1 + clean: true - task: DownloadPipelineArtifact@2 inputs: buildType: 'current' @@ -145,6 +151,9 @@ jobs: buildConfiguration: 'Release' VSTestRTMBuild: 'false' steps: + - checkout: self + fetchDepth: 1 + clean: true - script: chmod +x ./scripts/vsts-prebuild.sh displayName: 'Preparing for set version' From 020abf0b074944cb050fcca87570ca39451de08e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Jare=C5=A1?= Date: Mon, 28 Mar 2022 17:17:28 +0200 Subject: [PATCH 078/112] Free space --- azure-pipelines.yml | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 0deed3301f..96c3602587 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -63,12 +63,24 @@ jobs: - checkout: self fetchDepth: 1 clean: true + - task: PowerShell@2 + displayName: 'Free space' + inputs: + targetType: 'inline' + script: 'Get-PSDrive -PSProvider FileSystem | Out-String' + - task: DownloadPipelineArtifact@2 inputs: buildType: 'current' artifactName: 'testArtifacts' targetPath: '$(Build.SourcesDirectory)\artifacts' + - task: PowerShell@2 + displayName: 'Free space' + inputs: + targetType: 'inline' + script: 'Get-PSDrive -PSProvider FileSystem | Out-String' + - task: PowerShell@2 inputs: targetType: 'inline' @@ -90,6 +102,12 @@ jobs: targetType: 'inline' script: 'ls "$(Build.SourcesDirectory)\artifacts\Release\packages"' + - task: PowerShell@2 + displayName: 'Free space' + inputs: + targetType: 'inline' + script: 'Get-PSDrive -PSProvider FileSystem | Out-String' + - task: PowerShell@2 displayName: 'Disable Strong Name Validation' inputs: From 55e8714e583ba071ad9a9a82516686c20603f909 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Jare=C5=A1?= Date: Mon, 28 Mar 2022 17:24:26 +0200 Subject: [PATCH 079/112] Add results directory to blame --- .../BlameDataCollectorTests.cs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/BlameDataCollectorTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/BlameDataCollectorTests.cs index e9e6af25ee..f740998847 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/BlameDataCollectorTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/BlameDataCollectorTests.cs @@ -139,6 +139,7 @@ public void HangDumpOnTimeout(RunnerInfo runnerInfo) SetTestEnvironment(_testEnvironment, runnerInfo); var assemblyPaths = GetAssetFullPath("timeout.dll"); var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, string.Empty, runnerInfo.InIsolationValue); + arguments = string.Concat(arguments, $" /ResultsDirectory:{TempDirectory.Path}"); arguments = string.Concat(arguments, $@" /Blame:""CollectHangDump;HangDumpType=full;TestTimeout=3s"""); var env = new Dictionary @@ -162,6 +163,7 @@ public void CrashDumpWhenThereIsNoTimeout(RunnerInfo runnerInfo) SetTestEnvironment(_testEnvironment, runnerInfo); var assemblyPaths = GetAssetFullPath("timeout.dll"); var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, string.Empty, runnerInfo.InIsolationValue); + arguments = string.Concat(arguments, $" /ResultsDirectory:{TempDirectory.Path}"); arguments = string.Concat(arguments, $@" /Blame:""CollectDump;DumpType=full;CollectAlways=true;CollectHangDump"""); var env = new Dictionary @@ -185,6 +187,7 @@ public void CrashDumpOnExit(RunnerInfo runnerInfo) SetTestEnvironment(_testEnvironment, runnerInfo); var assemblyPaths = GetAssetFullPath("timeout.dll"); var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, string.Empty, runnerInfo.InIsolationValue); + arguments = string.Concat(arguments, $" /ResultsDirectory:{TempDirectory.Path}"); arguments = string.Concat(arguments, $@" /Blame:""CollectDump;DumpType=full;CollectAlways=true"""); var env = new Dictionary @@ -206,6 +209,7 @@ public void CrashDumpOnStackOverflow(RunnerInfo runnerInfo) SetTestEnvironment(_testEnvironment, runnerInfo); var assemblyPaths = GetAssetFullPath("crash.dll"); var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, string.Empty, runnerInfo.InIsolationValue); + arguments = string.Concat(arguments, $" /ResultsDirectory:{TempDirectory.Path}"); arguments = string.Concat(arguments, $@" /Blame:""CollectDump;DumpType=full"""); var env = new Dictionary @@ -227,6 +231,7 @@ public void CrashDumpChildProcesses(RunnerInfo runnerInfo) SetTestEnvironment(_testEnvironment, runnerInfo); var assemblyPaths = GetAssetFullPath("child-crash.dll"); var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, string.Empty, runnerInfo.InIsolationValue); + arguments = string.Concat(arguments, $" /ResultsDirectory:{TempDirectory.Path}"); arguments = string.Concat(arguments, $@" /Blame:""CollectDump;DumpType=full"""); InvokeVsTest(arguments); @@ -242,6 +247,7 @@ public void HangDumpChildProcesses(RunnerInfo runnerInfo) SetTestEnvironment(_testEnvironment, runnerInfo); var assemblyPaths = GetAssetFullPath("child-hang.dll"); var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, string.Empty, runnerInfo.InIsolationValue); + arguments = string.Concat(arguments, $" /ResultsDirectory:{TempDirectory.Path}"); arguments = string.Concat(arguments, $@" /Blame:""CollectHangDump;HangDumpType=full;TestTimeout=15s"""); InvokeVsTest(arguments); From 8c0cc0c62060da8c0cd8e95a6dcbcb9dcfc19860 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Jare=C5=A1?= Date: Tue, 29 Mar 2022 09:11:51 +0200 Subject: [PATCH 080/112] Add tools and fix gitingore --- .gitignore | 12 ++++++------ test/TestAssets/Tools/Program.cs | 7 +++++++ test/TestAssets/Tools/Tools.csproj | 23 +++++++++++++++++++++++ 3 files changed, 36 insertions(+), 6 deletions(-) create mode 100644 test/TestAssets/Tools/Program.cs create mode 100644 test/TestAssets/Tools/Tools.csproj diff --git a/.gitignore b/.gitignore index d13211f267..0d22715429 100644 --- a/.gitignore +++ b/.gitignore @@ -104,14 +104,14 @@ ehthumbs.db # =========================== # Custom ignores # =========================== -packages -artifacts -[tT]ools +/packages/ +/artifacts/ +/[tT]ools/ *.dmp *.nupkg *.zip -src/package/sign/sign.nuget.targets +/src/package/sign/sign.nuget.targets # =========================== # Localized resx files @@ -140,5 +140,5 @@ src/package/sign/sign.nuget.targets # =========================== logs/ -.fake -.ionide +.fake/ +.ionide/ diff --git a/test/TestAssets/Tools/Program.cs b/test/TestAssets/Tools/Program.cs new file mode 100644 index 0000000000..94783ab8a9 --- /dev/null +++ b/test/TestAssets/Tools/Program.cs @@ -0,0 +1,7 @@ +// See https://aka.ms/new-console-template for more information +using System; + +// This project is used to restore TestPlatform and TestPlatform.CLI tools packages +// for testing with older versions. + +Console.WriteLine("Hello, World!"); diff --git a/test/TestAssets/Tools/Tools.csproj b/test/TestAssets/Tools/Tools.csproj new file mode 100644 index 0000000000..5981f1c93c --- /dev/null +++ b/test/TestAssets/Tools/Tools.csproj @@ -0,0 +1,23 @@ + + + + ..\..\..\ + true + true + + + + + + net5.0 + Exe + hanging_child + + + + + + + + + From 23b5f663b7609e0e31573041e3335346a6d61f17 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Jare=C5=A1?= Date: Mon, 28 Mar 2022 16:56:11 +0200 Subject: [PATCH 081/112] Fix build --- scripts/build.ps1 | 35 +++------------------------- scripts/common.lib.ps1 | 2 +- src/package/external/external.csproj | 2 +- test/TestAssets/TestAssets.sln | 16 ++++++++++++- 4 files changed, 20 insertions(+), 35 deletions(-) diff --git a/scripts/build.ps1 b/scripts/build.ps1 index 685f3907a6..cff737dc9c 100644 --- a/scripts/build.ps1 +++ b/scripts/build.ps1 @@ -197,40 +197,11 @@ function Invoke-TestAssetsBuild { throw "VSTestConsoleVersion for $propertyName is empty." } - $packages = @( - "Microsoft.TestPlatform" - "Microsoft.TestPlatform.CLI", - "Microsoft.TestPlatform.TranslationLayer" - "Microsoft.NET.Test.SDK" - ) - - foreach ($package in $packages) { - $packagePath = "$env:TP_ROOT_DIR\packages\$($package.ToLower())" - $cachePath = "$packagePath\$vstestConsoleVersion" - - if ((Test-Path -Path $cachePath) -and (Get-ChildItem $cachePath)) { - "Package $package $vsTestConsoleVersion is already in nuget cache at $cachePath." - continue - } - - Invoke-Exe $nugetExe -Arguments "install $package -Version $vsTestConsoleVersion -OutputDirectory $packagePath -ConfigFile ""$nugetConfig""" - - # Install puts it in packages/microsoft.testplatform/Microsoft.TestPlatform.17.1.0, - # because we use that as our output folder. And it also caches it in packages/microsoft.testplatform/17.1.0 - # unless the package is from local source, then it does not do that. So we need to rename the folder and remove - # the original one. - if (-not (Test-Path -Path $cachePath) -or -not (Get-ChildItem $cachePath)) { - Rename-Item "$packagePath\$package.$vsTestConsoleVersion" $cachePath - # Nuget locks the locally copied package from time to time. - Start-Sleep -Milliseconds 300 - } - if (Test-Path "$packagePath\$package.$vsTestConsoleVersion") { - Remove-Item -Recurse -Force "$packagePath\$package.$vsTestConsoleVersion" - } - } + # We don't use the results of this build anywhere, we just use them to restore the packages to nuget cache + # because using nuget.exe install errors out in various weird ways. + Invoke-Exe $dotnetExe -Arguments "build $env:TP_ROOT_DIR\test\TestAssets\Tools\Tools.csproj --configuration $TPB_Configuration -v:minimal -p:CIBuild=$TPB_CIBuild -p:LocalizedBuild=$TPB_LocalizedBuild -p:NETTestSdkVersion=$vsTestConsoleVersion" } - # Build with multiple versions of MSTest. The projects are directly in the root. # The folder structure in VS is not echoed in the TestAssets directory. $projects = @( diff --git a/scripts/common.lib.ps1 b/scripts/common.lib.ps1 index 34fd49a1e8..38b98bd805 100644 --- a/scripts/common.lib.ps1 +++ b/scripts/common.lib.ps1 @@ -47,7 +47,7 @@ Write-Verbose "Setup dotnet configuration." $env:DOTNET_SKIP_FIRST_TIME_EXPERIENCE = 1 # Dotnet build doesn't support --packages yet. See https://github.com/dotnet/cli/issues/2712 $env:NUGET_PACKAGES = $env:TP_PACKAGES_DIR -$env:NUGET_EXE_Version = "5.8.1" +$env:NUGET_EXE_Version = "6.0.0" $env:DOTNET_CLI_VERSION = $GlobalJson.tools.dotnet # $env:DOTNET_RUNTIME_VERSION = "LATEST" $env:VSWHERE_VERSION = "2.0.2" diff --git a/src/package/external/external.csproj b/src/package/external/external.csproj index ebead5a9dc..4a0e70027d 100644 --- a/src/package/external/external.csproj +++ b/src/package/external/external.csproj @@ -33,7 +33,7 @@ - + diff --git a/test/TestAssets/TestAssets.sln b/test/TestAssets/TestAssets.sln index 834852d277..e349d2626f 100644 --- a/test/TestAssets/TestAssets.sln +++ b/test/TestAssets/TestAssets.sln @@ -116,10 +116,12 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "XUnit1000Passing", "perform EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "MSTestProject2", "MSTestProject2\MSTestProject2.csproj", "{10AA955C-B412-41A8-899F-8609AAE19F61}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "MSTestProject1", "MSTestProject1\MSTestProject1.csproj", "{E166D337-4033-4209-863F-8F77675EAEE8}" +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "MSTestProject1", "MSTestProject1\MSTestProject1.csproj", "{E166D337-4033-4209-863F-8F77675EAEE8}" EndProject Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "basic", "basic", "{2633D125-64A7-456C-AD37-F8A6B56C2403}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Tools", "Tools\Tools.csproj", "{85F9F2A8-D2A5-4EA1-8BE0-06CCE141EC7A}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -778,6 +780,18 @@ Global {E166D337-4033-4209-863F-8F77675EAEE8}.Release|x64.Build.0 = Release|Any CPU {E166D337-4033-4209-863F-8F77675EAEE8}.Release|x86.ActiveCfg = Release|Any CPU {E166D337-4033-4209-863F-8F77675EAEE8}.Release|x86.Build.0 = Release|Any CPU + {85F9F2A8-D2A5-4EA1-8BE0-06CCE141EC7A}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {85F9F2A8-D2A5-4EA1-8BE0-06CCE141EC7A}.Debug|Any CPU.Build.0 = Debug|Any CPU + {85F9F2A8-D2A5-4EA1-8BE0-06CCE141EC7A}.Debug|x64.ActiveCfg = Debug|Any CPU + {85F9F2A8-D2A5-4EA1-8BE0-06CCE141EC7A}.Debug|x64.Build.0 = Debug|Any CPU + {85F9F2A8-D2A5-4EA1-8BE0-06CCE141EC7A}.Debug|x86.ActiveCfg = Debug|Any CPU + {85F9F2A8-D2A5-4EA1-8BE0-06CCE141EC7A}.Debug|x86.Build.0 = Debug|Any CPU + {85F9F2A8-D2A5-4EA1-8BE0-06CCE141EC7A}.Release|Any CPU.ActiveCfg = Release|Any CPU + {85F9F2A8-D2A5-4EA1-8BE0-06CCE141EC7A}.Release|Any CPU.Build.0 = Release|Any CPU + {85F9F2A8-D2A5-4EA1-8BE0-06CCE141EC7A}.Release|x64.ActiveCfg = Release|Any CPU + {85F9F2A8-D2A5-4EA1-8BE0-06CCE141EC7A}.Release|x64.Build.0 = Release|Any CPU + {85F9F2A8-D2A5-4EA1-8BE0-06CCE141EC7A}.Release|x86.ActiveCfg = Release|Any CPU + {85F9F2A8-D2A5-4EA1-8BE0-06CCE141EC7A}.Release|x86.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE From 1d9003c64a481e859280245df34ba6763dd80f7f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Jare=C5=A1?= Date: Tue, 29 Mar 2022 09:11:51 +0200 Subject: [PATCH 082/112] Add tools and fix gitingore --- .gitignore | 12 ++++++------ test/TestAssets/Tools/Program.cs | 7 +++++++ test/TestAssets/Tools/Tools.csproj | 23 +++++++++++++++++++++++ 3 files changed, 36 insertions(+), 6 deletions(-) create mode 100644 test/TestAssets/Tools/Program.cs create mode 100644 test/TestAssets/Tools/Tools.csproj diff --git a/.gitignore b/.gitignore index d13211f267..0d22715429 100644 --- a/.gitignore +++ b/.gitignore @@ -104,14 +104,14 @@ ehthumbs.db # =========================== # Custom ignores # =========================== -packages -artifacts -[tT]ools +/packages/ +/artifacts/ +/[tT]ools/ *.dmp *.nupkg *.zip -src/package/sign/sign.nuget.targets +/src/package/sign/sign.nuget.targets # =========================== # Localized resx files @@ -140,5 +140,5 @@ src/package/sign/sign.nuget.targets # =========================== logs/ -.fake -.ionide +.fake/ +.ionide/ diff --git a/test/TestAssets/Tools/Program.cs b/test/TestAssets/Tools/Program.cs new file mode 100644 index 0000000000..94783ab8a9 --- /dev/null +++ b/test/TestAssets/Tools/Program.cs @@ -0,0 +1,7 @@ +// See https://aka.ms/new-console-template for more information +using System; + +// This project is used to restore TestPlatform and TestPlatform.CLI tools packages +// for testing with older versions. + +Console.WriteLine("Hello, World!"); diff --git a/test/TestAssets/Tools/Tools.csproj b/test/TestAssets/Tools/Tools.csproj new file mode 100644 index 0000000000..5981f1c93c --- /dev/null +++ b/test/TestAssets/Tools/Tools.csproj @@ -0,0 +1,23 @@ + + + + ..\..\..\ + true + true + + + + + + net5.0 + Exe + hanging_child + + + + + + + + + From 25c58025a91670eea1fe4dac099a7c7426ef571b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Jare=C5=A1?= Date: Tue, 29 Mar 2022 11:25:22 +0200 Subject: [PATCH 083/112] Make Cancel Discovery faster and more reliable --- .../TranslationLayerTests/DiscoverTests.cs | 30 +++- .../LongDiscoveryTestClass.cs | 135 +++++++++++++++++- 2 files changed, 158 insertions(+), 7 deletions(-) diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/DiscoverTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/DiscoverTests.cs index 6c5d6fea09..b982311927 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/DiscoverTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/DiscoverTests.cs @@ -3,9 +3,13 @@ using System; using System.Collections.Generic; +using System.Diagnostics; using System.Linq; using System.Threading.Tasks; +using FluentAssertions; +using FluentAssertions.Extensions; + using Microsoft.TestPlatform.TestUtilities; using Microsoft.TestPlatform.VsTestConsole.TranslationLayer.Interfaces; using Microsoft.VisualStudio.TestPlatform.Common.Telemetry; @@ -200,6 +204,7 @@ public void DiscoverTestsUsingSourceNavigation(RunnerInfo runnerInfo) [NetCoreTargetFrameworkDataSource] public async Task CancelTestDiscovery(RunnerInfo runnerInfo) { + var sw = Stopwatch.StartNew(); // Setup var testAssemblies = new List { @@ -212,11 +217,20 @@ public async Task CancelTestDiscovery(RunnerInfo runnerInfo) var discoveredTests = new List(); var discoveryEvents = new Mock(); + var alreadyCancelled = false; + TimeSpan cancellationCalled = TimeSpan.Zero; discoveryEvents.Setup(events => events.HandleDiscoveredTests(It.IsAny>())) .Callback((IEnumerable testcases) => { + // As soon as we get first test call cancel. That way we know there is discovery in progress. discoveredTests.AddRange(testcases); - _vstestConsoleWrapper.CancelDiscovery(); + if (!alreadyCancelled) + { + cancellationCalled = sw.Elapsed; + // Calling cancel many times crashes. https://github.com/microsoft/vstest/issues/3526 + alreadyCancelled = true; + _vstestConsoleWrapper.CancelDiscovery(); + } }); var isTestCancelled = false; discoveryEvents.Setup(events => events.HandleDiscoveryComplete(It.IsAny(), It.IsAny>(), It.IsAny())) @@ -229,11 +243,23 @@ public async Task CancelTestDiscovery(RunnerInfo runnerInfo) } }); + string runSettingsXml = + $@" + + + {FrameworkArgValue} + 1 + + "; + // Act - await Task.Run(() => _vstestConsoleWrapper.DiscoverTests(testAssemblies, GetDefaultRunSettings(), discoveryEvents.Object)); + await Task.Run(() => _vstestConsoleWrapper.DiscoverTests(testAssemblies, runSettingsXml, discoveryEvents.Object)); // Assert. Assert.IsTrue(isTestCancelled); + var done = sw.Elapsed; + var timeTillCancelled = done - cancellationCalled; + timeTillCancelled.Should().BeLessThan(2.Seconds()); int discoveredSourcesCount = discoveredTests.Select(testcase => testcase.Source).Distinct().Count(); Assert.AreNotEqual(testAssemblies.Count, discoveredSourcesCount, "All test assemblies discovered"); } diff --git a/test/TestAssets/DiscoveryTestProject/LongDiscoveryTestClass.cs b/test/TestAssets/DiscoveryTestProject/LongDiscoveryTestClass.cs index 26549f1961..4969cc84ed 100644 --- a/test/TestAssets/DiscoveryTestProject/LongDiscoveryTestClass.cs +++ b/test/TestAssets/DiscoveryTestProject/LongDiscoveryTestClass.cs @@ -10,18 +10,143 @@ namespace DiscoveryTestProject3 [TestClass] public class LongDiscoveryTestClass { - [MyTestMethod] - public void CustomTestMethod() + // This is for discovery cancellation test. + // 20 tests below to be discovered until we reach the X_ Y_ Z_LongDiscoveryTestMethod which haver attribute that + // takes a very long time to create, which prolongs the discovery time and keeps us discovering while we + // are cancelling the discovery from the test. + #region 20 empty tests + + [TestMethod] + public void TestMethod1() + { + } + + [TestMethod] + public void TestMethod2() + { + } + + [TestMethod] + public void TestMethod3() + { + } + + [TestMethod] + public void TestMethod4() + { + } + + [TestMethod] + public void TestMethod5() + { + } + + [TestMethod] + public void TestMethod6() + { + } + + [TestMethod] + public void TestMethod7() + { + } + + [TestMethod] + public void TestMethod8() + { + } + + [TestMethod] + public void TestMethod9() + { + } + + [TestMethod] + public void TestMethod10() + { + } + + [TestMethod] + public void TestMethod11() + { + } + + [TestMethod] + public void TestMethod12() + { + } + + [TestMethod] + public void TestMethod13() + { + } + + [TestMethod] + public void TestMethod14() + { + } + + [TestMethod] + public void TestMethod15() + { + } + + [TestMethod] + public void TestMethod16() + { + } + + [TestMethod] + public void TestMethod17() + { + } + + [TestMethod] + public void TestMethod18() + { + } + + [TestMethod] + public void TestMethod19() + { + } + + [TestMethod] + public void TestMethod20() + { + } + + #endregion + + // X_ to make it discover last. + [TestMethodWithDelay] + public void X_LongDiscoveryTestMethod() + { + + } + + // Y_ to make it discover last. + [TestMethodWithDelay] + public void Y_LongDiscoveryTestMethod() + { + + } + + // Z_ to make it discover last. + [TestMethodWithDelay] + public void Z_LongDiscoveryTestMethod() { } } - internal class MyTestMethodAttribute : TestMethodAttribute + internal class TestMethodWithDelayAttribute : TestMethodAttribute { - public MyTestMethodAttribute() + public TestMethodWithDelayAttribute() { - Thread.Sleep(10000); + // This will be multiplied by 3 because the framework will internally create this + // attribute 3 times. And by another 3 because we have 3 slow tests. + Thread.Sleep(100); } } } From 4dd10bc90e164a577aaf64f67f7cf930ab506ca8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Jare=C5=A1?= Date: Tue, 29 Mar 2022 12:14:41 +0200 Subject: [PATCH 084/112] Add fluent assertions --- .../Microsoft.TestPlatform.AcceptanceTests.csproj | 1 + 1 file changed, 1 insertion(+) diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/Microsoft.TestPlatform.AcceptanceTests.csproj b/test/Microsoft.TestPlatform.AcceptanceTests/Microsoft.TestPlatform.AcceptanceTests.csproj index c5a1f80019..543b5b5acc 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/Microsoft.TestPlatform.AcceptanceTests.csproj +++ b/test/Microsoft.TestPlatform.AcceptanceTests/Microsoft.TestPlatform.AcceptanceTests.csproj @@ -28,6 +28,7 @@ + all runtime; build; native; contentfiles; analyzers; buildtransitive From b346a3e2d7b7b4a4212e5c414258c5865fb01c24 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Jare=C5=A1?= Date: Tue, 29 Mar 2022 15:11:11 +0200 Subject: [PATCH 085/112] Renames and fixes --- .../ExecutionTests.cs | 5 +- .../Extension/CompatibilityRowsBuilder.cs | 42 ++++++++++------- .../Extension/Features.cs | 2 + .../MSTestCompatibilityDataSource.cs | 4 +- .../Extension/NetCoreRunner.cs | 4 +- .../NetCoreTargetFrameworkDataSource.cs | 4 +- .../Extension/NetFrameworkRunner.cs | 4 +- .../NetFullTargetFrameworkDataSource.cs | 8 ++-- .../RunnerCompatibilityDataSource.cs | 7 ++- .../TestPlatformCompatibilityDataSource.cs | 4 +- .../TesthostCompatibilityDataSource.cs | 4 +- .../Properties/AssemblyInfo.cs | 2 +- .../TranslationLayerTests/RunTests.cs | 47 +++++++++++++++++++ .../DebugInfo.cs | 2 +- .../IntegrationTestBase.cs | 2 +- 15 files changed, 102 insertions(+), 39 deletions(-) diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/ExecutionTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/ExecutionTests.cs index 037ec5deda..8d09297d02 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/ExecutionTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/ExecutionTests.cs @@ -251,7 +251,7 @@ public void UnhandleExceptionExceptionShouldBeLoggedToDiagLogFile(RunnerInfo run [TestMethod] [TestCategory("Windows-Review")] [NetFullTargetFrameworkDataSource] - public void IncompatibleSourcesWarningShouldBeDisplayedInTheConsole(RunnerInfo runnerInfo) + public void IncompatibleSourcesWarningShouldBeDisplayedInTheConsoleWhenGivenIncompatibleX86andX64Dll(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); @@ -273,7 +273,7 @@ public void IncompatibleSourcesWarningShouldBeDisplayedInTheConsole(RunnerInfo r [TestMethod] [TestCategory("Windows-Review")] [NetFullTargetFrameworkDataSource] - public void NoIncompatibleSourcesWarningShouldBeDisplayedInTheConsole(RunnerInfo runnerInfo) + public void NoIncompatibleSourcesWarningShouldBeDisplayedInTheConsoleWhenGivenSingleX86Dll(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); @@ -312,6 +312,7 @@ public void IncompatibleSourcesWarningShouldBeDisplayedInTheConsoleOnlyWhenRunni { StdOutputDoesNotContains(expectedWarningContains); } + // If we are running this test on 32 bit OS, it should output warning message else { diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/CompatibilityRowsBuilder.cs b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/CompatibilityRowsBuilder.cs index 157b2a14b0..70c130da5e 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/CompatibilityRowsBuilder.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/CompatibilityRowsBuilder.cs @@ -56,8 +56,8 @@ public CompatibilityRowsBuilder(string runnerFrameworks = AcceptanceTestBase.DEF public bool DebugVSTestConsole { get; set; } public bool DebugTestHost { get; set; } public bool DebugDataCollector { get; set; } - public bool NoDefaultBreakpoints { get; set; } = true; - + public bool DebugStopAtEntrypoint { get; set; } + public int? JustRow { get; internal set; } public List CreateData() { @@ -78,8 +78,8 @@ public List CreateData() if (WithInProcess) AddInProcess(dataRows); - var minVersion = SemanticVersion.Parse("0.0.0-alpha.1"); - var maxVersion = SemanticVersion.Parse("9999.0.0"); + var minVersion = ParseAndPatchSemanticVersion("0.0.0-alpha.1"); + var maxVersion = ParseAndPatchSemanticVersion("9999.0.0"); SemanticVersion? beforeRunnerVersion = maxVersion; SemanticVersion? afterRunnerVersion = minVersion; SemanticVersion? beforeTestHostVersion = maxVersion; @@ -90,37 +90,37 @@ public List CreateData() if (BeforeRunnerFeature != null) { var feature = Features.TestPlatformFeatures[BeforeRunnerFeature]; - beforeRunnerVersion = SemanticVersion.Parse(feature.Version.TrimStart('v')); + beforeRunnerVersion = ParseAndPatchSemanticVersion(feature.Version); } if (AfterRunnerFeature != null) { var feature = Features.TestPlatformFeatures[AfterRunnerFeature]; - afterRunnerVersion = SemanticVersion.Parse(feature.Version.TrimStart('v')); + afterRunnerVersion = ParseAndPatchSemanticVersion(feature.Version); } if (BeforeTestHostFeature != null) { var feature = Features.TestPlatformFeatures[BeforeTestHostFeature]; - beforeTestHostVersion = SemanticVersion.Parse(feature.Version.TrimStart('v')); + beforeTestHostVersion = ParseAndPatchSemanticVersion(feature.Version); } if (AfterTestHostFeature != null) { var feature = Features.TestPlatformFeatures[AfterTestHostFeature]; - afterTestHostVersion = SemanticVersion.Parse(feature.Version.TrimStart('v')); + afterTestHostVersion = ParseAndPatchSemanticVersion(feature.Version); } if (BeforeAdapterFeature != null) { var feature = Features.TestPlatformFeatures[BeforeAdapterFeature]; - beforeAdapterVersion = SemanticVersion.Parse(feature.Version.TrimStart('v')); + beforeAdapterVersion = ParseAndPatchSemanticVersion(feature.Version); } if (AfterAdapterFeature != null) { var feature = Features.AdapterFeatures[AfterAdapterFeature]; - afterAdapterVersion = SemanticVersion.Parse(feature.Version.TrimStart('v')); + afterAdapterVersion = ParseAndPatchSemanticVersion(feature.Version); } var isWindows = Environment.OSVersion.Platform.ToString().StartsWith("Win"); @@ -132,12 +132,12 @@ public List CreateData() // We probably don't have that need right now, because legacy version is 15.x.x, which is very old, and we are still keeping // compatibility. - Func isInRange = (version, before, after) => version < before && after < version; + Func isInRange = (version, before, after) => version < before && after <= version; var rows = dataRows.Where(r => r.VSTestConsoleInfo != null - && isInRange(SemanticVersion.Parse(r.VSTestConsoleInfo.Version), beforeRunnerVersion, afterRunnerVersion) - && r.TestHostInfo != null && isInRange(SemanticVersion.Parse(r.TestHostInfo.Version), beforeTestHostVersion, afterTestHostVersion) - && r.AdapterInfo != null && isInRange(SemanticVersion.Parse(r.AdapterInfo.Version), beforeAdapterVersion, afterAdapterVersion)).ToList(); + && isInRange(ParseAndPatchSemanticVersion(r.VSTestConsoleInfo.Version), beforeRunnerVersion, afterRunnerVersion) + && r.TestHostInfo != null && isInRange(ParseAndPatchSemanticVersion(r.TestHostInfo.Version), beforeTestHostVersion, afterTestHostVersion) + && r.AdapterInfo != null && isInRange(ParseAndPatchSemanticVersion(r.AdapterInfo.Version), beforeAdapterVersion, afterAdapterVersion)).ToList(); // We use ToString to determine which values are unique. Not great solution, but works better than using records. var distinctRows = new Dictionary(); @@ -149,7 +149,17 @@ public List CreateData() throw new InvalidOperationException("There were no rows that matched the specified criteria."); } - return distinctRows.Values.ToList(); + var allRows = distinctRows.Values.ToList(); + + return JustRow == null ? allRows : new List { allRows[JustRow.Value] }; + } + + private static SemanticVersion ParseAndPatchSemanticVersion(string? version) + { + // Our developer version is 17.2.0-dev, but we release few preview, that are named 17.2.0-preview or 17.2.0-release, yet we still + // want 17.2.0-dev to be considered the latest version. So we patch it. + var v = version != null && version.EndsWith("-dev") ? version?.Substring(0, version.Length - 4) + "-ZZZZZZZZZZ" : version; + return SemanticVersion.Parse(v?.TrimStart('v')); } private void AddInProcess(List dataRows) @@ -292,7 +302,7 @@ private DebugInfo GetDebugInfo() DebugDataCollector = DebugDataCollector, DebugTestHost = DebugTestHost, DebugVSTestConsole = DebugVSTestConsole, - NoDefaultBreakpoints = NoDefaultBreakpoints + DebugStopAtEntrypoint = DebugStopAtEntrypoint }; } diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/Features.cs b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/Features.cs index 0be9ac104c..f235879f46 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/Features.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/Features.cs @@ -10,10 +10,12 @@ public static class Features { public const string ATTACH_DEBUGGER_FLOW = nameof(ATTACH_DEBUGGER_FLOW); public const string MSTEST_EXAMPLE_FEATURE = nameof(MSTEST_EXAMPLE_FEATURE); + public const string MULTI_TFM = nameof(MULTI_TFM); public static IImmutableDictionary TestPlatformFeatures { get; } = new Dictionary { [ATTACH_DEBUGGER_FLOW] = new(version: "v16.7.0-preview-20200519-01", issue: "https://github.com/microsoft/vstest/pull/2325"), + [MULTI_TFM] = new(version: "17.2.0-dev", issue: "https://github.com/microsoft/vstest/pull/3412") }.ToImmutableDictionary(); public static IImmutableDictionary AdapterFeatures { get; internal set; } = new Dictionary diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/MSTestCompatibilityDataSource.cs b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/MSTestCompatibilityDataSource.cs index f6d20b577c..e75bc87594 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/MSTestCompatibilityDataSource.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/MSTestCompatibilityDataSource.cs @@ -34,7 +34,7 @@ public MSTestCompatibilityDataSource( public bool DebugVSTestConsole { get; set; } public bool DebugTestHost { get; set; } public bool DebugDataCollector { get; set; } - public bool NoDefaultBreakpoints { get; set; } = true; + public bool DebugStopAtEntrypoint { get; set; } /// /// Add run for in-process using the selected .NET Framework runners, and and all selected adapters. @@ -70,7 +70,7 @@ public override void CreateData(MethodInfo methodInfo) _builder.DebugDataCollector = DebugDataCollector; _builder.DebugVSTestConsole = DebugVSTestConsole; _builder.DebugTestHost = DebugTestHost; - _builder.NoDefaultBreakpoints = NoDefaultBreakpoints; + _builder.DebugStopAtEntrypoint = DebugStopAtEntrypoint; var data = _builder.CreateData(); data.ForEach(AddData); diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/NetCoreRunner.cs b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/NetCoreRunner.cs index 202264d0c7..477715e581 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/NetCoreRunner.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/NetCoreRunner.cs @@ -37,7 +37,7 @@ public NetCoreRunner(string targetFrameworks = AcceptanceTestBase.NETFX452_NET50 public bool DebugVSTestConsole { get; set; } public bool DebugTestHost { get; set; } public bool DebugDataCollector { get; set; } - public bool NoDefaultBreakpoints { get; set; } = true; + public bool DebugStopAtEntrypoint { get; set; } public IEnumerable GetData(MethodInfo methodInfo) { @@ -58,7 +58,7 @@ public IEnumerable GetData(MethodInfo methodInfo) DebugVSTestConsole = DebugVSTestConsole, DebugTestHost = DebugTestHost, DebugDataCollector = DebugDataCollector, - NoDefaultBreakpoints = NoDefaultBreakpoints, + DebugStopAtEntrypoint = DebugStopAtEntrypoint, }; dataRows.Add(new object[] { runnerInfo }); } diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/NetCoreTargetFrameworkDataSource.cs b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/NetCoreTargetFrameworkDataSource.cs index 56ff01036b..a5b51cc79b 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/NetCoreTargetFrameworkDataSource.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/NetCoreTargetFrameworkDataSource.cs @@ -51,7 +51,7 @@ public NetCoreTargetFrameworkDataSource( public bool DebugVSTestConsole { get; set; } public bool DebugTestHost { get; set; } public bool DebugDataCollector { get; set; } - public bool NoDefaultBreakpoints { get; set; } = true; + public bool DebugStopAtEntrypoint { get; set; } private void AddRunnerDataRow(List dataRows, string runnerFramework, string targetFramework) { @@ -66,7 +66,7 @@ private void AddRunnerDataRow(List dataRows, string runnerFramework, s DebugDataCollector = DebugDataCollector, DebugTestHost = DebugTestHost, DebugVSTestConsole = DebugVSTestConsole, - NoDefaultBreakpoints = NoDefaultBreakpoints, + DebugStopAtEntrypoint = DebugStopAtEntrypoint, }; dataRows.Add(new object[] { runnerInfo }); } diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/NetFrameworkRunner.cs b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/NetFrameworkRunner.cs index 0f1faa65f5..2d5ceb82a5 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/NetFrameworkRunner.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/NetFrameworkRunner.cs @@ -34,7 +34,7 @@ public NetFrameworkRunner(string targetFrameworks = AcceptanceTestBase.NETFX452_ public bool DebugVSTestConsole { get; set; } public bool DebugTestHost { get; set; } public bool DebugDataCollector { get; set; } - public bool NoDefaultBreakpoints { get; set; } = true; + public bool DebugStopAtEntrypoint { get; set; } private readonly string _targetFrameworks; @@ -60,7 +60,7 @@ public IEnumerable GetData(MethodInfo methodInfo) DebugVSTestConsole = DebugVSTestConsole, DebugTestHost = DebugTestHost, DebugDataCollector = DebugDataCollector, - NoDefaultBreakpoints = NoDefaultBreakpoints, + DebugStopAtEntrypoint = DebugStopAtEntrypoint, }; dataRows.Add(new object[] { runnerInfo }); diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/NetFullTargetFrameworkDataSource.cs b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/NetFullTargetFrameworkDataSource.cs index b809da7799..c756619ba5 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/NetFullTargetFrameworkDataSource.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/NetFullTargetFrameworkDataSource.cs @@ -45,7 +45,7 @@ public NetFullTargetFrameworkDataSource(bool inIsolation = true, bool inProcess public bool DebugVSTestConsole { get; set; } public bool DebugTestHost { get; set; } public bool DebugDataCollector { get; set; } - public bool NoDefaultBreakpoints { get; set; } = true; + public bool DebugStopAtEntrypoint { get; set; } public IEnumerable GetData(MethodInfo methodInfo) { @@ -64,7 +64,7 @@ public IEnumerable GetData(MethodInfo methodInfo) DebugVSTestConsole = DebugVSTestConsole, DebugTestHost = DebugTestHost, DebugDataCollector = DebugDataCollector, - NoDefaultBreakpoints = NoDefaultBreakpoints, + DebugStopAtEntrypoint = DebugStopAtEntrypoint, }; dataRows.Add(new object[] { runnerInfo }); } @@ -84,7 +84,7 @@ public IEnumerable GetData(MethodInfo methodInfo) DebugVSTestConsole = DebugVSTestConsole, DebugTestHost = DebugTestHost, DebugDataCollector = DebugDataCollector, - NoDefaultBreakpoints = NoDefaultBreakpoints, + DebugStopAtEntrypoint = DebugStopAtEntrypoint, }; dataRows.Add(new object[] { runnerInfo }); } @@ -102,7 +102,7 @@ public IEnumerable GetData(MethodInfo methodInfo) DebugVSTestConsole = DebugVSTestConsole, DebugTestHost = DebugTestHost, DebugDataCollector = DebugDataCollector, - NoDefaultBreakpoints = NoDefaultBreakpoints, + DebugStopAtEntrypoint = DebugStopAtEntrypoint, }; dataRows.Add(new object[] { runnerInfo }); } diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/RunnerCompatibilityDataSource.cs b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/RunnerCompatibilityDataSource.cs index 357dcd1070..026afa38b8 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/RunnerCompatibilityDataSource.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/RunnerCompatibilityDataSource.cs @@ -39,7 +39,8 @@ public RunnerCompatibilityDataSource( public bool DebugVSTestConsole { get; set; } public bool DebugTestHost { get; set; } public bool DebugDataCollector { get; set; } - public bool NoDefaultBreakpoints { get; set; } = true; + public bool DebugStopAtEntrypoint { get; set; } + public int JustRow { get; set; } = -1; /// /// Add run for in-process using the selected .NET Framework runners, and and all selected adapters. @@ -75,7 +76,9 @@ public override void CreateData(MethodInfo methodInfo) _builder.DebugDataCollector = DebugDataCollector; _builder.DebugVSTestConsole = DebugVSTestConsole; _builder.DebugTestHost = DebugTestHost; - _builder.NoDefaultBreakpoints = NoDefaultBreakpoints; + _builder.DebugStopAtEntrypoint = DebugStopAtEntrypoint; + + _builder.JustRow = JustRow < 0 ? null : JustRow; var data = _builder.CreateData(); data.ForEach(AddData); diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/TestPlatformCompatibilityDataSource.cs b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/TestPlatformCompatibilityDataSource.cs index 5f117fff0b..95c0e290d6 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/TestPlatformCompatibilityDataSource.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/TestPlatformCompatibilityDataSource.cs @@ -44,7 +44,7 @@ public TestPlatformCompatibilityDataSource( public bool DebugVSTestConsole { get; set; } public bool DebugTestHost { get; set; } public bool DebugDataCollector { get; set; } - public bool NoDefaultBreakpoints { get; set; } = true; + public bool DebugStopAtEntrypoint { get; set; } /// /// Add run for in-process using the selected .NET Framework runners, and and all selected adapters. @@ -88,7 +88,7 @@ public override void CreateData(MethodInfo methodInfo) _builder.DebugDataCollector = DebugDataCollector; _builder.DebugVSTestConsole = DebugVSTestConsole; _builder.DebugTestHost = DebugTestHost; - _builder.NoDefaultBreakpoints = NoDefaultBreakpoints; + _builder.DebugStopAtEntrypoint = DebugStopAtEntrypoint; var data = _builder.CreateData(); data.ForEach(AddData); diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/TesthostCompatibilityDataSource.cs b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/TesthostCompatibilityDataSource.cs index 4bcbba2f93..2af3e23e5d 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/TesthostCompatibilityDataSource.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/TesthostCompatibilityDataSource.cs @@ -39,7 +39,7 @@ public TestHostCompatibilityDataSource( public bool DebugVSTestConsole { get; set; } public bool DebugTestHost { get; set; } public bool DebugDataCollector { get; set; } - public bool NoDefaultBreakpoints { get; set; } = true; + public bool DebugStopAtEntrypoint { get; set; } public string? BeforeFeature { get; set; } public string? AfterFeature { get; set; } @@ -58,7 +58,7 @@ public override void CreateData(MethodInfo methodInfo) _builder.DebugDataCollector = DebugDataCollector; _builder.DebugVSTestConsole = DebugVSTestConsole; _builder.DebugTestHost = DebugTestHost; - _builder.NoDefaultBreakpoints = NoDefaultBreakpoints; + _builder.DebugStopAtEntrypoint = DebugStopAtEntrypoint; var data = _builder.CreateData(); data.ForEach(AddData); diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/Properties/AssemblyInfo.cs b/test/Microsoft.TestPlatform.AcceptanceTests/Properties/AssemblyInfo.cs index 71781d0b4a..a31c6c40a5 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/Properties/AssemblyInfo.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/Properties/AssemblyInfo.cs @@ -23,4 +23,4 @@ [assembly: Guid("755996fa-672a-4272-9776-7f707a520058")] // Enable IAP at class level with as many threads as possible based on CPU and core count. -[assembly: Parallelize(Workers = 0, Scope = ExecutionScope.ClassLevel)] +[assembly: Parallelize(Workers = 0, Scope = ExecutionScope.MethodLevel)] diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/RunTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/RunTests.cs index b17f17b57a..8dd5b1b193 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/RunTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/RunTests.cs @@ -6,6 +6,8 @@ using System.Diagnostics; using System.Linq; +using FluentAssertions; + using Microsoft.TestPlatform.TestUtilities; using Microsoft.TestPlatform.VsTestConsole.TranslationLayer.Interfaces; using Microsoft.VisualStudio.TestPlatform.Common.Telemetry; @@ -57,6 +59,51 @@ public void RunAllTests(RunnerInfo runnerInfo) Assert.AreEqual(2, runEventHandler.TestResults.Count(t => t.Outcome == TestOutcome.Skipped)); } + [TestMethod] + [TestCategory("Windows-Review")] + [RunnerCompatibilityDataSource(BeforeFeature = Features.MULTI_TFM)] + public void RunAllTestsWithMixedTFMsWillFailToRunTestsFromTheIncompatibleTFMDll(RunnerInfo runnerInfo) + { + // Arrange + SetTestEnvironment(_testEnvironment, runnerInfo); + + var vstestConsoleWrapper = GetVsTestConsoleWrapper(); + var runEventHandler = new RunEventHandler(); + var compatibleDll = GetTestDllForFramework("MSTestProject1.dll", "net451"); + var incompatibleDll = GetTestDllForFramework("MSTestProject1.dll", "netcoreapp2.1"); + + // Act + // We have no preference around what TFM is used. It will be autodetected. + var runsettingsXml = ""; + vstestConsoleWrapper.RunTests(new[] { compatibleDll, incompatibleDll }, runsettingsXml, runEventHandler); + + // Assert + runEventHandler.TestResults.Should().HaveCount(3, "we failed to run those tests because they are not compatible."); + } + + [TestMethod] + [TestCategory("Windows-Review")] + [RunnerCompatibilityDataSource(AfterFeature = Features.MULTI_TFM, DebugVSTestConsole = true, JustRow = 1)] + public void RunAllTestsWithMixedTFMsWillRunTestsFromAllProvidedDllEvenWhenTheyMixTFMs(RunnerInfo runnerInfo) + { + // Arrange + SetTestEnvironment(_testEnvironment, runnerInfo); + + var vstestConsoleWrapper = GetVsTestConsoleWrapper(); + var runEventHandler = new RunEventHandler(); + var netFrameworkDll = GetTestDllForFramework("MSTestProject1.dll", "net451"); + var netDll = GetTestDllForFramework("MSTestProject1.dll", "netcoreapp2.1"); + + // Act + // We have no preference around what TFM is used. It will be autodetected. + var runsettingsXml = ""; + vstestConsoleWrapper.RunTests(new[] { netFrameworkDll, netDll }, runsettingsXml, runEventHandler); + + // Assert + runEventHandler.Errors.Should().BeEmpty(); + runEventHandler.TestResults.Should().HaveCount(6, "we run all tests from both assemblies"); + } + [TestMethod] [NetFullTargetFrameworkDataSource] [NetCoreTargetFrameworkDataSource] diff --git a/test/Microsoft.TestPlatform.TestUtilities/DebugInfo.cs b/test/Microsoft.TestPlatform.TestUtilities/DebugInfo.cs index f9ac35d770..12eb25aaa5 100644 --- a/test/Microsoft.TestPlatform.TestUtilities/DebugInfo.cs +++ b/test/Microsoft.TestPlatform.TestUtilities/DebugInfo.cs @@ -14,5 +14,5 @@ public sealed class DebugInfo public bool DebugVSTestConsole { get; set; } public bool DebugTestHost { get; set; } public bool DebugDataCollector { get; set; } - public bool NoDefaultBreakpoints { get; set; } = true; + public bool DebugStopAtEntrypoint { get; set; } } diff --git a/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestBase.cs b/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestBase.cs index c0e317d0f6..f30635cec0 100644 --- a/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestBase.cs +++ b/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestBase.cs @@ -251,7 +251,7 @@ private Dictionary AddDebugEnvironmentVariables(Dictionary Date: Thu, 12 May 2022 11:49:14 +0200 Subject: [PATCH 086/112] Add debugger info --- .../Client/Interfaces/ITestHostLauncher2.cs | 14 +++ .../ExecutionTests.cs | 32 ------ .../CustomTestHostTests.cs | 105 ++++++++++++++++++ .../TranslationLayerTests/RunTests.cs | 32 +++++- 4 files changed, 149 insertions(+), 34 deletions(-) diff --git a/src/Microsoft.TestPlatform.ObjectModel/Client/Interfaces/ITestHostLauncher2.cs b/src/Microsoft.TestPlatform.ObjectModel/Client/Interfaces/ITestHostLauncher2.cs index 5f16818c44..a545bafc8d 100644 --- a/src/Microsoft.TestPlatform.ObjectModel/Client/Interfaces/ITestHostLauncher2.cs +++ b/src/Microsoft.TestPlatform.ObjectModel/Client/Interfaces/ITestHostLauncher2.cs @@ -1,6 +1,7 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT license. See LICENSE file in the project root for full license information. +using System; using System.Threading; #nullable disable @@ -27,3 +28,16 @@ public interface ITestHostLauncher2 : ITestHostLauncher /// if the debugger was successfully attached to the requested process, otherwise. bool AttachDebuggerToProcess(int pid, CancellationToken cancellationToken); } + +public interface ITestHostLauncher3 : ITestHostLauncher2 +{ + bool AttachDebuggerToProcess(AttachDebuggerInfo attachDebuggerInfo); +} + +public class AttachDebuggerInfo +{ + public Version Version { get; set; } + public int ProcessId { get; set; } + public Framework? TargetFramework { get; set; } + public CancellationToken CancellationToken { get; set; } +} diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/ExecutionTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/ExecutionTests.cs index 5e092b7ab7..c5230aa9f7 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/ExecutionTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/ExecutionTests.cs @@ -69,38 +69,6 @@ public void RunMultipleMSTestAssembliesOnVstestConsoleAndTesthostCombinations(Ru StdOutHasNoWarnings(); } - [TestMethod] - [TestCategory("Windows-Review")] - [TestPlatformCompatibilityDataSource()] - public void RunTestsFromMultipleMSTestAssemblies(RunnerInfo runnerInfo) - { - SetTestEnvironment(_testEnvironment, runnerInfo); - - var assemblyPaths = BuildMultipleAssemblyPath("MSTestProject1.dll", "MSTestProject2.dll"); - - InvokeVsTestForExecution(assemblyPaths, testAdapterPath: null, FrameworkArgValue, string.Empty); - - ValidateSummaryStatus(passed: 2, failed: 2, skipped: 2); - ExitCodeEquals(1); // failing tests - StdErrHasTestRunFailedMessageButNoOtherError(); - StdOutHasNoWarnings(); - } - - [TestMethod] - [TestCategory("Windows-Review")] - [TestHostCompatibilityDataSource] - public void RunMultipleMSTestAssembliesOnVstestConsoleAndTesthostCombinations(RunnerInfo runnerInfo) - { - SetTestEnvironment(_testEnvironment, runnerInfo); - - var assemblyPaths = BuildMultipleAssemblyPath("MSTestProject1.dll", "MSTestProject2.dll"); - - InvokeVsTestForExecution(assemblyPaths, testAdapterPath: null, FrameworkArgValue, string.Empty); - - ValidateSummaryStatus(2, 2, 2); - ExitCodeEquals(1); // failing tests - } - [TestMethod] [TestCategory("Windows-Review")] [RunnerCompatibilityDataSource] diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/CustomTestHostTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/CustomTestHostTests.cs index ac6b33001a..14a7b8386d 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/CustomTestHostTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/CustomTestHostTests.cs @@ -1,6 +1,7 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT license. See LICENSE file in the project root for full license information. +using System.Collections.Generic; using System.Diagnostics; using System.Linq; using System.Threading; @@ -138,6 +139,64 @@ public void RunTestsWithCustomTestHostLauncherUsesLaunchWhenGivenAnOutdatedITest customTestHostLauncher.LaunchProcessProcessId.Should().NotBeNull("we should launch some real process and save the pid of it"); } + [TestMethod] + [TestCategory("Windows-Review")] + [TestCategory("Feature")] + [RunnerCompatibilityDataSource(AfterFeature = Features.MULTI_TFM, JustRow = 1)] + public void RunAllTestsWithMixedTFMsWillProvideAdditionalInformationToTheDebugger(RunnerInfo runnerInfo) + { + // Arrange + SetTestEnvironment(_testEnvironment, runnerInfo); + + var vstestConsoleWrapper = GetVsTestConsoleWrapper(); + var runEventHandler = new RunEventHandler(); + var netFrameworkDll = GetTestDllForFramework("MSTestProject1.dll", "net451"); + var netDll = GetTestDllForFramework("MSTestProject1.dll", "netcoreapp2.1"); + var testHostLauncher = new TestHostLauncherV3(); + + // Act + // We have no preference around what TFM is used. It will be autodetected. + var runsettingsXml = ""; + vstestConsoleWrapper.RunTestsWithCustomTestHost(new[] { netFrameworkDll, netDll }, runsettingsXml, runEventHandler, testHostLauncher); + + // Assert + runEventHandler.Errors.Should().BeEmpty(); + testHostLauncher.AttachDebuggerInfos.Should().HaveCount(2); + var targetFrameworks = testHostLauncher.AttachDebuggerInfos.Select(i => i.TargetFramework).ToList(); + targetFrameworks.Should().OnlyContain(tfm => tfm == Framework.FromString("net451") || tfm == Framework.FromString("netcoreapp2.1")); + + runEventHandler.TestResults.Should().HaveCount(6, "we run all tests from both assemblies"); + } + + [TestMethod] + [TestCategory("Windows-Review")] + [TestCategory("BackwardCompatibilityWithRunner")] + [RunnerCompatibilityDataSource(BeforeFeature = Features.MULTI_TFM, JustRow = 1)] + public void RunAllTestsWithMixedTFMsCallsBackToTestHostLauncherV3EvenWhenRunnerDoesNotSupportItYet(RunnerInfo runnerInfo) + { + // Arrange + SetTestEnvironment(_testEnvironment, runnerInfo); + + var vstestConsoleWrapper = GetVsTestConsoleWrapper(); + var runEventHandler = new RunEventHandler(); + var netFrameworkDll = GetTestDllForFramework("MSTestProject1.dll", "net451"); + var netDll = GetTestDllForFramework("MSTestProject1.dll", "netcoreapp2.1"); + var testHostLauncher = new TestHostLauncherV3(); + + // Act + // We have no preference around what TFM is used. It will be autodetected. + var runsettingsXml = ""; + vstestConsoleWrapper.RunTestsWithCustomTestHost(new[] { netFrameworkDll, netDll }, runsettingsXml, runEventHandler, testHostLauncher); + + // Assert + runEventHandler.Errors.Should().BeEmpty(); + testHostLauncher.AttachDebuggerInfos.Should().HaveCount(2); + var targetFrameworks = testHostLauncher.AttachDebuggerInfos.Select(i => i.TargetFramework).ToList(); + targetFrameworks.Should().OnlyContain(tfm => tfm == Framework.FromString("net451") || tfm == Framework.FromString("netcoreapp2.1")); + + runEventHandler.TestResults.Should().HaveCount(6, "we run all tests from both assemblies"); + } + private static void EnsureTestsRunWithoutErrors(RunEventHandler runEventHandler, int passed, int failed, int skipped) { runEventHandler.Errors.Should().BeEmpty(); @@ -198,4 +257,50 @@ public bool AttachDebuggerToProcess(int pid, CancellationToken cancellationToken return true; } } + + private class TestHostLauncherV3 : ITestHostLauncher3 + { + public bool IsDebug => true; + + public List AttachDebuggerInfos { get; } = new(); + + public bool AttachDebuggerToProcess(AttachDebuggerInfo attachDebuggerInfo) + { + AttachDebuggerInfos.Add(attachDebuggerInfo); + + return true; + } + + public bool AttachDebuggerToProcess(int pid) + { + return AttachDebuggerToProcess(new AttachDebuggerInfo + { + ProcessId = pid, + TargetFramework = null, + Version = null, + CancellationToken = CancellationToken.None + }); + } + + public bool AttachDebuggerToProcess(int pid, CancellationToken cancellationToken) + { + return AttachDebuggerToProcess(new AttachDebuggerInfo + { + ProcessId = pid, + TargetFramework = null, + Version = null, + CancellationToken = cancellationToken + }); + } + + public int LaunchTestHost(TestProcessStartInfo defaultTestHostStartInfo) + { + return -1; + } + + public int LaunchTestHost(TestProcessStartInfo defaultTestHostStartInfo, CancellationToken cancellationToken) + { + return -1; + } + } } diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/RunTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/RunTests.cs index 8dd5b1b193..e31da3df04 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/RunTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/RunTests.cs @@ -5,6 +5,7 @@ using System.Collections.Generic; using System.Diagnostics; using System.Linq; +using System.Threading; using FluentAssertions; @@ -13,6 +14,7 @@ using Microsoft.VisualStudio.TestPlatform.Common.Telemetry; using Microsoft.VisualStudio.TestPlatform.ObjectModel; using Microsoft.VisualStudio.TestPlatform.ObjectModel.Client; +using Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.Interfaces; using Microsoft.VisualStudio.TestPlatform.ObjectModel.Logging; using Microsoft.VisualStudio.TestTools.UnitTesting; @@ -47,7 +49,7 @@ public void Cleanup() public void RunAllTests(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - + var vstestConsoleWrapper = GetVsTestConsoleWrapper(); var runEventHandler = new RunEventHandler(); vstestConsoleWrapper.RunTests(GetTestDlls("MSTestProject1.dll", "MSTestProject2.dll"), GetDefaultRunSettings(), runEventHandler); @@ -83,7 +85,8 @@ public void RunAllTestsWithMixedTFMsWillFailToRunTestsFromTheIncompatibleTFMDll( [TestMethod] [TestCategory("Windows-Review")] - [RunnerCompatibilityDataSource(AfterFeature = Features.MULTI_TFM, DebugVSTestConsole = true, JustRow = 1)] + [TestHostCompatibilityDataSource] + [RunnerCompatibilityDataSource(AfterFeature = Features.MULTI_TFM)] public void RunAllTestsWithMixedTFMsWillRunTestsFromAllProvidedDllEvenWhenTheyMixTFMs(RunnerInfo runnerInfo) { // Arrange @@ -242,4 +245,29 @@ private IList GetTestAssemblies() GetAssetFullPath("SimpleTestProject2.dll") }; } + + private class TestHostLauncher : ITestHostLauncher2 + { + public bool IsDebug => true; + + public bool AttachDebuggerToProcess(int pid) + { + return true; + } + + public bool AttachDebuggerToProcess(int pid, CancellationToken cancellationToken) + { + return true; + } + + public int LaunchTestHost(TestProcessStartInfo defaultTestHostStartInfo) + { + return -1; + } + + public int LaunchTestHost(TestProcessStartInfo defaultTestHostStartInfo, CancellationToken cancellationToken) + { + return -1; + } + } } From 69335488c48e3fd4fa01a2da6921bb3d100636a6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Jare=C5=A1?= Date: Thu, 12 May 2022 13:14:02 +0200 Subject: [PATCH 087/112] Half merged but is missing some test fixes --- .../Parallel/ParallelProxyDiscoveryManager.cs | 13 +- .../TestPlatformTests.cs | 4 +- .../Parallel/ParallelOperationManagerTests.cs | 8 +- .../ParallelProxyDiscoveryManagerTests.cs | 37 ++-- .../ParallelProxyExecutionManagerTests.cs | 183 +++++++++--------- .../TestEngineTests.cs | 4 +- 6 files changed, 126 insertions(+), 123 deletions(-) diff --git a/src/Microsoft.TestPlatform.CrossPlatEngine/Client/Parallel/ParallelProxyDiscoveryManager.cs b/src/Microsoft.TestPlatform.CrossPlatEngine/Client/Parallel/ParallelProxyDiscoveryManager.cs index 8340a65d80..c47be2aaac 100644 --- a/src/Microsoft.TestPlatform.CrossPlatEngine/Client/Parallel/ParallelProxyDiscoveryManager.cs +++ b/src/Microsoft.TestPlatform.CrossPlatEngine/Client/Parallel/ParallelProxyDiscoveryManager.cs @@ -24,12 +24,12 @@ namespace Microsoft.VisualStudio.TestPlatform.CrossPlatEngine.Client.Parallel; internal class ParallelProxyDiscoveryManager : IParallelProxyDiscoveryManager { private readonly IDataSerializer _dataSerializer; + private readonly DiscoveryDataAggregator _dataAggregator; private readonly ParallelOperationManager _parallelOperationManager; private readonly Dictionary _sourceToTestHostProviderMap; private int _discoveryCompletedClients; private int _availableTestSources = -1; - private ParallelDiscoveryDataAggregator _currentDiscoveryDataAggregator; private bool _skipDefaultAdapters; private readonly IRequestData _requestData; @@ -43,21 +43,24 @@ internal class ParallelProxyDiscoveryManager : IParallelProxyDiscoveryManager public ParallelProxyDiscoveryManager( IRequestData requestData, Func actualProxyManagerCreator, + DiscoveryDataAggregator dataAggregator, int parallelLevel, List testHostProviders) - : this(requestData, actualProxyManagerCreator, JsonDataSerializer.Instance, parallelLevel, testHostProviders) + : this(requestData, actualProxyManagerCreator, dataAggregator, JsonDataSerializer.Instance, parallelLevel, testHostProviders) { } internal ParallelProxyDiscoveryManager( IRequestData requestData, Func actualProxyManagerCreator, + DiscoveryDataAggregator dataAggregator, IDataSerializer dataSerializer, int parallelLevel, List testHostProviders) { _requestData = requestData; _dataSerializer = dataSerializer; + _dataAggregator = dataAggregator; _parallelOperationManager = new(actualProxyManagerCreator, parallelLevel); _sourceToTestHostProviderMap = testHostProviders .SelectMany(provider => provider.SourceDetails.Select(s => new KeyValuePair(s.Source, provider))) @@ -80,11 +83,9 @@ public void DiscoverTests(DiscoveryCriteria discoveryCriteria!!, ITestDiscoveryE EqtTrace.Verbose("ParallelProxyDiscoveryManager: Start discovery. Total sources: " + _availableTestSources); - // One data aggregator per parallel discovery - _currentDiscoveryDataAggregator = new ParallelDiscoveryDataAggregator(); // Marking all sources as not discovered before starting actual discovery - _currentDiscoveryDataAggregator.MarkSourcesWithStatus(discoveryCriteria.Sources.ToList(), DiscoveryStatus.NotDiscovered); + _dataAggregator.MarkSourcesWithStatus(discoveryCriteria.Sources.ToList(), DiscoveryStatus.NotDiscovered); _parallelOperationManager.StartWork(workloads, eventHandler, GetParallelEventHandler, DiscoverTestsOnConcurrentManager); } @@ -95,7 +96,7 @@ private ITestDiscoveryEventsHandler2 GetParallelEventHandler(ITestDiscoveryEvent concurrentManager, eventHandler, this, - _currentDiscoveryDataAggregator); + _dataAggregator); /// public void Abort() diff --git a/test/Microsoft.TestPlatform.Client.UnitTests/TestPlatformTests.cs b/test/Microsoft.TestPlatform.Client.UnitTests/TestPlatformTests.cs index 4ae6eb8e30..53ffbb4cc8 100644 --- a/test/Microsoft.TestPlatform.Client.UnitTests/TestPlatformTests.cs +++ b/test/Microsoft.TestPlatform.Client.UnitTests/TestPlatformTests.cs @@ -284,7 +284,7 @@ public void CreateTestRunRequestShouldSetCustomHostLauncherOnEngineDefaultLaunch { var mockCustomLauncher = new Mock(); _executionManager.Setup(dm => dm.Initialize(false)).Verifiable(); - _testEngine.Setup(te => te.GetExecutionManager(_mockRequestData.Object, _hostManager.Object, It.IsAny())).Returns(_executionManager.Object); + _testEngine.Setup(te => te.GetExecutionManager(_mockRequestData.Object, It.IsAny(), It.IsAny>())).Returns(_executionManager.Object); _testEngine.Setup(te => te.GetExtensionManager()).Returns(_extensionManager.Object); _testEngine.Setup(te => te.GetLoggerManager(_mockRequestData.Object)).Returns(_loggerManager.Object); @@ -293,7 +293,7 @@ public void CreateTestRunRequestShouldSetCustomHostLauncherOnEngineDefaultLaunch _hostManager.Setup(hm => hm.GetTestSources(testRunCriteria.Sources)) .Returns(testRunCriteria.Sources); - var testRunRequest = tp.CreateTestRunRequest(_mockRequestData.Object, testRunCriteria, new TestPlatformOptions()); + var testRunRequest = tp.CreateTestRunRequest(_mockRequestData.Object, testRunCriteria, new TestPlatformOptions(), new Dictionary()); var actualTestRunRequest = testRunRequest as TestRunRequest; Assert.AreEqual(testRunCriteria, actualTestRunRequest?.TestRunCriteria); diff --git a/test/Microsoft.TestPlatform.CrossPlatEngine.UnitTests/Client/Parallel/ParallelOperationManagerTests.cs b/test/Microsoft.TestPlatform.CrossPlatEngine.UnitTests/Client/Parallel/ParallelOperationManagerTests.cs index abe90f7359..ae305f4848 100644 --- a/test/Microsoft.TestPlatform.CrossPlatEngine.UnitTests/Client/Parallel/ParallelOperationManagerTests.cs +++ b/test/Microsoft.TestPlatform.CrossPlatEngine.UnitTests/Client/Parallel/ParallelOperationManagerTests.cs @@ -14,10 +14,10 @@ namespace TestPlatform.CrossPlatEngine.UnitTests.Client; //{ // private MockParallelOperationManager _proxyParallelManager; - public ParallelOperationManagerTests() - { - Func sampleCreator = - () => new SampleConcurrentClass(); + //public ParallelOperationManagerTests() + //{ + // Func sampleCreator = + // () => new SampleConcurrentClass(); // _proxyParallelManager = new MockParallelOperationManager(sampleCreator, 2, true); // } diff --git a/test/Microsoft.TestPlatform.CrossPlatEngine.UnitTests/Client/Parallel/ParallelProxyDiscoveryManagerTests.cs b/test/Microsoft.TestPlatform.CrossPlatEngine.UnitTests/Client/Parallel/ParallelProxyDiscoveryManagerTests.cs index 7486f0f5e2..cd4cee4c96 100644 --- a/test/Microsoft.TestPlatform.CrossPlatEngine.UnitTests/Client/Parallel/ParallelProxyDiscoveryManagerTests.cs +++ b/test/Microsoft.TestPlatform.CrossPlatEngine.UnitTests/Client/Parallel/ParallelProxyDiscoveryManagerTests.cs @@ -107,7 +107,7 @@ public void CreatingAndInitializingProxyExecutionManagerDoesNothingUntilThereIsA [TestMethod] public void AbortShouldCallAllConcurrentManagersOnce() { - var parallelDiscoveryManager = new ParallelProxyDiscoveryManager(_mockRequestData.Object, _createMockManager, parallelLevel: 1000, _runtimeProviders); + var parallelDiscoveryManager = new ParallelProxyDiscoveryManager(_mockRequestData.Object, _createMockManager, dataAggregator: new(), parallelLevel: 1000, _runtimeProviders); // Starting parallel discovery will create 2 proxy managers, which we will then promptly abort. parallelDiscoveryManager.DiscoverTests(_discoveryCriteriaWith2Sources, new Mock().Object); @@ -149,7 +149,7 @@ public void DiscoverTestsShouldProcessAllSources() [TestMethod] public void HandlePartialDiscoveryCompleteShouldReturnTrueIfDiscoveryWasAbortedWithEventHandler() { - var parallelDiscoveryManager = new ParallelProxyDiscoveryManager(_mockRequestData.Object, _createMockManager, 1, new List()); + var parallelDiscoveryManager = new ParallelProxyDiscoveryManager(_mockRequestData.Object, _createMockManager, dataAggregator: new(), parallelLevel: 1, new List()); var proxyDiscovermanager = new ProxyDiscoveryManager(_mockRequestData.Object, new Mock().Object, new Mock().Object); parallelDiscoveryManager.Abort(_mockEventHandler.Object); @@ -161,7 +161,7 @@ public void HandlePartialDiscoveryCompleteShouldReturnTrueIfDiscoveryWasAbortedW [TestMethod] public void HandlePartialDiscoveryCompleteShouldReturnTrueIfDiscoveryWasAborted() { - var parallelDiscoveryManager = new ParallelProxyDiscoveryManager(_mockRequestData.Object, _createMockManager, 1, new List()); + var parallelDiscoveryManager = new ParallelProxyDiscoveryManager(_mockRequestData.Object, _createMockManager, dataAggregator: new(), parallelLevel: 1, new List()); var proxyDiscovermanager = new ProxyDiscoveryManager(_mockRequestData.Object, new Mock().Object, new Mock().Object); parallelDiscoveryManager.Abort(); @@ -269,7 +269,7 @@ public void DiscoveryTestsShouldCatchExceptionAndHandleRawMessageOfTestMessage() [TestMethod] public void HandlePartialDiscoveryCompleteShouldCreateANewProxyDiscoveryManagerIfIsAbortedIsTrue() { - var parallelDiscoveryManager = new ParallelProxyDiscoveryManager(_mockRequestData.Object, _createMockManager, 1, _runtimeProviders); + var parallelDiscoveryManager = new ParallelProxyDiscoveryManager(_mockRequestData.Object, _createMockManager, dataAggregator: new(), parallelLevel: 1, _runtimeProviders); // Trigger discover tests, this will create a manager by calling the _createMockManager func // which dequeues it to _usedMockManagers. @@ -286,24 +286,25 @@ public void HandlePartialDiscoveryCompleteShouldCreateANewProxyDiscoveryManagerI Assert.AreEqual(2, _createMockManagerCalled); } - [TestMethod] - public void DiscoveryTestsWithCompletionMarksAllSourcesAsFullyDiscovered() - { - _testDiscoveryCriteria.TestCaseFilter = "Name~Test"; - var parallelDiscoveryManager = SetupDiscoveryManager(_proxyManagerFunc, 2, false); + // NOMERGE: fix this test, it is a new test that uses the old approach + //[TestMethod] + //public void DiscoveryTestsWithCompletionMarksAllSourcesAsFullyDiscovered() + //{ + // _testDiscoveryCriteria.TestCaseFilter = "Name~Test"; + // var parallelDiscoveryManager = SetupDiscoveryManager(_proxyManagerFunc, 2, false); - Task.Run(() => parallelDiscoveryManager.DiscoverTests(_testDiscoveryCriteria, _mockHandler.Object)); + // Task.Run(() => parallelDiscoveryManager.DiscoverTests(_testDiscoveryCriteria, _mockHandler.Object)); - Assert.IsTrue(_discoveryCompleted.Wait(TaskTimeout), "Test discovery not completed."); - Assert.AreEqual(_sources.Count, _processedSources.Count, "All Sources must be processed."); - CollectionAssert.AreEquivalent(_sources, _dataAggregator.GetSourcesWithStatus(DiscoveryStatus.FullyDiscovered)); - Assert.AreEqual(0, _dataAggregator.GetSourcesWithStatus(DiscoveryStatus.PartiallyDiscovered).Count); - Assert.AreEqual(0, _dataAggregator.GetSourcesWithStatus(DiscoveryStatus.NotDiscovered).Count); - } + // Assert.IsTrue(_discoveryCompleted.Wait(TaskTimeout), "Test discovery not completed."); + // Assert.AreEqual(_sources.Count, _processedSources.Count, "All Sources must be processed."); + // CollectionAssert.AreEquivalent(_sources, _dataAggregator.GetSourcesWithStatus(DiscoveryStatus.FullyDiscovered)); + // Assert.AreEqual(0, _dataAggregator.GetSourcesWithStatus(DiscoveryStatus.PartiallyDiscovered).Count); + // Assert.AreEqual(0, _dataAggregator.GetSourcesWithStatus(DiscoveryStatus.NotDiscovered).Count); + //} private IParallelProxyDiscoveryManager SetupDiscoveryManager(Func getProxyManager, int parallelLevel, bool abortDiscovery, int totalTests = 20) { - var parallelDiscoveryManager = new ParallelProxyDiscoveryManager(_mockRequestData.Object, getProxyManager, parallelLevel, _runtimeProviders); + var parallelDiscoveryManager = new ParallelProxyDiscoveryManager(_mockRequestData.Object, getProxyManager, dataAggregator: new(), parallelLevel, _runtimeProviders); SetupDiscoveryTests(_processedSources, abortDiscovery); // Setup a complete handler for parallel discovery manager @@ -366,7 +367,7 @@ private void AssertMissingAndDuplicateSources(List processedSources) private void InvokeAndVerifyInitialize(int maxParallelLevel, bool skipDefaultAdapters = false) { - var parallelDiscoveryManager = new ParallelProxyDiscoveryManager(_mockRequestData.Object, _createMockManager, maxParallelLevel, new List()); + var parallelDiscoveryManager = new ParallelProxyDiscoveryManager(_mockRequestData.Object, _createMockManager, dataAggregator: new(), maxParallelLevel, new List()); // Action parallelDiscoveryManager.Initialize(skipDefaultAdapters); diff --git a/test/Microsoft.TestPlatform.CrossPlatEngine.UnitTests/Client/Parallel/ParallelProxyExecutionManagerTests.cs b/test/Microsoft.TestPlatform.CrossPlatEngine.UnitTests/Client/Parallel/ParallelProxyExecutionManagerTests.cs index 6f08a992ce..cfe568b8f2 100644 --- a/test/Microsoft.TestPlatform.CrossPlatEngine.UnitTests/Client/Parallel/ParallelProxyExecutionManagerTests.cs +++ b/test/Microsoft.TestPlatform.CrossPlatEngine.UnitTests/Client/Parallel/ParallelProxyExecutionManagerTests.cs @@ -338,97 +338,98 @@ public void StartTestRunShouldCatchExceptionAndHandleRawMessageOfTestMessage() _mockEvenHandler.Verify(s => s.HandleRawMessage(It.Is(str => str.Contains(MessageType.TestMessage)))); } - [TestMethod] - public void StartTestRunShouldAggregateRunData() - { - var parallelExecutionManager = new ParallelProxyExecutionManager(_mockRequestData.Object, _createMockManager, 2, _runtimeProviders); - var syncObject = new object(); - - foreach (var manager in _preCreatedMockManagers) - { - manager.Setup(m => m.StartTestRun(It.IsAny(), It.IsAny())). - Callback( - (criteria, handler) => - { - lock (syncObject) - { - _processedSources.AddRange(criteria.Sources); - } - - Task.Delay(100).Wait(); - var stats = new Dictionary - { - { TestOutcome.Passed, 3 }, - { TestOutcome.Failed, 2 } - }; - var runAttachments = new Collection - { - new AttachmentSet(new Uri("hello://x/"), "Hello") - }; - var executorUris = new List() { "hello1" }; - bool isCanceled = false; - bool isAborted = false; - TimeSpan timespan = TimeSpan.FromMilliseconds(100); - - if (string.Equals(criteria.Sources?.FirstOrDefault(), "2.dll")) - { - isCanceled = true; - isAborted = true; - timespan = TimeSpan.FromMilliseconds(200); - } - - var completeArgs = new TestRunCompleteEventArgs(new - TestRunStatistics(5, stats), isCanceled, isAborted, null, runAttachments, new Collection(), timespan); - handler.HandleTestRunComplete(completeArgs, null, runAttachments, executorUris); - }); - } - - Exception? assertException = null; - _mockHandler.Setup(m => m.HandleTestRunComplete( - It.IsAny(), - It.IsAny(), - It.IsAny>(), - It.IsAny>())).Callback - , ICollection>( - (completeArgs, runChangedArgs, runAttachments, executorUris) => - { - try - { - Assert.AreEqual(TimeSpan.FromMilliseconds(200), completeArgs.ElapsedTimeInRunningTests, - "Time should be max of all"); - Assert.AreEqual(2, completeArgs.AttachmentSets.Count, - "All Complete Arg attachments should return"); - Assert.AreEqual(2, runAttachments.Count, "All RunContextAttachments should return"); - - Assert.IsTrue(completeArgs.IsAborted, "Aborted value must be OR of all values"); - Assert.IsTrue(completeArgs.IsCanceled, "Canceled value must be OR of all values"); - - Assert.AreEqual(10, completeArgs.TestRunStatistics.ExecutedTests, - "Stats must be aggregated properly"); - - Assert.AreEqual(6, completeArgs.TestRunStatistics.Stats[TestOutcome.Passed], - "Stats must be aggregated properly"); - Assert.AreEqual(4, completeArgs.TestRunStatistics.Stats[TestOutcome.Failed], - "Stats must be aggregated properly"); - } - catch (Exception ex) - { - assertException = ex; - } - finally - { - _executionCompleted.Set(); - } - }); - - Task.Run(() => parallelExecutionManager.StartTestRun(_testRunCriteriaWith2Sources, _mockEvenHandler.Object)); - - Assert.IsTrue(_executionCompleted.Wait(Timeout3Seconds), "Test run not completed."); - - Assert.IsNull(assertException, assertException?.ToString()); - Assert.AreEqual(_sources.Count, _processedSources.Count, "All Sources must be processed."); - AssertMissingAndDuplicateSources(_processedSources); - } + // NOMERGE: fix this test, it is a new test that uses the old approach + //[TestMethod] + //public void StartTestRunShouldAggregateRunData() + //{ + // var parallelExecutionManager = new ParallelProxyExecutionManager(_mockRequestData.Object, _createMockManager, 2, _runtimeProviders); + // var syncObject = new object(); + + // foreach (var manager in _preCreatedMockManagers) + // { + // manager.Setup(m => m.StartTestRun(It.IsAny(), It.IsAny())). + // Callback( + // (criteria, handler) => + // { + // lock (syncObject) + // { + // _processedSources.AddRange(criteria.Sources); + // } + + // Task.Delay(100).Wait(); + // var stats = new Dictionary + // { + // { TestOutcome.Passed, 3 }, + // { TestOutcome.Failed, 2 } + // }; + // var runAttachments = new Collection + // { + // new AttachmentSet(new Uri("hello://x/"), "Hello") + // }; + // var executorUris = new List() { "hello1" }; + // bool isCanceled = false; + // bool isAborted = false; + // TimeSpan timespan = TimeSpan.FromMilliseconds(100); + + // if (string.Equals(criteria.Sources?.FirstOrDefault(), "2.dll")) + // { + // isCanceled = true; + // isAborted = true; + // timespan = TimeSpan.FromMilliseconds(200); + // } + + // var completeArgs = new TestRunCompleteEventArgs(new + // TestRunStatistics(5, stats), isCanceled, isAborted, null, runAttachments, new Collection(), timespan); + // handler.HandleTestRunComplete(completeArgs, null, runAttachments, executorUris); + // }); + // } + + // Exception? assertException = null; + // _mockHandler.Setup(m => m.HandleTestRunComplete( + // It.IsAny(), + // It.IsAny(), + // It.IsAny>(), + // It.IsAny>())).Callback + // , ICollection>( + // (completeArgs, runChangedArgs, runAttachments, executorUris) => + // { + // try + // { + // Assert.AreEqual(TimeSpan.FromMilliseconds(200), completeArgs.ElapsedTimeInRunningTests, + // "Time should be max of all"); + // Assert.AreEqual(2, completeArgs.AttachmentSets.Count, + // "All Complete Arg attachments should return"); + // Assert.AreEqual(2, runAttachments.Count, "All RunContextAttachments should return"); + + // Assert.IsTrue(completeArgs.IsAborted, "Aborted value must be OR of all values"); + // Assert.IsTrue(completeArgs.IsCanceled, "Canceled value must be OR of all values"); + + // Assert.AreEqual(10, completeArgs.TestRunStatistics.ExecutedTests, + // "Stats must be aggregated properly"); + + // Assert.AreEqual(6, completeArgs.TestRunStatistics.Stats[TestOutcome.Passed], + // "Stats must be aggregated properly"); + // Assert.AreEqual(4, completeArgs.TestRunStatistics.Stats[TestOutcome.Failed], + // "Stats must be aggregated properly"); + // } + // catch (Exception ex) + // { + // assertException = ex; + // } + // finally + // { + // _executionCompleted.Set(); + // } + // }); + + // Task.Run(() => parallelExecutionManager.StartTestRun(_testRunCriteriaWith2Sources, _mockEvenHandler.Object)); + + // Assert.IsTrue(_executionCompleted.Wait(Timeout3Seconds), "Test run not completed."); + + // Assert.IsNull(assertException, assertException?.ToString()); + // Assert.AreEqual(_sources.Count, _processedSources.Count, "All Sources must be processed."); + // AssertMissingAndDuplicateSources(_processedSources); + //} private ParallelProxyExecutionManager SetupExecutionManager(Func proxyManagerFunc, int parallelLevel) { diff --git a/test/Microsoft.TestPlatform.CrossPlatEngine.UnitTests/TestEngineTests.cs b/test/Microsoft.TestPlatform.CrossPlatEngine.UnitTests/TestEngineTests.cs index 064e8cd7e2..d2d04fdccc 100644 --- a/test/Microsoft.TestPlatform.CrossPlatEngine.UnitTests/TestEngineTests.cs +++ b/test/Microsoft.TestPlatform.CrossPlatEngine.UnitTests/TestEngineTests.cs @@ -560,7 +560,7 @@ public void ProxyDataCollectionManagerShouldBeInitialzedWithCorrectTestSourcesWh var testRunCriteria = new TestRunCriteria(new List { "1.dll", "2.dll" }, 100, false, settingXml); - var executionManager = _testEngine.GetExecutionManager(_mockRequestData.Object, _testableTestRuntimeProvider, testRunCriteria); + var executionManager = _testEngine.GetExecutionManager(_mockRequestData.Object, testRunCriteria, new Dictionary()); Assert.IsTrue(((ProxyExecutionManagerWithDataCollection)executionManager).ProxyDataCollectionManager.Sources.Contains("1.dll")); } @@ -573,7 +573,7 @@ public void ProxyDataCollectionManagerShouldBeInitialzedWithCorrectTestSourcesWh var testCaseList = new List { new TestCase("x.y.z", new Uri("uri://dummy"), "x.dll") }; var testRunCriteria = new TestRunCriteria(testCaseList, 100, false, settingXml); - var executionManager = _testEngine.GetExecutionManager(_mockRequestData.Object, _testableTestRuntimeProvider, testRunCriteria); + var executionManager = _testEngine.GetExecutionManager(_mockRequestData.Object, testRunCriteria, new Dictionary()); Assert.IsTrue(((ProxyExecutionManagerWithDataCollection)executionManager).ProxyDataCollectionManager.Sources.Contains("x.dll")); } From 65e1e0fbeddcea04f75944dd459f901cc171a109 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Jare=C5=A1?= Date: Thu, 12 May 2022 15:00:04 +0200 Subject: [PATCH 088/112] Programmer tests build and run, I think I left some not passing before I left this task, to remind myself that warnings should not be written. --- playground/TestPlatform.Playground/Program.cs | 20 +++++++-- .../Properties/launchSettings.json | 2 +- .../Discovery/DiscoveryRequest.cs | 2 +- .../Execution/TestRunRequest.cs | 2 +- .../Parallel/ParallelProxyDiscoveryManager.cs | 45 ++++++++++++++----- .../Fakes/FakeTestBatchBuilder.cs | 19 ++++++-- .../Fakes/FakeTestDllBuilder.cs | 2 + .../MultiTFMRunAndDiscovery.cs | 29 +++++++++--- 8 files changed, 95 insertions(+), 26 deletions(-) diff --git a/playground/TestPlatform.Playground/Program.cs b/playground/TestPlatform.Playground/Program.cs index 130c862494..a799a03514 100644 --- a/playground/TestPlatform.Playground/Program.cs +++ b/playground/TestPlatform.Playground/Program.cs @@ -54,7 +54,7 @@ static void Main(string[] args) Path.Combine(playground, "MSTest1", "bin", "Debug", "net472", "MSTest1.dll"), Path.Combine(playground, "MSTest1", "bin", "Debug", "net5.0", "MSTest1.dll"), @"C:\Users\jajares\source\repos\TestProject48\TestProject48\bin\Debug\net48\TestProject48.dll", - @"C:\Users\jajares\source\repos\TestProject48\TestProject1\bin\x64\Debug\net48\win10-x64\TestProject1.dll" + @"C:\Users\jajares\source\repos\TestProject48\TestProject1\bin\Debug\net48\win10-x64\TestProject1.dll" }; // console mode @@ -62,7 +62,12 @@ static void Main(string[] args) try { File.WriteAllText(settingsFile, sourceSettings); - Process.Start(console, string.Join(" ", sources) + " --settings:" + settingsFile).WaitForExit(); + var process = Process.Start(console, string.Join(" ", sources) + " --settings:" + settingsFile + " --listtests"); + process.WaitForExit(); + if (process.ExitCode != 0) + { + throw new Exception($"Process failed with {process.ExitCode}"); + } } finally { @@ -77,18 +82,23 @@ static void Main(string[] args) }; var options = new TestPlatformOptions(); var r = new VsTestConsoleWrapper(console, consoleOptions); - r.RunTestsWithCustomTestHost(sources, sourceSettings, options, new TestRunHandler(), new DebuggerTestHostLauncher()); + var discoveryHandler = new PlaygroundTestDiscoveryHandler(); + r.DiscoverTests(sources, sourceSettings, options, discoveryHandler); + r.RunTestsWithCustomTestHost(discoveryHandler.TestCases, sourceSettings, options, new TestRunHandler(), new DebuggerTestHostLauncher()); } public class PlaygroundTestDiscoveryHandler : ITestDiscoveryEventsHandler, ITestDiscoveryEventsHandler2 { private int _testCasesCount; + public List TestCases { get; internal set; } = new List(); + public void HandleDiscoveredTests(IEnumerable discoveredTestCases) { Console.WriteLine($"[DISCOVERY.PROGRESS]"); Console.WriteLine(WriteTests(discoveredTestCases)); _testCasesCount += discoveredTestCases.Count(); + if (discoveredTestCases != null) { TestCases.AddRange(discoveredTestCases); } } public void HandleDiscoveryComplete(long totalTests, IEnumerable lastChunk, bool isAborted) @@ -96,6 +106,7 @@ public void HandleDiscoveryComplete(long totalTests, IEnumerable lastC Console.WriteLine($"[DISCOVERY.COMPLETE] aborted? {isAborted}, tests count: {totalTests}"); Console.WriteLine("Last chunk:"); Console.WriteLine(WriteTests(lastChunk)); + if (lastChunk != null) { TestCases.AddRange(lastChunk); } } public void HandleDiscoveryComplete(DiscoveryCompleteEventArgs discoveryCompleteEventArgs, IEnumerable lastChunk) @@ -109,6 +120,7 @@ public void HandleDiscoveryComplete(DiscoveryCompleteEventArgs discoveryComplete Console.WriteLine(WriteSources(discoveryCompleteEventArgs.PartiallyDiscoveredSources)); Console.WriteLine("Not discovered:"); Console.WriteLine(WriteSources(discoveryCompleteEventArgs.NotDiscoveredSources)); + if (lastChunk != null) { TestCases.AddRange(lastChunk); } } public void HandleLogMessage(TestMessageLevel level, string message) @@ -123,7 +135,7 @@ public void HandleRawMessage(string rawMessage) private static string WriteTests(IEnumerable testCases) => testCases?.Any() == true - ? "\t" + string.Join("\n\t", testCases.Select(r => r.DisplayName)) + ? "\t" + string.Join("\n\t", testCases.Select(r => r.Source + " " + r.DisplayName)) : "\t"; private static string WriteSources(IEnumerable sources) diff --git a/playground/TestPlatform.Playground/Properties/launchSettings.json b/playground/TestPlatform.Playground/Properties/launchSettings.json index 8f51ff3b0e..f0124eac48 100644 --- a/playground/TestPlatform.Playground/Properties/launchSettings.json +++ b/playground/TestPlatform.Playground/Properties/launchSettings.json @@ -7,7 +7,7 @@ "VSTEST_DEBUG_NOBP": "1", "VSTEST_RUNNER_DEBUG_ATTACHVS": "1", "VSTEST_HOST_DEBUG_ATTACHVS": "0", - "VSTEST_DATACOLLECTOR_DEBUG_ATTACHVS": "1" + "VSTEST_DATACOLLECTOR_DEBUG_ATTACHVS": "0" } } } diff --git a/src/Microsoft.TestPlatform.Client/Discovery/DiscoveryRequest.cs b/src/Microsoft.TestPlatform.Client/Discovery/DiscoveryRequest.cs index 25211c24f4..a9e5e58e94 100644 --- a/src/Microsoft.TestPlatform.Client/Discovery/DiscoveryRequest.cs +++ b/src/Microsoft.TestPlatform.Client/Discovery/DiscoveryRequest.cs @@ -256,7 +256,7 @@ public void HandleDiscoveryComplete(DiscoveryCompleteEventArgs discoveryComplete // and then we write again here. We should refactor this code and write only once. discoveryCompleteEventArgs.DiscoveredExtensions = TestExtensions.CreateMergedDictionary( discoveryCompleteEventArgs.DiscoveredExtensions, - TestPluginCache.Instance.TestExtensions.GetCachedExtensions()); + TestPluginCache.Instance.TestExtensions?.GetCachedExtensions()); if (RequestData.IsTelemetryOptedIn) { diff --git a/src/Microsoft.TestPlatform.Client/Execution/TestRunRequest.cs b/src/Microsoft.TestPlatform.Client/Execution/TestRunRequest.cs index 0efe23df94..f4cd9f93c7 100644 --- a/src/Microsoft.TestPlatform.Client/Execution/TestRunRequest.cs +++ b/src/Microsoft.TestPlatform.Client/Execution/TestRunRequest.cs @@ -409,7 +409,7 @@ public void HandleTestRunComplete(TestRunCompleteEventArgs runCompleteArgs!!, Te // and then we write again here. We should refactor this code and write only once. runCompleteArgs.DiscoveredExtensions = TestExtensions.CreateMergedDictionary( runCompleteArgs.DiscoveredExtensions, - TestPluginCache.Instance.TestExtensions.GetCachedExtensions()); + TestPluginCache.Instance.TestExtensions?.GetCachedExtensions()); if (_requestData.IsTelemetryOptedIn) { diff --git a/src/Microsoft.TestPlatform.CrossPlatEngine/Client/Parallel/ParallelProxyDiscoveryManager.cs b/src/Microsoft.TestPlatform.CrossPlatEngine/Client/Parallel/ParallelProxyDiscoveryManager.cs index c47be2aaac..56dfa78202 100644 --- a/src/Microsoft.TestPlatform.CrossPlatEngine/Client/Parallel/ParallelProxyDiscoveryManager.cs +++ b/src/Microsoft.TestPlatform.CrossPlatEngine/Client/Parallel/ParallelProxyDiscoveryManager.cs @@ -6,6 +6,7 @@ using System.Linq; using System.Threading.Tasks; +using Microsoft.TestPlatform; using Microsoft.VisualStudio.TestPlatform.CommunicationUtilities; using Microsoft.VisualStudio.TestPlatform.CommunicationUtilities.Interfaces; using Microsoft.VisualStudio.TestPlatform.CommunicationUtilities.ObjectModel; @@ -81,10 +82,14 @@ public void DiscoverTests(DiscoveryCriteria discoveryCriteria!!, ITestDiscoveryE var workloads = SplitToWorkloads(discoveryCriteria, _sourceToTestHostProviderMap); _availableTestSources = workloads.Count; - EqtTrace.Verbose("ParallelProxyDiscoveryManager: Start discovery. Total sources: " + _availableTestSources); + EqtTrace.Verbose("ParallelProxyDiscoveryManager.DiscoverTests: Start discovery. Total sources: " + _availableTestSources); - - // Marking all sources as not discovered before starting actual discovery + // Mark all sources as NotDiscovered here because if we get an early cancellation it's + // possible that we didn't yet start all the proxy managers and so we didn't mark all sources + // as NotDiscovered. + // For example, let's assume we have 10 sources, a batch size of 10 but only 8 cores, we + // will then spawn 8 instances of this and if we now cancel, we will have 2 sources not + // marked as NotDiscovered. _dataAggregator.MarkSourcesWithStatus(discoveryCriteria.Sources.ToList(), DiscoveryStatus.NotDiscovered); _parallelOperationManager.StartWork(workloads, eventHandler, GetParallelEventHandler, DiscoverTestsOnConcurrentManager); @@ -125,6 +130,17 @@ public void Close() /// public bool HandlePartialDiscoveryComplete(IProxyDiscoveryManager proxyDiscoveryManager, long totalTests, IEnumerable lastChunk, bool isAborted) { +#if DEBUG + // Ensures that the total count of sources remains the same between each discovery + // completion of the same initial discovery request. + var notDiscoveredCount = _dataAggregator.GetSourcesWithStatus(DiscoveryStatus.NotDiscovered).Count; + var partiallyDiscoveredCount = _dataAggregator.GetSourcesWithStatus(DiscoveryStatus.PartiallyDiscovered).Count; + var fullyDiscoveredCount = _dataAggregator.GetSourcesWithStatus(DiscoveryStatus.FullyDiscovered).Count; + var expectedCount = _availableTestSources; + Debug.Assert(notDiscoveredCount + partiallyDiscoveredCount + fullyDiscoveredCount == expectedCount, + $"Total count of sources ({expectedCount}) should match the count of sources with status not discovered ({notDiscoveredCount}), partially discovered ({partiallyDiscoveredCount}) and fully discovered ({fullyDiscoveredCount})."); +#endif + var allDiscoverersCompleted = false; // TODO: Interlocked.Increment the count, and the condition below probably does not need to be in a lock? lock (_discoveryStatusLockObject) @@ -136,19 +152,26 @@ public bool HandlePartialDiscoveryComplete(IProxyDiscoveryManager proxyDiscovery // If there are no more sources/testcases, a parallel executor is truly done with discovery allDiscoverersCompleted = _discoveryCompletedClients == _availableTestSources; - EqtTrace.Verbose("ParallelProxyDiscoveryManager: HandlePartialDiscoveryComplete: Total completed clients = {0}, Discovery complete = {1}.", _discoveryCompletedClients, allDiscoverersCompleted); + EqtTrace.Verbose("ParallelProxyDiscoveryManager.HandlePartialDiscoveryComplete: Total completed clients = {0}, Discovery complete = {1}, Aborted = {2}, Abort requested: {3}.", _discoveryCompletedClients, allDiscoverersCompleted, isAborted, IsAbortRequested); } - /* - If discovery is complete or discovery aborting was requsted by testPlatfrom(user) - we need to stop all ongoing discoveries, because we want to separate aborting request - when testhost crashed by itself and when user requested it (f.e. through TW) - Schedule the clean up for managers and handlers. - */ + // If discovery is complete or discovery aborting was requested by testPlatfrom(user) + // we need to stop all ongoing discoveries, because we want to separate aborting request + // when testhost crashed by itself and when user requested it (e.g. through TW). + // Schedule the clean up for managers and handlers. if (allDiscoverersCompleted || IsAbortRequested) { _parallelOperationManager.StopAllManagers(); + if (allDiscoverersCompleted) + { + EqtTrace.Verbose("ParallelProxyDiscoveryManager.HandlePartialDiscoveryComplete: All sources were discovered."); + } + else + { + EqtTrace.Verbose($"ParallelProxyDiscoveryManager.HandlePartialDiscoveryComplete: Abort was requested."); + } + return true; } @@ -226,5 +249,7 @@ private void DiscoverTestsOnConcurrentManager(IProxyDiscoveryManager proxyDiscov handler.HandleDiscoveryComplete(discoveryCompleteEventsArgs, null); }, TaskContinuationOptions.OnlyOnFaulted); + + EqtTrace.Verbose("ProxyParallelDiscoveryManager.DiscoverTestsOnConcurrentManager: No sources available for discovery."); } } diff --git a/test/vstest.ProgrammerTests/Fakes/FakeTestBatchBuilder.cs b/test/vstest.ProgrammerTests/Fakes/FakeTestBatchBuilder.cs index cfecaeb09a..54c9e9f3a9 100644 --- a/test/vstest.ProgrammerTests/Fakes/FakeTestBatchBuilder.cs +++ b/test/vstest.ProgrammerTests/Fakes/FakeTestBatchBuilder.cs @@ -11,6 +11,7 @@ internal class FakeTestBatchBuilder public TimeSpan Duration { get; private set; } public int BatchSize { get; private set; } public static List> Empty => new(); + public string Source { get; private set; } public FakeTestBatchBuilder() { @@ -45,6 +46,17 @@ internal FakeTestBatchBuilder WithBatchSize(int batchSize) return this; } + /// + /// Sets the dll path (source) to be the provided value. + /// + /// + /// + internal FakeTestBatchBuilder WithDllPath(string path) + { + Source = path; + return this; + } + internal List> Build() { if (BatchSize == 0 && TotalCount != 0) @@ -55,16 +67,17 @@ internal List> Build() var numberOfBatches = Math.DivRem(TotalCount, BatchSize, out int remainder); - // TODO: Add adapter uri, and dll name + var source = Source ?? "DummySourceFileName"; + // TODO: Add adapter uri // TODO: set duration var batches = Enumerable.Range(0, numberOfBatches) .Select(batchNumber => Enumerable.Range(0, BatchSize) - .Select((index) => new TestResult(new TestCase($"Test{batchNumber}-{index}", new Uri("some://uri"), "DummySourceFileName"))).ToList()).ToList(); + .Select((index) => new TestResult(new TestCase($"Test{batchNumber}-{index}", new Uri("some://uri"), source))).ToList()).ToList(); if (remainder > 0) { var reminderBatch = Enumerable.Range(0, remainder) - .Select((index) => new TestResult(new TestCase($"Test{numberOfBatches + 1}-{index}", new Uri("some://uri"), "DummySourceFileName"))).ToList(); + .Select((index) => new TestResult(new TestCase($"Test{numberOfBatches + 1}-{index}", new Uri("some://uri"), source))).ToList(); batches.Add(reminderBatch); } diff --git a/test/vstest.ProgrammerTests/Fakes/FakeTestDllBuilder.cs b/test/vstest.ProgrammerTests/Fakes/FakeTestDllBuilder.cs index 14e7bca260..47b53aca90 100644 --- a/test/vstest.ProgrammerTests/Fakes/FakeTestDllBuilder.cs +++ b/test/vstest.ProgrammerTests/Fakes/FakeTestDllBuilder.cs @@ -51,6 +51,7 @@ internal FakeTestDllBuilder WithTestBatches(List> testBatches) internal FakeTestDllBuilder WithTestCount(int totalCount, int? batchSize = null) { _testBatches = new FakeTestBatchBuilder() + .WithDllPath(_path) .WithTotalCount(totalCount) .WithBatchSize(batchSize ?? totalCount) .Build(); @@ -63,6 +64,7 @@ internal FakeTestDllFile Build() if (_testBatches == null) { _testBatches = new FakeTestBatchBuilder() + .WithDllPath(_path) .WithTotalCount(10) .WithBatchSize(5) .Build(); diff --git a/test/vstest.ProgrammerTests/MultiTFMRunAndDiscovery.cs b/test/vstest.ProgrammerTests/MultiTFMRunAndDiscovery.cs index 582ae70d62..9149627475 100644 --- a/test/vstest.ProgrammerTests/MultiTFMRunAndDiscovery.cs +++ b/test/vstest.ProgrammerTests/MultiTFMRunAndDiscovery.cs @@ -10,6 +10,8 @@ using Microsoft.VisualStudio.TestPlatform.ObjectModel.Client; using Microsoft.VisualStudio.TestPlatform.Utilities; using Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.Payloads; +using Microsoft.VisualStudio.TestPlatform.CommunicationUtilities; +using System.Reflection; namespace vstest.ProgrammerTests; @@ -108,14 +110,14 @@ public async Task A() fixture.ProcessHelper.Processes.Where(p => p.Started).Should().HaveCount(2); var startWithSources1 = testhost1.FakeCommunicationChannel.ProcessedMessages.Single(m => m.Request.MessageType == MessageType.StartDiscovery); - var startWithSources1Text = startWithSources1.Request.Payload.Select(t => t.ToString()).JoinBySpace(); + var startWithSources1Text = startWithSources1.Request.GetRawMessage(); // We sent mstest1.dll. startWithSources1Text.Should().Contain("mstest1.dll"); // And we sent net5 as the target framework, because that is the framework of mstest1.dll. startWithSources1Text.Should().Contain(KnownFrameworkStrings.Net5); var startWithSources2 = testhost2.FakeCommunicationChannel.ProcessedMessages.Single(m => m.Request.MessageType == MessageType.StartDiscovery); - var startWithSources2Text = startWithSources2.Request.Payload.Select(t => t.ToString()).JoinBySpace(); + var startWithSources2Text = startWithSources2.Request.GetRawMessage(); // We sent mstest2.dll. startWithSources2Text.Should().Contain("mstest2.dll"); // And we sent net48 as the target framework, because that is the framework of mstest2.dll. @@ -216,13 +218,13 @@ public async Task B() fixture.ProcessHelper.Processes.Where(p => p.Started).Should().HaveCount(2); var startWithSources1 = testhost1.FakeCommunicationChannel.ProcessedMessages.Single(m => m.Request.MessageType == MessageType.StartDiscovery); - var startWithSources1Text = startWithSources1.Request.Payload.Select(t => t.ToString()).JoinBySpace(); + var startWithSources1Text = startWithSources1.Request.GetRawMessage(); // We sent mstest1.dll and net7 because that is what we have in settings. startWithSources1Text.Should().Contain("mstest1.dll"); startWithSources1Text.Should().Contain(KnownFrameworkStrings.Net7); var startWithSources2 = testhost2.FakeCommunicationChannel.ProcessedMessages.Single(m => m.Request.MessageType == MessageType.StartDiscovery); - var startWithSources2Text = startWithSources2.Request.Payload.Select(t => t.ToString()).JoinBySpace(); + var startWithSources2Text = startWithSources2.Request.GetRawMessage(); // We sent mstest2.dll and net7 because that is what we have in settings. startWithSources2Text.Should().Contain("mstest2.dll"); startWithSources2Text.Should().Contain(KnownFrameworkStrings.Net7); @@ -324,14 +326,14 @@ public async Task C() fixture.ProcessHelper.Processes.Where(p => p.Started).Should().HaveCount(2); var startWithSources1 = testhost1.FakeCommunicationChannel.ProcessedMessages.Single(m => m.Request.MessageType == MessageType.StartTestExecutionWithSources); - var startWithSources1Text = startWithSources1.Request.Payload.Select(t => t.ToString()).JoinBySpace(); + var startWithSources1Text = startWithSources1.Request.GetRawMessage(); // We sent mstest1.dll. startWithSources1Text.Should().Contain("mstest1.dll"); // And we sent net5 as the target framework, because that is the framework of mstest1.dll. startWithSources1Text.Should().Contain(KnownFrameworkStrings.Net5); var startWithSources2 = testhost2.FakeCommunicationChannel.ProcessedMessages.Single(m => m.Request.MessageType == MessageType.StartTestExecutionWithSources); - var startWithSources2Text = startWithSources2.Request.Payload.Select(t => t.ToString()).JoinBySpace(); + var startWithSources2Text = startWithSources2.Request.GetRawMessage(); // We sent mstest2.dll. startWithSources2Text.Should().Contain("mstest2.dll"); // And we sent net48 as the target framework, because that is the framework of mstest2.dll. @@ -685,6 +687,21 @@ public async Task GivenMultipleMsTestAssembliesThatHaveTheSameArchitecture_AndHa } } +internal static class MessageExtensions +{ + private static MethodInfo? s_messageProperty; + + internal static string GetRawMessage(this Message request) + { + if (s_messageProperty == null) + { + s_messageProperty = request.GetType().GetProperty("RawMessage")!.GetGetMethod(); + } + + return (string)s_messageProperty!.Invoke(request, new object[0])!; + } +} + // Test and improvmement ideas: // TODO: passing null runsettings does not fail fast, instead it fails in Fakes settings code // TODO: passing empty string fails in the xml parser code From 0b25dff462b60c1bbb11b22ffc269dac6c098210 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Jare=C5=A1?= Date: Thu, 12 May 2022 15:20:45 +0200 Subject: [PATCH 089/112] Try with test sessions --- playground/TestPlatform.Playground/Program.cs | 31 +++++++++++++++++-- 1 file changed, 29 insertions(+), 2 deletions(-) diff --git a/playground/TestPlatform.Playground/Program.cs b/playground/TestPlatform.Playground/Program.cs index a799a03514..493b23200b 100644 --- a/playground/TestPlatform.Playground/Program.cs +++ b/playground/TestPlatform.Playground/Program.cs @@ -82,9 +82,11 @@ static void Main(string[] args) }; var options = new TestPlatformOptions(); var r = new VsTestConsoleWrapper(console, consoleOptions); + var sessionHandler = new TestSessionHandler(); + r.StartTestSession(sources, sourceSettings, sessionHandler); var discoveryHandler = new PlaygroundTestDiscoveryHandler(); - r.DiscoverTests(sources, sourceSettings, options, discoveryHandler); - r.RunTestsWithCustomTestHost(discoveryHandler.TestCases, sourceSettings, options, new TestRunHandler(), new DebuggerTestHostLauncher()); + r.DiscoverTests(sources, sourceSettings, options, sessionHandler.TestSessionInfo, discoveryHandler); + r.RunTestsWithCustomTestHost(discoveryHandler.TestCases, sourceSettings, options, sessionHandler.TestSessionInfo, new TestRunHandler(), new DebuggerTestHostLauncher()); } public class PlaygroundTestDiscoveryHandler : ITestDiscoveryEventsHandler, ITestDiscoveryEventsHandler2 @@ -212,3 +214,28 @@ public int LaunchTestHost(TestProcessStartInfo defaultTestHostStartInfo, Cancell } } } + +internal class TestSessionHandler : ITestSessionEventsHandler +{ + public TestSessionInfo TestSessionInfo { get; private set; } + + public void HandleLogMessage(TestMessageLevel level, string message) + { + + } + + public void HandleRawMessage(string rawMessage) + { + + } + + public void HandleStartTestSessionComplete(StartTestSessionCompleteEventArgs eventArgs) + { + TestSessionInfo = eventArgs.TestSessionInfo; + } + + public void HandleStopTestSessionComplete(StopTestSessionCompleteEventArgs eventArgs) + { + + } +} From 0f1e8bd19ebf2662e5738b2d96caf1f532068ae4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Jare=C5=A1?= Date: Thu, 12 May 2022 16:12:41 +0200 Subject: [PATCH 090/112] Fix programmer tests and add DISABLE_MULTI_TFM --- .../Properties/launchSettings.json | 2 +- .../FeatureFlag/FeatureFlag.cs | 2 +- .../TestPlatformHelpers/TestRequestManager.cs | 35 ++++++-- .../MultiTFMRunAndDiscovery.cs | 87 ++++++------------- 4 files changed, 54 insertions(+), 72 deletions(-) diff --git a/playground/TestPlatform.Playground/Properties/launchSettings.json b/playground/TestPlatform.Playground/Properties/launchSettings.json index f0124eac48..7cd5a38d95 100644 --- a/playground/TestPlatform.Playground/Properties/launchSettings.json +++ b/playground/TestPlatform.Playground/Properties/launchSettings.json @@ -5,7 +5,7 @@ "environmentVariables": { "VSTEST_CONNECTION_TIMEOUT": "999", "VSTEST_DEBUG_NOBP": "1", - "VSTEST_RUNNER_DEBUG_ATTACHVS": "1", + "VSTEST_RUNNER_DEBUG_ATTACHVS": "0", "VSTEST_HOST_DEBUG_ATTACHVS": "0", "VSTEST_DATACOLLECTOR_DEBUG_ATTACHVS": "0" } diff --git a/src/Microsoft.TestPlatform.CoreUtilities/FeatureFlag/FeatureFlag.cs b/src/Microsoft.TestPlatform.CoreUtilities/FeatureFlag/FeatureFlag.cs index 595953133b..0307607a40 100644 --- a/src/Microsoft.TestPlatform.CoreUtilities/FeatureFlag/FeatureFlag.cs +++ b/src/Microsoft.TestPlatform.CoreUtilities/FeatureFlag/FeatureFlag.cs @@ -49,7 +49,7 @@ private FeatureFlag() { } public const string DISABLE_FASTER_JSON_SERIALIZATION = VSTEST_ + nameof(DISABLE_FASTER_JSON_SERIALIZATION); // TODO: Invert this to disable. - public const string MULTI_TFM_RUN = VSTEST_ + nameof(MULTI_TFM_RUN); + public const string DISABLE_MULTI_TFM_RUN = VSTEST_ + nameof(DISABLE_MULTI_TFM_RUN); [Obsolete("Only use this in tests.")] internal static void Reset() diff --git a/src/vstest.console/TestPlatformHelpers/TestRequestManager.cs b/src/vstest.console/TestPlatformHelpers/TestRequestManager.cs index ffca39d024..38c11f73b2 100644 --- a/src/vstest.console/TestPlatformHelpers/TestRequestManager.cs +++ b/src/vstest.console/TestPlatformHelpers/TestRequestManager.cs @@ -727,15 +727,34 @@ private bool UpdateRunSettingsIfRequired( defaultArchitecture, out Architecture chosenPlatform, out sourceToArchitectureMap); - CheckSourcesForCompatibility( - chosenFramework, - chosenPlatform, - defaultArchitecture, - sourceToArchitectureMap, - sourceToFrameworkMap, - registrar); - // TODO: NOMERGE: revert all architectures and frameworks on our map to the one common framework and architecture. + if (FeatureFlag.Instance.IsSet(FeatureFlag.DISABLE_MULTI_TFM_RUN)) + { + // Before MULTI_TFM feature the sourceToArchitectureMap and sourceToFrameworkMap were only used as informational + // to be able to do this compatibility check and print warning. And in the later steps only chosenPlatform, chosenFramework + // were used, that represented the single architecture and framework to be used. + // + // After MULTI_TFM sourceToArchitectureMap and sourceToFrameworkMap are the source of truth, and are propagated forward, + // so when we want to revert to the older behavior we need to re-enable the check, and unify all the architecture and + // framework entries to the same chosen value. + CheckSourcesForCompatibility( + chosenFramework, + chosenPlatform, + defaultArchitecture, + sourceToArchitectureMap, + sourceToFrameworkMap, + registrar); + + foreach (var key in sourceToArchitectureMap.Keys) + { + sourceToArchitectureMap[key] = chosenPlatform; + } + + foreach (var key in sourceToFrameworkMap.Keys) + { + sourceToFrameworkMap[key] = chosenFramework; + } + } settingsUpdated |= UpdateDesignMode(document, runConfiguration); settingsUpdated |= UpdateCollectSourceInformation(document, runConfiguration); diff --git a/test/vstest.ProgrammerTests/MultiTFMRunAndDiscovery.cs b/test/vstest.ProgrammerTests/MultiTFMRunAndDiscovery.cs index 9149627475..5e916315e1 100644 --- a/test/vstest.ProgrammerTests/MultiTFMRunAndDiscovery.cs +++ b/test/vstest.ProgrammerTests/MultiTFMRunAndDiscovery.cs @@ -30,15 +30,7 @@ Then two testhosts should be started that target the same framework as each asse public async Task A() { // -- arrange - using var fixture = new Fixture( - new FixtureOptions - { - FeatureFlags = new Dictionary - { - [FeatureFlag.MULTI_TFM_RUN] = true - } - } - ); + using var fixture = new Fixture(); var mstest1Dll = new FakeTestDllBuilder() .WithPath(@"X:\fake\mstest1.dll") @@ -138,15 +130,7 @@ Then two testhosts should be started that target the framework chosen by runsett public async Task B() { // -- arrange - using var fixture = new Fixture( - new FixtureOptions - { - FeatureFlags = new Dictionary - { - [FeatureFlag.MULTI_TFM_RUN] = true - } - } - ); + using var fixture = new Fixture(); var mstest1Dll = new FakeTestDllBuilder() .WithPath(@"X:\fake\mstest1.dll") @@ -246,15 +230,7 @@ Then two testhosts should be started that target the same framework as each asse public async Task C() { // -- arrange - using var fixture = new Fixture( - new FixtureOptions - { - FeatureFlags = new Dictionary - { - [FeatureFlag.MULTI_TFM_RUN] = true - } - } - ); + using var fixture = new Fixture(); var mstest1Dll = new FakeTestDllBuilder() .WithPath(@"X:\fake\mstest1.dll") @@ -354,15 +330,7 @@ Then two testhosts should be started that target the framework chosen by runsett public async Task D() { // -- arrange - using var fixture = new Fixture( - new FixtureOptions - { - FeatureFlags = new Dictionary - { - [FeatureFlag.MULTI_TFM_RUN] = true - } - } - ); + using var fixture = new Fixture(); var mstest1Dll = new FakeTestDllBuilder() .WithPath(@"X:\fake\mstest1.dll") @@ -434,13 +402,13 @@ public async Task D() fixture.ProcessHelper.Processes.Where(p => p.Started).Should().HaveCount(2); var startWithSources1 = testhost1.FakeCommunicationChannel.ProcessedMessages.Single(m => m.Request.MessageType == MessageType.StartTestExecutionWithSources); - var startWithSources1Text = startWithSources1.Request.Payload.Select(t => t.ToString()).JoinBySpace(); + var startWithSources1Text = startWithSources1.Request.GetRawMessage(); // We sent mstest1.dll. startWithSources1Text.Should().Contain("mstest1.dll"); startWithSources1Text.Should().Contain(KnownFrameworkStrings.Net7); var startWithSources2 = testhost2.FakeCommunicationChannel.ProcessedMessages.Single(m => m.Request.MessageType == MessageType.StartTestExecutionWithSources); - var startWithSources2Text = startWithSources2.Request.Payload.Select(t => t.ToString()).JoinBySpace(); + var startWithSources2Text = startWithSources2.Request.GetRawMessage(); // We sent mstest2.dll. startWithSources2Text.Should().Contain("mstest2.dll"); startWithSources2Text.Should().Contain(KnownFrameworkStrings.Net7); @@ -464,15 +432,7 @@ Then two testhosts should be started that target the framework chosen by runsett public async Task E() { // -- arrange - using var fixture = new Fixture( - new FixtureOptions - { - FeatureFlags = new Dictionary - { - [FeatureFlag.MULTI_TFM_RUN] = true - } - } - ); + using var fixture = new Fixture(); var mstest1Dll = new FakeTestDllBuilder() .WithPath(@"X:\fake\mstest1.dll") @@ -559,13 +519,13 @@ public async Task E() fixture.ProcessHelper.Processes.Where(p => p.Started).Should().HaveCount(2); var startWithSources1 = testhost1.FakeCommunicationChannel.ProcessedMessages.Single(m => m.Request.MessageType == MessageType.StartTestExecutionWithSources); - var startWithSources1Text = startWithSources1.Request.Payload.Select(t => t.ToString()).JoinBySpace(); + var startWithSources1Text = startWithSources1.Request.GetRawMessage(); // We sent mstest1.dll. startWithSources1Text.Should().Contain("mstest1.dll"); startWithSources1Text.Should().Contain(mstest1Dll.FrameworkName.ToString()); var startWithSources2 = testhost2.FakeCommunicationChannel.ProcessedMessages.Single(m => m.Request.MessageType == MessageType.StartTestExecutionWithSources); - var startWithSources2Text = startWithSources2.Request.Payload.Select(t => t.ToString()).JoinBySpace(); + var startWithSources2Text = startWithSources2.Request.GetRawMessage(); // We sent mstest2.dll. startWithSources2Text.Should().Contain("mstest2.dll"); startWithSources2Text.Should().Contain(mstest2Dll.FrameworkName.ToString()); @@ -577,9 +537,16 @@ public async Task E() public class MultiTFMRunAndDiscoveryCompatibilityMode { - // TODO: NOMERGE: Add backward compats flag functionality and test it. - [Exclude] - public async Task GivenMultipleMsTestAssembliesThatHaveTheSameArchitecture_AndHaveDifferentTargetFrameworks_AndMULTI_TFM_RUNFeatureFlagIsDisabled_WhenTestsAreRun_ThenTwoTesthostsAreStartedBothForTheSameTFM() + [Test(@" + Given two test assemblies that have the same architecture + but have different target frameworks. + + When DISABLE_MULTI_TFM_RUN is enabled + and we execute tests. + + Then two testhosts are both started for the same TFM. + ")] + public async Task E() { // -- arrange using var fixture = new Fixture( @@ -587,7 +554,7 @@ public async Task GivenMultipleMsTestAssembliesThatHaveTheSameArchitecture_AndHa { FeatureFlags = new Dictionary { - [FeatureFlag.MULTI_TFM_RUN] = false + [FeatureFlag.DISABLE_MULTI_TFM_RUN] = true } } ); @@ -649,14 +616,10 @@ public async Task GivenMultipleMsTestAssembliesThatHaveTheSameArchitecture_AndHa mstest1Dll.FrameworkName.Should().NotBe(mstest2Dll.FrameworkName); // -- act - // TODO: Building whole default runconfiguration is needed here, because TestRequestManager does not ensure the basic settings are populated, - // and all methods that populate them just silently fail, so TestHostProvider does not get any useful settings. - var runConfiguration = new RunConfiguration().ToXml().OuterXml; var testRunRequestPayload = new TestRunRequestPayload { Sources = new List { mstest1Dll.Path, mstest2Dll.Path }, - - RunSettings = $"{runConfiguration}" + RunSettings = $"", }; await testRequestManager.ExecuteWithAbort(tm => tm.RunTests(testRunRequestPayload, testHostLauncher: null, fixture.TestRunEventsRegistrar, fixture.ProtocolConfig)); @@ -670,17 +633,17 @@ public async Task GivenMultipleMsTestAssembliesThatHaveTheSameArchitecture_AndHa // We started both testhosts, even thought we know one of them is incompatible. fixture.ProcessHelper.Processes.Where(p => p.Started).Should().HaveCount(2); var startWithSources1 = testhost1.FakeCommunicationChannel.ProcessedMessages.Single(m => m.Request.MessageType == MessageType.StartTestExecutionWithSources); - var startWithSources1Text = startWithSources1.Request.Payload.Select(t => t.ToString()).JoinBySpace(); + var startWithSources1Text = startWithSources1.Request.GetRawMessage(); // We sent mstest1.dll startWithSources1Text.Should().Contain("mstest1.dll"); - // And we sent netcoreapp1.0 as the target framework + // And we sent netcoreapp1.0 as the target framework, because that is the common fallback startWithSources1Text.Should().Contain(KnownFrameworkStrings.Netcoreapp1); var startWithSources2 = testhost2.FakeCommunicationChannel.ProcessedMessages.Single(m => m.Request.MessageType == MessageType.StartTestExecutionWithSources); - var startWithSources2Text = startWithSources2.Request.Payload.Select(t => t.ToString()).JoinBySpace(); + var startWithSources2Text = startWithSources2.Request.GetRawMessage(); // We sent mstest2.dll startWithSources2Text.Should().Contain("mstest2.dll"); - // And we sent netcoreapp1.0 as the target framework, even though it is incompatible + // And we sent netcoreapp1.0 as the target framework, because that is the common fallback, even though the source is not compatible with it startWithSources2Text.Should().Contain(KnownFrameworkStrings.Netcoreapp1); fixture.ExecutedTests.Should().HaveCount(mstest1Dll.TestCount); From f7b8e941833b301b9e554e15083def9e51e643c1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Jare=C5=A1?= Date: Thu, 12 May 2022 17:23:00 +0200 Subject: [PATCH 091/112] Re-enable warnings for multtfm when there are settings forcing framework or platform --- .../Resources/CommonResources.Designer.cs | 6 +- .../Resources/CommonResources.resx | 6 +- .../TestPlatformHelpers/TestRequestManager.cs | 84 ++++++++++++++----- .../Fakes/FakeTestRunEventsRegistrar.cs | 1 + test/vstest.ProgrammerTests/Fakes/Fixture.cs | 2 + .../MultiTFMRunAndDiscovery.cs | 18 ++-- 6 files changed, 82 insertions(+), 35 deletions(-) diff --git a/src/Microsoft.TestPlatform.ObjectModel/Resources/CommonResources.Designer.cs b/src/Microsoft.TestPlatform.ObjectModel/Resources/CommonResources.Designer.cs index 3a37b375e5..696900893f 100644 --- a/src/Microsoft.TestPlatform.ObjectModel/Resources/CommonResources.Designer.cs +++ b/src/Microsoft.TestPlatform.ObjectModel/Resources/CommonResources.Designer.cs @@ -20,7 +20,7 @@ namespace Microsoft.VisualStudio.TestPlatform.ObjectModel.Resources { // class via a tool like ResGen or Visual Studio. // To add or remove a member, edit your .ResX file then rerun ResGen // with the /str option, or rebuild your VS project. - [global::System.CodeDom.Compiler.GeneratedCodeAttribute("System.Resources.Tools.StronglyTypedResourceBuilder", "16.0.0.0")] + [global::System.CodeDom.Compiler.GeneratedCodeAttribute("System.Resources.Tools.StronglyTypedResourceBuilder", "17.0.0.0")] [global::System.Diagnostics.DebuggerNonUserCodeAttribute()] [global::System.Runtime.CompilerServices.CompilerGeneratedAttribute()] public class CommonResources { @@ -71,7 +71,7 @@ public static string CannotBeNullOrEmpty { } /// - /// Looks up a localized string similar to Test run detected DLL(s) which were built for different framework and platform versions. Following DLL(s) do not match current settings, which are {0} framework and {1} platform.{2}Go to {3} for more details on managing these settings.. + /// Looks up a localized string similar to Test run detected DLL(s) which would use different framework and platform versions. Following DLL(s) do not match current settings, which are {0} framework and {1} platform.{2}Go to {3} for more details on managing these settings.. /// public static string DisplayChosenSettings { get { @@ -98,7 +98,7 @@ public static string NoMatchingSourcesFound { } /// - /// Looks up a localized string similar to {0} is built for Framework {1} and Platform {2}.. + /// Looks up a localized string similar to {0} would use Framework {1} and Platform {2}.. /// public static string SourceIncompatible { get { diff --git a/src/Microsoft.TestPlatform.ObjectModel/Resources/CommonResources.resx b/src/Microsoft.TestPlatform.ObjectModel/Resources/CommonResources.resx index be18466964..329f3050ef 100644 --- a/src/Microsoft.TestPlatform.ObjectModel/Resources/CommonResources.resx +++ b/src/Microsoft.TestPlatform.ObjectModel/Resources/CommonResources.resx @@ -121,7 +121,7 @@ The parameter cannot be null or empty. - Test run detected DLL(s) which were built for different framework and platform versions. Following DLL(s) do not match current settings, which are {0} framework and {1} platform.{2}Go to {3} for more details on managing these settings. + Test run detected DLL(s) which would use different framework and platform versions. Following DLL(s) do not match current settings, which are {0} framework and {1} platform.{2}Go to {3} for more details on managing these settings. Settings file provided does not conform to required format. @@ -130,6 +130,6 @@ None of the provided test containers match the Platform Architecture and .Net Framework settings for the test run. Platform: {0} .Net Framework: {1}. Go to http://go.microsoft.com/fwlink/?LinkID=330428 for more details on managing these settings. - {0} is built for Framework {1} and Platform {2}. + {0} would use Framework {1} and Platform {2}. - \ No newline at end of file + diff --git a/src/vstest.console/TestPlatformHelpers/TestRequestManager.cs b/src/vstest.console/TestPlatformHelpers/TestRequestManager.cs index 38c11f73b2..718d753eef 100644 --- a/src/vstest.console/TestPlatformHelpers/TestRequestManager.cs +++ b/src/vstest.console/TestPlatformHelpers/TestRequestManager.cs @@ -672,7 +672,11 @@ private bool UpdateRunSettingsIfRequired( var loggerRunSettings = XmlRunSettingsUtilities.GetLoggerRunSettings(runsettingsXml) ?? new LoggerRunSettings(); - settingsUpdated |= UpdateFrameworkInRunSettingsIfRequired( + + // True when runsettings don't set target framework. False when runsettings force target framework + // in both cases the sourceToFrameworkMap is populated with the real frameworks as we inferred them + // from dlls. For sources like .js, we return the default framework. + var frameworkWasAutodetected = UpdateFrameworkInRunSettingsIfRequired( document, navigator, sources, @@ -680,6 +684,9 @@ private bool UpdateRunSettingsIfRequired( out Framework chosenFramework, out sourceToFrameworkMap); + settingsUpdated |= frameworkWasAutodetected; + var frameworkSetByRunsettings = !frameworkWasAutodetected; + // Choose default architecture based on the framework. // For .NET core, the default platform architecture should be based on the process. Architecture defaultArchitecture = Architecture.X86; @@ -720,7 +727,10 @@ private bool UpdateRunSettingsIfRequired( EqtTrace.Verbose($"TestRequestManager.UpdateRunSettingsIfRequired: Default architecture: {defaultArchitecture} IsDefaultTargetArchitecture: {RunSettingsHelper.Instance.IsDefaultTargetArchitecture}, Current process architecture: {_processHelper.GetCurrentProcessArchitecture()} OperatingSystem: {_environment.OperatingSystem}."); - settingsUpdated |= UpdatePlatform( + // True when runsettings don't set platformk. False when runsettings force platform + // in both cases the sourceToArchitectureMap is populated with the real architecture as we inferred it + // from dlls. For sources like .js, we return the default architecture. + var platformWasAutodetected = UpdatePlatform( document, navigator, sources, @@ -728,15 +738,23 @@ private bool UpdateRunSettingsIfRequired( out Architecture chosenPlatform, out sourceToArchitectureMap); - if (FeatureFlag.Instance.IsSet(FeatureFlag.DISABLE_MULTI_TFM_RUN)) + settingsUpdated |= platformWasAutodetected; + var platformSetByRunsettings = !platformWasAutodetected; + + // Before MULTI_TFM feature the sourceToArchitectureMap and sourceToFrameworkMap were only used as informational + // to be able to do this compatibility check and print warning. And in the later steps only chosenPlatform, chosenFramework + // were used, that represented the single architecture and framework to be used. + // + // After MULTI_TFM sourceToArchitectureMap and sourceToFrameworkMap are the source of truth, and are propagated forward, + // so when we want to revert to the older behavior we need to re-enable the check, and unify all the architecture and + // framework entries to the same chosen value. + var disableMultiTfm = FeatureFlag.Instance.IsSet(FeatureFlag.DISABLE_MULTI_TFM_RUN); + + // Do the check only when we enable MULTI_TFM and platform or framework are forced by settings, because then we maybe have some sources + // that are not compatible with the chosen settings. And do the check always when MULTI_TFM is disabled, because then we want to warn every + // time there are multiple tfms or architectures mixed. + if (disableMultiTfm || (!disableMultiTfm && (platformSetByRunsettings || frameworkSetByRunsettings))) { - // Before MULTI_TFM feature the sourceToArchitectureMap and sourceToFrameworkMap were only used as informational - // to be able to do this compatibility check and print warning. And in the later steps only chosenPlatform, chosenFramework - // were used, that represented the single architecture and framework to be used. - // - // After MULTI_TFM sourceToArchitectureMap and sourceToFrameworkMap are the source of truth, and are propagated forward, - // so when we want to revert to the older behavior we need to re-enable the check, and unify all the architecture and - // framework entries to the same chosen value. CheckSourcesForCompatibility( chosenFramework, chosenPlatform, @@ -744,12 +762,20 @@ private bool UpdateRunSettingsIfRequired( sourceToArchitectureMap, sourceToFrameworkMap, registrar); + } + // The sourceToArchitectureMap contains the real architecture, overwrite it by the value chosen by runsettings, to force one unified platform to be used. + if (disableMultiTfm || platformSetByRunsettings) + { foreach (var key in sourceToArchitectureMap.Keys) { sourceToArchitectureMap[key] = chosenPlatform; } + } + // The sourceToFrameworkMap contains the real framework, overwrite it by the value chosen by runsettings, to force one unified framework to be used. + if (disableMultiTfm || frameworkSetByRunsettings) + { foreach (var key in sourceToFrameworkMap.Keys) { sourceToFrameworkMap[key] = chosenFramework; @@ -902,11 +928,20 @@ private bool UpdatePlatform( if (platformSetByRunsettings) { EqtTrace.Info($"Platform is set by runsettings to be '{commonPlatform}' for all sources."); - sourceToPlatformMap = new Dictionary(); - foreach (var source in sources) - { - sourceToPlatformMap.Add(source, commonPlatform); - } + // Autodetect platforms from sources, so we can check that they are compatible with the settings, and report + // incompatibilities as warnings. + // + // DO NOT overwrite the common platform, the one forced by runsettings should be used. + var _ = _inferHelper.AutoDetectArchitecture(sources, defaultArchitecture, out sourceToPlatformMap); + + // If we would not want to report the incompatibilities later, we would simply return dictionary populated to the + // platform that is set by the settings. + // + // sourceToPlatformMap = new Dictionary(); + // foreach (var source in sources) + // { + // sourceToPlatformMap.Add(source, commonPlatform); + // } // Return false, because we did not update runsettings. return false; @@ -933,11 +968,20 @@ private bool UpdateFrameworkInRunSettingsIfRequired( if (frameworkSetByRunsettings) { - sourceToFrameworkMap = new Dictionary(); - foreach (var source in sources) - { - sourceToFrameworkMap.Add(source, commonFramework); - } + // Autodetect frameworks from sources, so we can check that they are compatible with the settings, and report + // incompatibilities as warnings. + // + // DO NOT overwrite the common framework, the one forced by runsettings should be used. + var _ = _inferHelper.AutoDetectFramework(sources, out sourceToFrameworkMap); + + // If we would not want to report the incompatibilities later, we would simply return dictionary populated to the + // framework that is set by the settings. + // + // sourceToFrameworkMap = new Dictionary(); + // foreach (var source in sources) + // { + // sourceToFrameworkMap.Add(source, commonFramework); + // } WriteWarningForNetFramework35IsUnsupported(registrar, commonFramework); // Return false because we did not update runsettings. diff --git a/test/vstest.ProgrammerTests/Fakes/FakeTestRunEventsRegistrar.cs b/test/vstest.ProgrammerTests/Fakes/FakeTestRunEventsRegistrar.cs index 857966e62e..b6d7571b79 100644 --- a/test/vstest.ProgrammerTests/Fakes/FakeTestRunEventsRegistrar.cs +++ b/test/vstest.ProgrammerTests/Fakes/FakeTestRunEventsRegistrar.cs @@ -9,6 +9,7 @@ namespace vstest.ProgrammerTests.Fakes; internal class FakeTestRunEventsRegistrar : ITestRunEventsRegistrar { + public Guid Id { get; } = Guid.NewGuid(); public FakeTestRunEventsRegistrar(FakeErrorAggregator fakeErrorAggregator) { FakeErrorAggregator = fakeErrorAggregator; diff --git a/test/vstest.ProgrammerTests/Fakes/Fixture.cs b/test/vstest.ProgrammerTests/Fakes/Fixture.cs index ba1b3a4b28..d0890f2c00 100644 --- a/test/vstest.ProgrammerTests/Fakes/Fixture.cs +++ b/test/vstest.ProgrammerTests/Fakes/Fixture.cs @@ -43,6 +43,8 @@ internal class Fixture : IDisposable public List ExecutedTests => TestRunEventsRegistrar.RunChangedEvents.SelectMany(er => er.Data.NewTestResults).ToList(); public List DiscoveredTests => TestDiscoveryEventsRegistrar.DiscoveredTestsEvents.SelectMany(er => er.Data.DiscoveredTestCases).ToList(); + public List LoggedWarnings => TestRunEventsRegistrar.LoggedWarnings.Concat(TestDiscoveryEventsRegistrar.LoggedWarnings).ToList(); + public FakeTestSessionEventsHandler TestSessionEventsHandler { get; } public Fixture(FixtureOptions? fixtureOptions = null) diff --git a/test/vstest.ProgrammerTests/MultiTFMRunAndDiscovery.cs b/test/vstest.ProgrammerTests/MultiTFMRunAndDiscovery.cs index 5e916315e1..8905e5fa6d 100644 --- a/test/vstest.ProgrammerTests/MultiTFMRunAndDiscovery.cs +++ b/test/vstest.ProgrammerTests/MultiTFMRunAndDiscovery.cs @@ -98,7 +98,7 @@ public async Task A() // -- assert fixture.AssertNoErrors(); // We figure out the framework for each assembly so there should be no incompatibility warnings - fixture.TestRunEventsRegistrar.LoggedWarnings.Should().NotContainMatch("Test run detected DLL(s) which were built for different framework*"); + fixture.LoggedWarnings.Should().NotContainMatch("Test run detected DLL(s) which would use different framework*"); fixture.ProcessHelper.Processes.Where(p => p.Started).Should().HaveCount(2); var startWithSources1 = testhost1.FakeCommunicationChannel.ProcessedMessages.Single(m => m.Request.MessageType == MessageType.StartDiscovery); @@ -197,8 +197,8 @@ public async Task B() // -- assert fixture.AssertNoErrors(); - // We figure out the framework for each assembly so there should be no incompatibility warnings - fixture.TestRunEventsRegistrar.LoggedWarnings.Should().NotContainMatch("Test run detected DLL(s) which were built for different framework*"); + // Runsettings will force NET7, so we should get a warning. + fixture.LoggedWarnings.Should().ContainMatch("Test run detected DLL(s) which would use different framework*"); fixture.ProcessHelper.Processes.Where(p => p.Started).Should().HaveCount(2); var startWithSources1 = testhost1.FakeCommunicationChannel.ProcessedMessages.Single(m => m.Request.MessageType == MessageType.StartDiscovery); @@ -298,7 +298,7 @@ public async Task C() // -- assert fixture.AssertNoErrors(); // We figure out the framework for each assembly so there should be no incompatibility warnings - fixture.TestRunEventsRegistrar.LoggedWarnings.Should().NotContainMatch("Test run detected DLL(s) which were built for different framework*"); + fixture.LoggedWarnings.Should().NotContainMatch("Test run detected DLL(s) which would use different framework*"); fixture.ProcessHelper.Processes.Where(p => p.Started).Should().HaveCount(2); var startWithSources1 = testhost1.FakeCommunicationChannel.ProcessedMessages.Single(m => m.Request.MessageType == MessageType.StartTestExecutionWithSources); @@ -397,8 +397,8 @@ public async Task D() // -- assert fixture.AssertNoErrors(); - // We figure out the framework for each assembly so there should be no incompatibility warnings - fixture.TestRunEventsRegistrar.LoggedWarnings.Should().NotContainMatch("Test run detected DLL(s) which were built for different framework*"); + // We specify net7 which is not compatible with either, so we should get warnings + fixture.LoggedWarnings.Should().ContainMatch("Test run detected DLL(s) which would use different framework*"); fixture.ProcessHelper.Processes.Where(p => p.Started).Should().HaveCount(2); var startWithSources1 = testhost1.FakeCommunicationChannel.ProcessedMessages.Single(m => m.Request.MessageType == MessageType.StartTestExecutionWithSources); @@ -515,7 +515,7 @@ public async Task E() // -- assert fixture.AssertNoErrors(); // We figure out the framework for each assembly so there should be no incompatibility warnings - fixture.TestRunEventsRegistrar.LoggedWarnings.Should().NotContainMatch("Test run detected DLL(s) which were built for different framework*"); + fixture.LoggedWarnings.Should().NotContainMatch("Test run detected DLL(s) which would use different framework*"); fixture.ProcessHelper.Processes.Where(p => p.Started).Should().HaveCount(2); var startWithSources1 = testhost1.FakeCommunicationChannel.ProcessedMessages.Single(m => m.Request.MessageType == MessageType.StartTestExecutionWithSources); @@ -541,7 +541,7 @@ public class MultiTFMRunAndDiscoveryCompatibilityMode Given two test assemblies that have the same architecture but have different target frameworks. - When DISABLE_MULTI_TFM_RUN is enabled + When DISABLE_MULTI_TFM_RUN is set and we execute tests. Then two testhosts are both started for the same TFM. @@ -628,7 +628,7 @@ public async Task E() fixture.AssertNoErrors(); // We unify the frameworks to netcoreapp1.0 (because the vstest.console dll we are loading is built for netcoreapp and prefers netcoreapp), and because the // behavior is to choose the common oldest framework. We then log warning about incompatible sources. - fixture.TestRunEventsRegistrar.LoggedWarnings.Should().ContainMatch($"Test run detected DLL(s) which were built for different framework and platform versions*{KnownFrameworkNames.Netcoreapp1}*"); + fixture.LoggedWarnings.Should().ContainMatch($"Test run detected DLL(s) which would use different framework and platform versions*{KnownFrameworkNames.Netcoreapp1}*"); // We started both testhosts, even thought we know one of them is incompatible. fixture.ProcessHelper.Processes.Where(p => p.Started).Should().HaveCount(2); From 08d86c4394cb52c756bd12e08612c3d1b964d9dd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Jare=C5=A1?= Date: Fri, 13 May 2022 12:04:54 +0200 Subject: [PATCH 092/112] Smaller PR comments addressed --- .../Hosting/TestRunTimeProviderManager.cs | 2 +- .../IParallelOperationManager.cs | 18 +++++++++++ .../Engine/ClientProtocol/ITestEngine.cs | 6 ++-- .../FeatureFlag/FeatureFlag.cs | 1 - .../Parallel/ParallelOperationManager.cs | 19 ++++++++++-- .../Parallel/ParallelProxyDiscoveryManager.cs | 2 +- .../Parallel/ParallelProxyExecutionManager.cs | 30 ++++++++----------- .../TestEngine.cs | 10 +++---- .../InferRunSettingsHelper.cs | 2 +- .../TestPlatformTests.cs | 8 ++--- 10 files changed, 61 insertions(+), 37 deletions(-) create mode 100644 src/Microsoft.TestPlatform.Common/Interfaces/Engine/ClientProtocol/IParallelOperationManager.cs diff --git a/src/Microsoft.TestPlatform.Common/Hosting/TestRunTimeProviderManager.cs b/src/Microsoft.TestPlatform.Common/Hosting/TestRunTimeProviderManager.cs index 74d6457593..d0cbf55fec 100644 --- a/src/Microsoft.TestPlatform.Common/Hosting/TestRunTimeProviderManager.cs +++ b/src/Microsoft.TestPlatform.Common/Hosting/TestRunTimeProviderManager.cs @@ -2,11 +2,11 @@ // Licensed under the MIT license. See LICENSE file in the project root for full license information. using System; +using System.Collections.Generic; using Microsoft.VisualStudio.TestPlatform.Common.Logging; using Microsoft.VisualStudio.TestPlatform.ObjectModel.Host; using Microsoft.VisualStudio.TestPlatform.ObjectModel.Logging; -using System.Collections.Generic; #nullable disable diff --git a/src/Microsoft.TestPlatform.Common/Interfaces/Engine/ClientProtocol/IParallelOperationManager.cs b/src/Microsoft.TestPlatform.Common/Interfaces/Engine/ClientProtocol/IParallelOperationManager.cs new file mode 100644 index 0000000000..f0300abddd --- /dev/null +++ b/src/Microsoft.TestPlatform.Common/Interfaces/Engine/ClientProtocol/IParallelOperationManager.cs @@ -0,0 +1,18 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +#nullable disable + +namespace Microsoft.VisualStudio.TestPlatform.ObjectModel.Engine; + +/// +/// Interface defining the parallel operation manager +/// +public interface IParallelOperationManager +{ + /// + /// Update the parallelism level of the manager + /// + /// Parallelism level + void UpdateParallelLevel(int parallelLevel); +} diff --git a/src/Microsoft.TestPlatform.Common/Interfaces/Engine/ClientProtocol/ITestEngine.cs b/src/Microsoft.TestPlatform.Common/Interfaces/Engine/ClientProtocol/ITestEngine.cs index 1505e7c059..369049c605 100644 --- a/src/Microsoft.TestPlatform.Common/Interfaces/Engine/ClientProtocol/ITestEngine.cs +++ b/src/Microsoft.TestPlatform.Common/Interfaces/Engine/ClientProtocol/ITestEngine.cs @@ -29,7 +29,7 @@ public interface ITestEngine IProxyDiscoveryManager GetDiscoveryManager( IRequestData requestData, DiscoveryCriteria discoveryCriteria, - Dictionary sourceToSourceDetailMap); + IDictionary sourceToSourceDetailMap); /// /// Fetches the ExecutionManager for this engine. This manager would provide all @@ -46,7 +46,7 @@ IProxyDiscoveryManager GetDiscoveryManager( IProxyExecutionManager GetExecutionManager( IRequestData requestData, TestRunCriteria testRunCriteria, - Dictionary sourceToSourceDetailMap); + IDictionary sourceToSourceDetailMap); /// /// Fetches the TestSessionManager for this engine. This manager would provide all @@ -64,7 +64,7 @@ IProxyExecutionManager GetExecutionManager( IProxyTestSessionManager GetTestSessionManager( IRequestData requestData, StartTestSessionCriteria testSessionCriteria, - Dictionary sourceToSourceDetailMap); + IDictionary sourceToSourceDetailMap); /// /// Fetches the extension manager for this engine. This manager would provide extensibility diff --git a/src/Microsoft.TestPlatform.CoreUtilities/FeatureFlag/FeatureFlag.cs b/src/Microsoft.TestPlatform.CoreUtilities/FeatureFlag/FeatureFlag.cs index 0307607a40..14cfceb4a4 100644 --- a/src/Microsoft.TestPlatform.CoreUtilities/FeatureFlag/FeatureFlag.cs +++ b/src/Microsoft.TestPlatform.CoreUtilities/FeatureFlag/FeatureFlag.cs @@ -48,7 +48,6 @@ private FeatureFlag() { } // Faster JSON serialization relies on less internals of NewtonsoftJson, and on some additional caching. public const string DISABLE_FASTER_JSON_SERIALIZATION = VSTEST_ + nameof(DISABLE_FASTER_JSON_SERIALIZATION); - // TODO: Invert this to disable. public const string DISABLE_MULTI_TFM_RUN = VSTEST_ + nameof(DISABLE_MULTI_TFM_RUN); [Obsolete("Only use this in tests.")] diff --git a/src/Microsoft.TestPlatform.CrossPlatEngine/Client/Parallel/ParallelOperationManager.cs b/src/Microsoft.TestPlatform.CrossPlatEngine/Client/Parallel/ParallelOperationManager.cs index 45356ec443..4c5b445499 100644 --- a/src/Microsoft.TestPlatform.CrossPlatEngine/Client/Parallel/ParallelOperationManager.cs +++ b/src/Microsoft.TestPlatform.CrossPlatEngine/Client/Parallel/ParallelOperationManager.cs @@ -12,7 +12,7 @@ namespace Microsoft.VisualStudio.TestPlatform.CrossPlatEngine.Client; /// -/// Manages work that is done on multiple testhosts in parallel. +/// Manages work that is done on multiple managers (testhosts) in parallel such as parallel discovery or parallel run. /// internal sealed class ParallelOperationManager : IDisposable { @@ -30,6 +30,12 @@ internal sealed class ParallelOperationManager + /// Creates new instance of ParallelOperationManager. + /// + /// Creates a new manager that is responsible for running a single part of the overall workload. + /// A manager is typically a testhost, and the part of workload is discovering or running a single test dll. + /// Determines the maximum amount of parallel managers that can be active at the same time. public ParallelOperationManager(Func createNewManager, int parallelLevel) { _createNewManager = createNewManager; @@ -65,6 +71,8 @@ public void StartWork( } } + // This does not do anything in parallel, all the workloads we schedule are offloaded to separate Task in the callback. + // I did not want to change that, yet but this is the correct place to do that offloading. Not each manager. private bool RunWorkInParallel() { // TODO: Right now we don't re-use shared hosts, but if we did, this is the place @@ -95,7 +103,7 @@ private bool RunWorkInParallel() { var slot = availableSlots[i]; slot.IsAvailable = false; - var workload = availableWorkloads[i]; + var workload = workloadsToRun[i]; workToRun.Add(new SlotWorkloadPair(slot, workload)); _workloads.Remove(workload); } @@ -135,7 +143,12 @@ public bool RunNextWork(TManager completedManager!!) throw new InvalidOperationException("The provided manager was not found in any slot."); } - var slot = completedSlot.Single(); + if (completedSlot.Count > 1) + { + throw new InvalidOperationException("The provided manager was found in multiple slots."); + } + + var slot = completedSlot[0]; slot.IsAvailable = true; return RunWorkInParallel(); diff --git a/src/Microsoft.TestPlatform.CrossPlatEngine/Client/Parallel/ParallelProxyDiscoveryManager.cs b/src/Microsoft.TestPlatform.CrossPlatEngine/Client/Parallel/ParallelProxyDiscoveryManager.cs index 56dfa78202..d9837880a2 100644 --- a/src/Microsoft.TestPlatform.CrossPlatEngine/Client/Parallel/ParallelProxyDiscoveryManager.cs +++ b/src/Microsoft.TestPlatform.CrossPlatEngine/Client/Parallel/ParallelProxyDiscoveryManager.cs @@ -90,7 +90,7 @@ public void DiscoverTests(DiscoveryCriteria discoveryCriteria!!, ITestDiscoveryE // For example, let's assume we have 10 sources, a batch size of 10 but only 8 cores, we // will then spawn 8 instances of this and if we now cancel, we will have 2 sources not // marked as NotDiscovered. - _dataAggregator.MarkSourcesWithStatus(discoveryCriteria.Sources.ToList(), DiscoveryStatus.NotDiscovered); + _dataAggregator.MarkSourcesWithStatus(discoveryCriteria.Sources, DiscoveryStatus.NotDiscovered); _parallelOperationManager.StartWork(workloads, eventHandler, GetParallelEventHandler, DiscoverTestsOnConcurrentManager); } diff --git a/src/Microsoft.TestPlatform.CrossPlatEngine/Client/Parallel/ParallelProxyExecutionManager.cs b/src/Microsoft.TestPlatform.CrossPlatEngine/Client/Parallel/ParallelProxyExecutionManager.cs index a06ea03839..0ac7477188 100644 --- a/src/Microsoft.TestPlatform.CrossPlatEngine/Client/Parallel/ParallelProxyExecutionManager.cs +++ b/src/Microsoft.TestPlatform.CrossPlatEngine/Client/Parallel/ParallelProxyExecutionManager.cs @@ -178,7 +178,7 @@ public bool HandlePartialRunComplete( /// private List> SplitToWorkloads(TestRunCriteria testRunCriteria, Dictionary sourceToTestHostProviderMap) { - // We split the work to workloads that will run on each testhost, we and add all of them + // We split the work to workloads that will run on each testhost, and add all of them // to a bag of work that needs to be processed. (The workloads are just // a single source, or all test cases for a given source.) // @@ -190,28 +190,29 @@ private List> SplitToWorkloads(TestRun // and we start sending them work. Once any testhost is done processing a given workload, // we will get notified with the completed event for the work we are doing. For example for StartTestRun // we will get TestExecutionCompleted, and we will call HandlePartialTestExecutionComplete. - // (The "partial" here refers to posibly having more work in the work bag. It does not mean that + // (The "partial" here refers to possibly having more work in the work bag. It does not mean that // there was an error in the testhost and we only did part of execution.). // // At that point we know that at least one testhost is not busy doing work anymore. It either - // processed the workload and waits for another one, or it maybe crashed and we should move to + // processed the workload and waits for another one, or it crashed and we should move to // another source. // // In the "partial" step we check if we have more workloads, and if the currently running testhost // is shared we try to find a workload that is appropriate for it. If we don't find any work that the - // running testhost can do. Or if the testhost already exited (or maybe crashed), we start another one + // running testhost can do. Or if the testhost already exited (possibly because of crash), we start another one // and give it the next workload. List> workloads = new(); if (testRunCriteria.HasSpecificTests) { // We split test cases to their respective sources, and associate them with additional info about on // which type of provider they can run so we can later select the correct workload for the provider - // if we already have a provider running, and it is shared. - var testCasesPerSource = testRunCriteria.Tests.GroupBy(t => t.Source); + // if we already have a shared provider running, that can take more sources. + var testCasesPerSource = testRunCriteria.Tests.GroupBy(t => t.Source); foreach (var group in testCasesPerSource) { var testHostProviderInfo = sourceToTestHostProviderMap[group.Key]; var runsettings = testHostProviderInfo.RunSettings; + // ToList because it is easier to see what is going on when debugging. var testCases = group.ToList(); var updatedCriteria = CreateTestRunCriteriaFromTestCasesAndSettings(testCases, testRunCriteria, runsettings); var workload = new ProviderSpecificWorkload(updatedCriteria, testHostProviderInfo); @@ -340,25 +341,18 @@ private void StartTestRunOnConcurrentManager(IProxyExecutionManager proxyExecuti /// /// A workload with a specification of a provider that can run that workload. The workload is a list of sources, -/// or a list of testcases. Provider is a given testhost manager, that is capable of running this workload, so -/// we end up running .NET sources on .NET testhost, and .NET Framework sources on .NET Framework provider. +/// or a list of testcases. Provider is a testhost manager, that is capable of running this workload, so +/// we end up running .NET sources on .NET testhost, and .NET Framework sources on .NET Framework testhost. /// -internal class ProviderSpecificWorkload : ProviderSpecificWorkload +internal class ProviderSpecificWorkload { public T Work { get; } + public TestRuntimeProviderInfo Provider { get; protected set; } + public ProviderSpecificWorkload(T work, TestRuntimeProviderInfo provider) { Provider = provider; Work = work; } } - -/// -/// A "marker" for ProviderSpecificWorkload of T, so we can have a collection that either holds test cases, -/// or just sources and still know which provider they require. -/// -internal abstract class ProviderSpecificWorkload -{ - public TestRuntimeProviderInfo Provider { get; protected set; } -} diff --git a/src/Microsoft.TestPlatform.CrossPlatEngine/TestEngine.cs b/src/Microsoft.TestPlatform.CrossPlatEngine/TestEngine.cs index 50f35a2676..af028c76f3 100644 --- a/src/Microsoft.TestPlatform.CrossPlatEngine/TestEngine.cs +++ b/src/Microsoft.TestPlatform.CrossPlatEngine/TestEngine.cs @@ -5,6 +5,7 @@ using System.Collections.Generic; using System.Globalization; using System.Linq; +using System.Text; using Microsoft.VisualStudio.TestPlatform.Common; using Microsoft.VisualStudio.TestPlatform.Common.Hosting; @@ -24,7 +25,6 @@ using Microsoft.VisualStudio.TestPlatform.PlatformAbstractions; using Microsoft.VisualStudio.TestPlatform.PlatformAbstractions.Interfaces; using Microsoft.VisualStudio.TestPlatform.Utilities; -using System.Text; #nullable disable @@ -63,7 +63,7 @@ internal TestEngine( public IProxyDiscoveryManager GetDiscoveryManager( IRequestData requestData, DiscoveryCriteria discoveryCriteria, - Dictionary sourceToSourceDetailMap) + IDictionary sourceToSourceDetailMap) { // Parallel level determines how many processes at most we should start at the same time. We take the number from settings, and if user // has no preference or the preference is 0 then we use the number of logical processors. Or the number of sources, whatever is lower. @@ -170,7 +170,7 @@ public IProxyDiscoveryManager GetDiscoveryManager( public IProxyExecutionManager GetExecutionManager( IRequestData requestData, TestRunCriteria testRunCriteria, - Dictionary sourceToSourceDetailMap) + IDictionary sourceToSourceDetailMap) { // We use mulitple "different" runsettings here. We have runsettings that come with the testRunCriteria, // and we use that to figure out the common stuff before we try to setup the run. Later we patch the settings @@ -318,7 +318,7 @@ internal IProxyExecutionManager CreateNonParallelExecutionManager(IRequestData r public IProxyTestSessionManager GetTestSessionManager( IRequestData requestData, StartTestSessionCriteria testSessionCriteria, - Dictionary sourceToSourceDetailMap) + IDictionary sourceToSourceDetailMap) { var parallelLevel = VerifyParallelSettingAndCalculateParallelLevel( testSessionCriteria.Sources.Count, @@ -400,7 +400,7 @@ public IProxyTestSessionManager GetTestSessionManager( private List GetTestRuntimeProvidersForUniqueConfigurations( string runSettings, - Dictionary sourceToSourceDetailMap, + IDictionary sourceToSourceDetailMap, out ITestRuntimeProvider mostRecentlyCreatedInstance) { // Group source details to get unique frameworks and architectures for which we will run, so we can figure diff --git a/src/Microsoft.TestPlatform.Utilities/InferRunSettingsHelper.cs b/src/Microsoft.TestPlatform.Utilities/InferRunSettingsHelper.cs index 32f2cc4f67..b75811c1d1 100644 --- a/src/Microsoft.TestPlatform.Utilities/InferRunSettingsHelper.cs +++ b/src/Microsoft.TestPlatform.Utilities/InferRunSettingsHelper.cs @@ -497,7 +497,7 @@ private static void AddNodeIfNotPresent(XmlDocument xmlDocument, string nodeP { if (root.Name == RunSettingsNodeName) { - // When just is provided to as runsettings, then this will add the common inner node. + // When just is provided in the runsettings string, then this will add the common RunConfiguration inner node. XmlUtilities.AppendOrModifyChild(xmlDocument, RunConfigurationNodePath, RunConfigurationNodeName, innerXml: null); } else diff --git a/test/Microsoft.TestPlatform.Client.UnitTests/TestPlatformTests.cs b/test/Microsoft.TestPlatform.Client.UnitTests/TestPlatformTests.cs index 53ffbb4cc8..7de499871f 100644 --- a/test/Microsoft.TestPlatform.Client.UnitTests/TestPlatformTests.cs +++ b/test/Microsoft.TestPlatform.Client.UnitTests/TestPlatformTests.cs @@ -485,7 +485,7 @@ public void StartTestSessionShouldReturnFalseIfTestSessionManagerIsNull() It.IsAny(), It.IsAny(), It.IsAny>())) - .Returns((IProxyTestSessionManager)null); + .Returns((IProxyTestSessionManager)null!); var tp = new TestableTestPlatform(_testEngine.Object, _hostManager.Object); var mockEventsHandler = new Mock(); @@ -642,7 +642,7 @@ public TestableTestPlatform(ITestEngine testEngine, ITestRuntimeProvider hostPro { } - public TestableTestPlatform(ITestEngine testEngine, IFileHelper fileHelper, ITestRuntimeProvider? hostProvider) + public TestableTestPlatform(ITestEngine testEngine, IFileHelper fileHelper, ITestRuntimeProvider hostProvider) : base(testEngine, fileHelper, new TestableTestRuntimeProviderManager(hostProvider)) { } @@ -650,9 +650,9 @@ public TestableTestPlatform(ITestEngine testEngine, IFileHelper fileHelper, ITes private class TestableTestRuntimeProviderManager : TestRuntimeProviderManager { - private readonly ITestRuntimeProvider? _hostProvider; + private readonly ITestRuntimeProvider _hostProvider; - public TestableTestRuntimeProviderManager(ITestRuntimeProvider? hostProvider) + public TestableTestRuntimeProviderManager(ITestRuntimeProvider hostProvider) : base(TestSessionMessageLogger.Instance) { _hostProvider = hostProvider; From 3c22472c5a5dd2be2cbe1d694332e44199b855ef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Jare=C5=A1?= Date: Fri, 13 May 2022 14:08:09 +0200 Subject: [PATCH 093/112] Rewrite ParallelOperationManagerTests because that class fully changed --- .../Parallel/ParallelOperationManager.cs | 27 +- .../Parallel/ParallelOperationManagerTests.cs | 436 +++++++++++------- 2 files changed, 283 insertions(+), 180 deletions(-) diff --git a/src/Microsoft.TestPlatform.CrossPlatEngine/Client/Parallel/ParallelOperationManager.cs b/src/Microsoft.TestPlatform.CrossPlatEngine/Client/Parallel/ParallelOperationManager.cs index 4c5b445499..bd47704944 100644 --- a/src/Microsoft.TestPlatform.CrossPlatEngine/Client/Parallel/ParallelOperationManager.cs +++ b/src/Microsoft.TestPlatform.CrossPlatEngine/Client/Parallel/ParallelOperationManager.cs @@ -17,7 +17,6 @@ namespace Microsoft.VisualStudio.TestPlatform.CrossPlatEngine.Client; internal sealed class ParallelOperationManager : IDisposable { private readonly Func _createNewManager; - private readonly int _maxParallelLevel; /// /// Default number of Processes @@ -30,6 +29,10 @@ internal sealed class ParallelOperationManager /// Creates new instance of ParallelOperationManager. /// @@ -39,7 +42,7 @@ internal sealed class ParallelOperationManager createNewManager, int parallelLevel) { _createNewManager = createNewManager; - _maxParallelLevel = parallelLevel; + MaxParallelLevel = parallelLevel; ClearSlots(); } @@ -48,10 +51,17 @@ private void ClearSlots() lock (_lock) { _managerSlots.Clear(); - _managerSlots.AddRange(Enumerable.Range(0, _maxParallelLevel).Select(_ => new Slot())); + _managerSlots.AddRange(Enumerable.Range(0, MaxParallelLevel).Select(_ => new Slot())); + SetOccupiedSlotCount(); } } + private void SetOccupiedSlotCount() + { + AvailableSlotCount = _managerSlots.Count(s => s.IsAvailable); + OccupiedSlotCount = _managerSlots.Count - AvailableSlotCount; + } + public void StartWork( List> workloads!!, TEventHandler eventHandler!!, @@ -66,6 +76,9 @@ public void StartWork( lock (_lock) { + // This creates as many slots as possible even though we might not use them when we get less workloads to process, + // this is not a big issue, and not worth optimizing, because the parallel level is determined by the logical CPU count, + // so it is a small number. ClearSlots(); RunWorkInParallel(); } @@ -107,6 +120,8 @@ private bool RunWorkInParallel() workToRun.Add(new SlotWorkloadPair(slot, workload)); _workloads.Remove(workload); } + + SetOccupiedSlotCount(); } foreach (var pair in workToRun) @@ -151,6 +166,8 @@ public bool RunNextWork(TManager completedManager!!) var slot = completedSlot[0]; slot.IsAvailable = true; + SetOccupiedSlotCount(); + return RunWorkInParallel(); } } @@ -158,8 +175,8 @@ public bool RunNextWork(TManager completedManager!!) public void DoActionOnAllManagers(Action action, bool doActionsInParallel = false) { // We don't need to lock here, we just grab the current list of - // managers and run action on each one of them. - var managers = _managerSlots.Select(slot => slot.Manager).ToList(); + // slots that are occupied (have managers) and run action on each one of them. + var managers = _managerSlots.Where(slot => !slot.IsAvailable).Select(slot => slot.Manager).ToList(); int i = 0; var actionTasks = new Task[managers.Count]; foreach (var manager in managers) diff --git a/test/Microsoft.TestPlatform.CrossPlatEngine.UnitTests/Client/Parallel/ParallelOperationManagerTests.cs b/test/Microsoft.TestPlatform.CrossPlatEngine.UnitTests/Client/Parallel/ParallelOperationManagerTests.cs index ae305f4848..4239fa4083 100644 --- a/test/Microsoft.TestPlatform.CrossPlatEngine.UnitTests/Client/Parallel/ParallelOperationManagerTests.cs +++ b/test/Microsoft.TestPlatform.CrossPlatEngine.UnitTests/Client/Parallel/ParallelOperationManagerTests.cs @@ -1,180 +1,266 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT license. See LICENSE file in the project root for full license information. +//Copyright(c) Microsoft Corporation.All rights reserved. +//Licensed under the MIT license.See LICENSE file in the project root for full license information. -//using System; -//using System.Collections.Generic; -//using System.Linq; +using System; +using System.Collections.Generic; +using System.Linq; -//using Microsoft.VisualStudio.TestTools.UnitTesting; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using Microsoft.VisualStudio.TestPlatform.CrossPlatEngine.Client; +using Microsoft.VisualStudio.TestPlatform.CrossPlatEngine.Client.Parallel; +using FluentAssertions; namespace TestPlatform.CrossPlatEngine.UnitTests.Client; -//[TestClass] -//public class ParallelOperationManagerTests -//{ -// private MockParallelOperationManager _proxyParallelManager; - - //public ParallelOperationManagerTests() - //{ - // Func sampleCreator = - // () => new SampleConcurrentClass(); - -// _proxyParallelManager = new MockParallelOperationManager(sampleCreator, 2, true); -// } - -// [TestMethod] -// public void AbstractProxyParallelManagerShouldCreateCorrectNumberOfConcurrentObjects() -// { -// var createdSampleClasses = new List(); -// Func sampleCreator = -// () => -// { -// var sample = new SampleConcurrentClass(); -// createdSampleClasses.Add(sample); -// return sample; -// }; - -// _proxyParallelManager = new MockParallelOperationManager(sampleCreator, 3, true); - -// Assert.AreEqual(3, createdSampleClasses.Count, "Number of Concurrent Objects created should be 3"); -// } - -// [TestMethod] -// public void AbstractProxyParallelManagerShouldUpdateToCorrectNumberOfConcurrentObjects() -// { -// var createdSampleClasses = new List(); -// Func sampleCreator = -// () => -// { -// var sample = new SampleConcurrentClass(); -// createdSampleClasses.Add(sample); -// return sample; -// }; - -// _proxyParallelManager = new MockParallelOperationManager(sampleCreator, 1, true); - -// Assert.AreEqual(1, createdSampleClasses.Count, "Number of Concurrent Objects created should be 1"); - -// _proxyParallelManager.UpdateParallelLevel(4); - -// Assert.AreEqual(4, createdSampleClasses.Count, "Number of Concurrent Objects created should be 4"); -// } - -// [TestMethod] -// public void DoActionOnConcurrentObjectsShouldCallAllObjects() -// { -// var createdSampleClasses = new List(); -// Func sampleCreator = -// () => -// { -// var sample = new SampleConcurrentClass(); -// createdSampleClasses.Add(sample); -// return sample; -// }; - -// _proxyParallelManager = new MockParallelOperationManager(sampleCreator, 4, true); - -// Assert.AreEqual(4, createdSampleClasses.Count, "Number of Concurrent Objects created should be 4"); - -// int count = 0; -// _proxyParallelManager.DoActionOnAllConcurrentObjects( -// (sample) => -// { -// count++; -// Assert.IsTrue(createdSampleClasses.Contains(sample), "Called object must be in the created list."); -// // Make sure action is not called on same object multiple times -// createdSampleClasses.Remove(sample); -// }); - -// Assert.AreEqual(4, count, "Number of Concurrent Objects called should be 4"); - -// Assert.AreEqual(0, createdSampleClasses.Count, "All concurrent objects must be called."); -// } - -// [TestMethod] -// public void AddManagerShouldAddAManagerWithHandlerInConcurrentManagerList() -// { -// // At the beginning it should be equal to parallel level -// Assert.AreEqual(2, _proxyParallelManager.GetConcurrentManagersCount()); - -// _proxyParallelManager.AddManager(new SampleConcurrentClass(true), new SampleHandlerClass()); - -// Assert.AreEqual(3, _proxyParallelManager.GetConcurrentManagersCount()); -// Assert.AreEqual(1, _proxyParallelManager.GetConcurrentManagerInstances().Count(m => m.CheckValue)); -// } - -// [TestMethod] -// public void RemoveManagerShouldRemoveAManagerFromConcurrentManagerList() -// { -// var manager = new SampleConcurrentClass(true); -// _proxyParallelManager.AddManager(manager, new SampleHandlerClass()); - -// Assert.AreEqual(3, _proxyParallelManager.GetConcurrentManagersCount()); - -// _proxyParallelManager.RemoveManager(manager); - -// Assert.AreEqual(2, _proxyParallelManager.GetConcurrentManagersCount()); -// Assert.AreEqual(0, _proxyParallelManager.GetConcurrentManagerInstances().Count(m => m.CheckValue)); -// } - -// [TestMethod] -// public void UpdateHandlerForManagerShouldAddNewHandlerIfNotexist() -// { -// var manager = new SampleConcurrentClass(true); -// _proxyParallelManager.UpdateHandlerForManager(manager, new SampleHandlerClass()); - -// Assert.AreEqual(3, _proxyParallelManager.GetConcurrentManagersCount()); -// Assert.AreEqual(1, _proxyParallelManager.GetConcurrentManagerInstances().Count(m => m.CheckValue)); -// } - -// [TestMethod] -// public void UpdateHandlerForManagerShouldUpdateHandlerForGivenManager() -// { -// var manager = new SampleConcurrentClass(true); -// //_proxyParallelManager.AddManager(manager, new SampleHandlerClass()); - -// // For current handler the value of variable CheckValue should be false; -// Assert.IsFalse(_proxyParallelManager.GetHandlerForGivenManager(manager).CheckValue); - -// var newHandler = new SampleHandlerClass(true); - -// // Update manager with new handler -// _proxyParallelManager.UpdateHandlerForManager(manager, newHandler); - -// // It should not add new manager but update the current one -// Assert.AreEqual(3, _proxyParallelManager.GetConcurrentManagersCount()); -// Assert.IsTrue(_proxyParallelManager.GetHandlerForGivenManager(manager).CheckValue); -// } - -// private class MockParallelOperationManager : ParallelOperationManager -// { -// public MockParallelOperationManager(Func createNewClient, int parallelLevel, bool sharedHosts) : -// base(createNewClient, parallelLevel, sharedHosts) -// { -// } - -// public void DoActionOnAllConcurrentObjects(Action action) -// { -// DoActionOnAllManagers(action, false); -// } -// } - -// private class SampleConcurrentClass -// { -// public readonly bool CheckValue; -// public SampleConcurrentClass(bool value = false) -// { -// CheckValue = value; -// } -// } - -// private class SampleHandlerClass -// { -// public readonly bool CheckValue; -// public SampleHandlerClass(bool value = false) -// { -// CheckValue = value; -// } - -// } -//} +[TestClass] +public class ParallelOperationManagerTests +{ + [TestMethod] + public void OperationManagerShouldRunOnlyMaximumParallelLevelOfWorkInParallelEvenWhenThereAreMoreWorkloads() + { + // Arrange + Func createNewManager = _ => new SampleManager(); + var maxParallelLevel = 3; + var parallelOperationManager = new ParallelOperationManager(createNewManager, maxParallelLevel); + + // Create more workloads than our parallel level so we can observe that the maximum parallel level is reached but not more + var workloads = Enumerable.Range(1, maxParallelLevel + 2) + .Select(i => new ProviderSpecificWorkload(new SampleWorkload { Id = i }, provider: null)) + .ToList(); + var eventHandler = new SampleHandler(); + + List workerCounts = new(); + + Func getEventHandler = (handler, _) => handler; + Action runWorkload = (manager, _, _) => + { + // Every time we run a workload check how many slots are occupied, + // we should see 3 slots at max, because that is our max parallel level, we should NOT see 4 or more: + // This is what the data should be: + // - At the start we schedule as much work as we can, workloads 1, 2, 3 + // are started and grab a slot. + // We only update the slot count after scheduling all the work up to the max parallel level, + // so when we reach this method, all the slots are already occupied, so for workloads 1, 2, 3 we record 3, 3, 3. + // - Workload 1 finishes and leaves the slot, 4 starts and grabs a slot, 2, 3, 4 are now running we record 3. + // - workload 2 finishes and leaves the slot, 5 starts and grabs a slot, 3, 4, 5 are now running we record 3. + // - workload 2 finishes and leaves the slot, 5 starts and grabs a slot, 3, 4, 5 are now running we record 3. + // - workload 3 finishes and leaves the slot, there is no more work to do so we don't grab any additional slot. Just 4, 5 are now running we record 2. + // - workload 4 finishes and leaves the slot, there is no more work to do so we don't grab any additional slot. Just 5 is now running we record 1. + + workerCounts.Add(parallelOperationManager.OccupiedSlotCount); + + System.Threading.Thread.Sleep(100); + + // Tell the operation manager that we are done, and it should move to the next piece of work. + // Normally the operation manager would get this notification via the handler because the work we do + // is asynchronous, but here we know that we are already done so we just tell the operation manager directly + // and pass on the current manager that is done. + parallelOperationManager.RunNextWork(manager); + }; + + // Act + parallelOperationManager.StartWork(workloads, eventHandler, getEventHandler, runWorkload); + + // Assert + workerCounts.Should().BeEquivalentTo(3, 3, 3, 2, 1); + } + + [TestMethod] + public void OperationManagerShouldCreateOnlyAsManyParallelWorkersAsThereAreWorkloadsWhenTheAmountOfWorkloadsIsSmallerThanMaxParallelLevel() + { + // Arrange + Func createNewManager = _ => new SampleManager(); + var maxParallelLevel = 10; + var parallelOperationManager = new ParallelOperationManager(createNewManager, maxParallelLevel); + + // Create less workloads than our parallel level so we can observe that only as many slots are created as there are workloads. + var workloads = Enumerable.Range(1, 2) + .Select(i => new ProviderSpecificWorkload(new SampleWorkload { Id = i }, provider: null)) + .ToList(); + var eventHandler = new SampleHandler(); + + List workerCounts = new(); + + Func getEventHandler = (handler, _) => handler; + Action runWorkload = (manager, _, _) => + { + // See comments in test above for explanation. + workerCounts.Add(parallelOperationManager.OccupiedSlotCount); + System.Threading.Thread.Sleep(100); + + parallelOperationManager.RunNextWork(manager); + }; + + // Act + parallelOperationManager.StartWork(workloads, eventHandler, getEventHandler, runWorkload); + + // Assert + workerCounts.Should().BeEquivalentTo(2, 1); + } + + + [TestMethod] + public void OperationManagerShouldCreateAsManyMaxParallelLevel() + { + // Arrange + Func createNewManager = _ => new SampleManager(); + var maxParallelLevel = 10; + var parallelOperationManager = new ParallelOperationManager(createNewManager, maxParallelLevel); + + // Create less workloads than our parallel level so we can observe that only as many slots are created as there are workloads. + var workloads = Enumerable.Range(1, 2) + .Select(i => new ProviderSpecificWorkload(new SampleWorkload { Id = i }, provider: null)) + .ToList(); + var eventHandler = new SampleHandler(); + + List workerCounts = new(); + List availableWorkerCounts = new(); + + Func getEventHandler = (handler, _) => handler; + Action runWorkload = (manager, _, _) => + { + // See comments in test above for explanation. + workerCounts.Add(parallelOperationManager.OccupiedSlotCount); + availableWorkerCounts.Add(parallelOperationManager.AvailableSlotCount); + System.Threading.Thread.Sleep(100); + + parallelOperationManager.RunNextWork(manager); + }; + + // Act + parallelOperationManager.StartWork(workloads, eventHandler, getEventHandler, runWorkload); + + // Assert + workerCounts.Should().BeEquivalentTo(2, 1); + // We create 10 slots, because that is the max parallel level, when we observe, there are 2 workloads running, + // and then 1 workload running, so we see 8 and 9 (10 - 2, and 10 - 1). + availableWorkerCounts.Should().BeEquivalentTo(8, 9); + } + + [TestMethod] + public void OperationManagerMovesToTheNextWorkloadOnlyWhenRunNextWorkIsCalled() + { + // Arrange + Func createNewManager = _ => new SampleManager(); + var maxParallelLevel = 2; + var parallelOperationManager = new ParallelOperationManager(createNewManager, maxParallelLevel); + + // Create more workloads than our parallel level so we can observe that when one workload is finished, calling RunNextWork will move on + // to the next workload. + var workloads = Enumerable.Range(1, maxParallelLevel + 3) + .Select(i => new ProviderSpecificWorkload(new SampleWorkload { Id = i }, provider: null)) + .ToList(); + var eventHandler = new SampleHandler(); + + List workloadsProcessed = new(); + + Func getEventHandler = (handler, _) => handler; + Action runWorkload = (manager, _, workload) => + { + // See comments in test above for explanation. + System.Threading.Thread.Sleep(100); + + workloadsProcessed.Add(workload.Id); + // Only move to next when we run the first workload. Meaning we process 1, 2, and then 3, but not 4 and 5. + if (workload.Id == 1) + { + parallelOperationManager.RunNextWork(manager); + } + }; + + // Act + parallelOperationManager.StartWork(workloads, eventHandler, getEventHandler, runWorkload); + + // Assert + // We start by scheduling 2 workloads (1 and 2) becuase that is the max parallel level. + // Then we call next and go to 3. After that, we don't call next anymore which means we are done, + // even though we did not process workloads 4 and 5. + // (e.g. In real life Abort was called so the handler won't call RunNextWork, because we don't want to run the remaining sources, + // and all the sources that are currently running be aborted by calling Abort on each manager via DoActionOnAllManagers.) + workloadsProcessed.Should().BeEquivalentTo(1, 2, 3); + } + + [TestMethod] + public void OperationManagerRunsAnOperationOnAllActiveManagersWhenDoActionOnAllManagersIsCalled() + { + // Arrange + var createdManagers = new List(); + // Store the managers we created so we can inspect them later and see if Abort was called on them. + Func createNewManager = _ => + { + var manager = new SampleManager(); + createdManagers.Add(manager); + return manager; + }; + + var maxParallelLevel = 2; + // Create more workloads than the parallel level so we can go past max parallel level of active workers and simulate that we + // are aborting in the middle of a run. + var workloads = Enumerable.Range(1, maxParallelLevel + 3) + .Select(i => new ProviderSpecificWorkload(new SampleWorkload { Id = i }, provider: null)) + .ToList(); + + var parallelOperationManager = new ParallelOperationManager(createNewManager, maxParallelLevel); + var eventHandler = new SampleHandler(); + + Func getEventHandler = (handler, _) => handler; + Action runWorkload = (manager, _, workload) => + { + // See comments in test above for explanation. + + // Make workload 1 fast, we want to put this in state where 2 and 3 are running and we call abort on them. + if (workload.Id != 1) + { + System.Threading.Thread.Sleep(100); + } + + // Only move to next when we run the first workload. Meaning we process 1, 2, and then 3, but not 4 and 5. + if (workload.Id == 1) + { + parallelOperationManager.RunNextWork(manager); + } + }; + + // Start the work, so we process workload 1 and then move to 2. + parallelOperationManager.StartWork(workloads, eventHandler, getEventHandler, runWorkload); + + // Act + parallelOperationManager.DoActionOnAllManagers(manager => manager.Abort(), doActionsInParallel: true); + + // Assert + // When we aborted workload 1 was already processed, and 2, and 3 were active. + // We should see that the first manager did not call abort, but second and third called abort, + // and there were no more managers created because we stopped calling next after 1 was done. + createdManagers.Select(manager => manager.AbortCalled).Should().BeEquivalentTo(false, true, true); + } + + /// + /// Represents a manager that is responsible for processing a single given workload. Normally this would be a testhost. + /// + private class SampleManager + { + public bool AbortCalled { get; private set; } + + public void Abort() + { + AbortCalled = true; + } + } + + /// + /// Represents a handler, in our tests it does nothing, because we are not running any "async" work + /// so we don't need a handler to call us back when processing one workload is done and we can progress to next + /// workload. + /// + private class SampleHandler + { + + } + + // Represents a workload, normally this would be a test dll, or a collection of testcases from a single dll that + // are supposed to run on 1 testhost. + private class SampleWorkload + { + public int Id { get; set; } + } +} From d624583133f0fa82eb43a190b166d0d4015540c7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Jare=C5=A1?= Date: Fri, 13 May 2022 14:56:55 +0200 Subject: [PATCH 094/112] Fix build warnings --- playground/TestPlatform.Playground/Program.cs | 10 ++++++---- .../TestPlatform.cs | 2 +- .../PublicAPI/PublicAPI.Shipped.txt | 8 +++++++- .../Parallel/ParallelProxyExecutionManager.cs | 18 +++++++++--------- .../PublicAPI/PublicAPI.Shipped.txt | 11 +++++++++++ .../TestEngine.cs | 14 +++++++------- .../Client/Interfaces/ITestHostLauncher2.cs | 4 +++- .../PublicAPI/PublicAPI.Shipped.txt | 15 ++++++++++++++- .../PublicAPI/net/PublicAPI.Unshipped.txt | 2 +- .../CustomTestHostTests.cs | 3 +++ .../Parallel/ParallelOperationManagerTests.cs | 4 ++-- .../ParallelProxyExecutionManagerTests.cs | 16 +++++----------- .../TestEngineTests.cs | 6 +----- .../TestSession/TestSessionPoolTests.cs | 6 +++--- .../Fakes/ActionRecord.cs | 2 +- .../Fakes/FakeMessage.cs | 2 +- .../Fakes/FakeTestBatchBuilder.cs | 2 +- .../Fakes/FakeTestRuntimeProviderManager.cs | 2 +- .../Fakes/FakeTestSessionEventsHandler.cs | 2 +- test/vstest.ProgrammerTests/Fakes/Fixture.cs | 2 +- .../CommandLine/InferHelperTests.cs | 4 ++-- .../TestRequestManagerTests.cs | 16 ++++++++-------- 22 files changed, 89 insertions(+), 62 deletions(-) diff --git a/playground/TestPlatform.Playground/Program.cs b/playground/TestPlatform.Playground/Program.cs index 493b23200b..184a5f28fc 100644 --- a/playground/TestPlatform.Playground/Program.cs +++ b/playground/TestPlatform.Playground/Program.cs @@ -83,7 +83,9 @@ static void Main(string[] args) var options = new TestPlatformOptions(); var r = new VsTestConsoleWrapper(console, consoleOptions); var sessionHandler = new TestSessionHandler(); +#pragma warning disable CS0618 // Type or member is obsolete r.StartTestSession(sources, sourceSettings, sessionHandler); +#pragma warning restore CS0618 // Type or member is obsolete var discoveryHandler = new PlaygroundTestDiscoveryHandler(); r.DiscoverTests(sources, sourceSettings, options, sessionHandler.TestSessionInfo, discoveryHandler); r.RunTestsWithCustomTestHost(discoveryHandler.TestCases, sourceSettings, options, sessionHandler.TestSessionInfo, new TestRunHandler(), new DebuggerTestHostLauncher()); @@ -137,7 +139,7 @@ public void HandleRawMessage(string rawMessage) private static string WriteTests(IEnumerable testCases) => testCases?.Any() == true - ? "\t" + string.Join("\n\t", testCases.Select(r => r.Source + " " + r.DisplayName)) + ? "\t" + string.Join("\n\t", testCases.Select(r => r.Source + " " + r.DisplayName)) : "\t"; private static string WriteSources(IEnumerable sources) @@ -221,12 +223,12 @@ internal class TestSessionHandler : ITestSessionEventsHandler public void HandleLogMessage(TestMessageLevel level, string message) { - + } public void HandleRawMessage(string rawMessage) { - + } public void HandleStartTestSessionComplete(StartTestSessionCompleteEventArgs eventArgs) @@ -236,6 +238,6 @@ public void HandleStartTestSessionComplete(StartTestSessionCompleteEventArgs eve public void HandleStopTestSessionComplete(StopTestSessionCompleteEventArgs eventArgs) { - + } } diff --git a/src/Microsoft.TestPlatform.Client/TestPlatform.cs b/src/Microsoft.TestPlatform.Client/TestPlatform.cs index 9ae70c8ac5..cb5b37e842 100644 --- a/src/Microsoft.TestPlatform.Client/TestPlatform.cs +++ b/src/Microsoft.TestPlatform.Client/TestPlatform.cs @@ -93,7 +93,7 @@ public IDiscoveryRequest CreateDiscoveryRequest( ITestLoggerManager loggerManager = _testEngine.GetLoggerManager(requestData); loggerManager.Initialize(discoveryCriteria.RunSettings); - IProxyDiscoveryManager discoveryManager = _testEngine.GetDiscoveryManager(requestData, discoveryCriteria, sourceToSourceDetailMap); + IProxyDiscoveryManager discoveryManager = _testEngine.GetDiscoveryManager(requestData, discoveryCriteria, sourceToSourceDetailMap); discoveryManager.Initialize(options?.SkipDefaultAdapters ?? false); return new DiscoveryRequest(requestData, discoveryCriteria, discoveryManager, loggerManager); diff --git a/src/Microsoft.TestPlatform.Common/PublicAPI/PublicAPI.Shipped.txt b/src/Microsoft.TestPlatform.Common/PublicAPI/PublicAPI.Shipped.txt index 0d97b361d3..b9d09e94a1 100644 --- a/src/Microsoft.TestPlatform.Common/PublicAPI/PublicAPI.Shipped.txt +++ b/src/Microsoft.TestPlatform.Common/PublicAPI/PublicAPI.Shipped.txt @@ -246,7 +246,7 @@ static Microsoft.VisualStudio.TestPlatform.Common.Utilities.RunSettingsUtilities virtual Microsoft.VisualStudio.TestPlatform.Common.ExtensionFramework.TestPluginCache.GetFilteredExtensions(System.Collections.Generic.List extensions, string endsWithPattern) -> System.Collections.Generic.IEnumerable virtual Microsoft.VisualStudio.TestPlatform.Common.ExtensionFramework.Utilities.TestPluginInformation.IdentifierData.get -> string virtual Microsoft.VisualStudio.TestPlatform.Common.ExtensionFramework.Utilities.TestPluginInformation.Metadata.get -> System.Collections.Generic.ICollection -virtual Microsoft.VisualStudio.TestPlatform.Common.Hosting.TestRuntimeProviderManager.GetTestHostManagerByRunConfiguration(string runConfiguration) -> Microsoft.VisualStudio.TestPlatform.ObjectModel.Host.ITestRuntimeProvider +virtual Microsoft.VisualStudio.TestPlatform.Common.Hosting.TestRuntimeProviderManager.GetTestHostManagerByRunConfiguration(string runConfiguration, System.Collections.Generic.List _) -> Microsoft.VisualStudio.TestPlatform.ObjectModel.Host.ITestRuntimeProvider Microsoft.VisualStudio.TestPlatform.ObjectModel.Engine.IProxyTestSessionManager.StartSession(Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.ITestSessionEventsHandler eventsHandler, Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.IRequestData requestData) -> bool Microsoft.VisualStudio.TestPlatform.ObjectModel.Engine.IProxyTestSessionManager.StopSession(Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.IRequestData requestData) -> bool Microsoft.VisualStudio.TestPlatform.ObjectModel.Engine.IParallelProxyDiscoveryManager.IsAbortRequested.get -> bool @@ -256,3 +256,9 @@ Microsoft.VisualStudio.TestPlatform.ObjectModel.Engine.DiscoveryStatus Microsoft.VisualStudio.TestPlatform.ObjectModel.Engine.DiscoveryStatus.NotDiscovered = 0 -> Microsoft.VisualStudio.TestPlatform.ObjectModel.Engine.DiscoveryStatus Microsoft.VisualStudio.TestPlatform.ObjectModel.Engine.DiscoveryStatus.PartiallyDiscovered = 1 -> Microsoft.VisualStudio.TestPlatform.ObjectModel.Engine.DiscoveryStatus Microsoft.VisualStudio.TestPlatform.ObjectModel.Engine.DiscoveryStatus.FullyDiscovered = 2 -> Microsoft.VisualStudio.TestPlatform.ObjectModel.Engine.DiscoveryStatus +Microsoft.VisualStudio.TestPlatform.ObjectModel.Engine.IParallelOperationManager +Microsoft.VisualStudio.TestPlatform.ObjectModel.Engine.IParallelOperationManager.UpdateParallelLevel(int parallelLevel) -> void +Microsoft.VisualStudio.TestPlatform.ObjectModel.Engine.ITestEngine.GetDiscoveryManager(Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.IRequestData requestData, Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.DiscoveryCriteria discoveryCriteria, System.Collections.Generic.IDictionary sourceToSourceDetailMap) -> Microsoft.VisualStudio.TestPlatform.ObjectModel.Engine.IProxyDiscoveryManager +Microsoft.VisualStudio.TestPlatform.ObjectModel.Engine.ITestEngine.GetExecutionManager(Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.IRequestData requestData, Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.TestRunCriteria testRunCriteria, System.Collections.Generic.IDictionary sourceToSourceDetailMap) -> Microsoft.VisualStudio.TestPlatform.ObjectModel.Engine.IProxyExecutionManager +Microsoft.VisualStudio.TestPlatform.ObjectModel.Engine.ITestEngine.GetTestSessionManager(Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.IRequestData requestData, Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.StartTestSessionCriteria testSessionCriteria, System.Collections.Generic.IDictionary sourceToSourceDetailMap) -> Microsoft.VisualStudio.TestPlatform.ObjectModel.Engine.IProxyTestSessionManager + diff --git a/src/Microsoft.TestPlatform.CrossPlatEngine/Client/Parallel/ParallelProxyExecutionManager.cs b/src/Microsoft.TestPlatform.CrossPlatEngine/Client/Parallel/ParallelProxyExecutionManager.cs index 0ac7477188..b732a865fc 100644 --- a/src/Microsoft.TestPlatform.CrossPlatEngine/Client/Parallel/ParallelProxyExecutionManager.cs +++ b/src/Microsoft.TestPlatform.CrossPlatEngine/Client/Parallel/ParallelProxyExecutionManager.cs @@ -238,15 +238,15 @@ private List> SplitToWorkloads(TestRun TestRunCriteria CreateTestRunCriteriaFromTestCasesAndSettings(IEnumerable testCases, TestRunCriteria criteria, string runsettingsXml) { - return new TestRunCriteria( - testCases, - testRunCriteria.FrequencyOfRunStatsChangeEvent, - testRunCriteria.KeepAlive, - runsettingsXml, - testRunCriteria.RunStatsChangeEventTimeout, - testRunCriteria.TestHostLauncher, - testRunCriteria.TestSessionInfo, - testRunCriteria.DebugEnabledForTestSession); + return new TestRunCriteria( + testCases, + testRunCriteria.FrequencyOfRunStatsChangeEvent, + testRunCriteria.KeepAlive, + runsettingsXml, + testRunCriteria.RunStatsChangeEventTimeout, + testRunCriteria.TestHostLauncher, + testRunCriteria.TestSessionInfo, + testRunCriteria.DebugEnabledForTestSession); } TestRunCriteria CreateTestRunCriteriaFromSourceAndSettings(IEnumerable sources, TestRunCriteria criteria, string runsettingsXml) diff --git a/src/Microsoft.TestPlatform.CrossPlatEngine/PublicAPI/PublicAPI.Shipped.txt b/src/Microsoft.TestPlatform.CrossPlatEngine/PublicAPI/PublicAPI.Shipped.txt index 0930cf4876..cec5b51cbd 100644 --- a/src/Microsoft.TestPlatform.CrossPlatEngine/PublicAPI/PublicAPI.Shipped.txt +++ b/src/Microsoft.TestPlatform.CrossPlatEngine/PublicAPI/PublicAPI.Shipped.txt @@ -126,3 +126,14 @@ virtual Microsoft.VisualStudio.TestPlatform.CrossPlatEngine.ProxyTestSessionMana virtual Microsoft.VisualStudio.TestPlatform.CrossPlatEngine.ProxyTestSessionManager.StopSession(Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.IRequestData requestData) -> bool Microsoft.VisualStudio.TestPlatform.CrossPlatEngine.Client.ProxyDiscoveryManager.Abort(Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.ITestDiscoveryEventsHandler2 eventHandler) -> void Microsoft.VisualStudio.TestPlatform.CrossPlatEngine.Discovery.DiscoveryManager.Abort(Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.ITestDiscoveryEventsHandler2 eventHandler) -> void +Microsoft.VisualStudio.TestPlatform.CrossPlatEngine.Client.TestRuntimeProviderInfo +Microsoft.VisualStudio.TestPlatform.CrossPlatEngine.Client.TestRuntimeProviderInfo.RunSettings.get -> string +Microsoft.VisualStudio.TestPlatform.CrossPlatEngine.Client.TestRuntimeProviderInfo.Shared.get -> bool +Microsoft.VisualStudio.TestPlatform.CrossPlatEngine.Client.TestRuntimeProviderInfo.SourceDetails.get -> System.Collections.Generic.List +Microsoft.VisualStudio.TestPlatform.CrossPlatEngine.Client.TestRuntimeProviderInfo.TestRuntimeProviderInfo(System.Type type, bool shared, string runSettings, System.Collections.Generic.List sourceDetails) -> void +Microsoft.VisualStudio.TestPlatform.CrossPlatEngine.Client.TestRuntimeProviderInfo.Type.get -> System.Type +Microsoft.VisualStudio.TestPlatform.CrossPlatEngine.TestEngine.GetDiscoveryManager(Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.IRequestData requestData, Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.DiscoveryCriteria discoveryCriteria, System.Collections.Generic.IDictionary sourceToSourceDetailMap) -> Microsoft.VisualStudio.TestPlatform.ObjectModel.Engine.IProxyDiscoveryManager +Microsoft.VisualStudio.TestPlatform.CrossPlatEngine.TestEngine.GetExecutionManager(Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.IRequestData requestData, Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.TestRunCriteria testRunCriteria, System.Collections.Generic.IDictionary sourceToSourceDetailMap) -> Microsoft.VisualStudio.TestPlatform.ObjectModel.Engine.IProxyExecutionManager +Microsoft.VisualStudio.TestPlatform.CrossPlatEngine.TestEngine.GetTestSessionManager(Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.IRequestData requestData, Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.StartTestSessionCriteria testSessionCriteria, System.Collections.Generic.IDictionary sourceToSourceDetailMap) -> Microsoft.VisualStudio.TestPlatform.ObjectModel.Engine.IProxyTestSessionManager +Microsoft.VisualStudio.TestPlatform.CrossPlatEngine.ProxyTestSessionManager.ProxyTestSessionManager(Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.StartTestSessionCriteria criteria, int maxTesthostCount, System.Func proxyCreator, System.Collections.Generic.List runtimeProviders) -> void + diff --git a/src/Microsoft.TestPlatform.CrossPlatEngine/TestEngine.cs b/src/Microsoft.TestPlatform.CrossPlatEngine/TestEngine.cs index af028c76f3..3e2d4ef16b 100644 --- a/src/Microsoft.TestPlatform.CrossPlatEngine/TestEngine.cs +++ b/src/Microsoft.TestPlatform.CrossPlatEngine/TestEngine.cs @@ -271,13 +271,13 @@ internal IProxyExecutionManager CreateNonParallelExecutionManager(IRequestData r if (proxyOperationManager == null) { - // If the proxy creation process based on test session info failed, then - // we'll proceed with the normal creation process as if no test session - // info was passed in in the first place. - // - // WARNING: This should not normally happen and it raises questions - // regarding the test session pool operation and consistency. - EqtTrace.Warning("ProxyExecutionManager creation with test session failed."); + // If the proxy creation process based on test session info failed, then + // we'll proceed with the normal creation process as if no test session + // info was passed in in the first place. + // + // WARNING: This should not normally happen and it raises questions + // regarding the test session pool operation and consistency. + EqtTrace.Warning("ProxyExecutionManager creation with test session failed."); proxyOperationManager = new ProxyOperationManager( requestData, diff --git a/src/Microsoft.TestPlatform.ObjectModel/Client/Interfaces/ITestHostLauncher2.cs b/src/Microsoft.TestPlatform.ObjectModel/Client/Interfaces/ITestHostLauncher2.cs index a545bafc8d..91db204e1e 100644 --- a/src/Microsoft.TestPlatform.ObjectModel/Client/Interfaces/ITestHostLauncher2.cs +++ b/src/Microsoft.TestPlatform.ObjectModel/Client/Interfaces/ITestHostLauncher2.cs @@ -29,15 +29,17 @@ public interface ITestHostLauncher2 : ITestHostLauncher bool AttachDebuggerToProcess(int pid, CancellationToken cancellationToken); } +[Obsolete("Do not use this api, it is not ready yet.")] public interface ITestHostLauncher3 : ITestHostLauncher2 { bool AttachDebuggerToProcess(AttachDebuggerInfo attachDebuggerInfo); } +[Obsolete("Do not use this api, it is not ready yet.")] public class AttachDebuggerInfo { public Version Version { get; set; } public int ProcessId { get; set; } - public Framework? TargetFramework { get; set; } + public Framework TargetFramework { get; set; } public CancellationToken CancellationToken { get; set; } } diff --git a/src/Microsoft.TestPlatform.ObjectModel/PublicAPI/PublicAPI.Shipped.txt b/src/Microsoft.TestPlatform.ObjectModel/PublicAPI/PublicAPI.Shipped.txt index 4712fee9ea..691575a4da 100644 --- a/src/Microsoft.TestPlatform.ObjectModel/PublicAPI/PublicAPI.Shipped.txt +++ b/src/Microsoft.TestPlatform.ObjectModel/PublicAPI/PublicAPI.Shipped.txt @@ -893,7 +893,7 @@ Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.DiscoveryCriteria.TestSes Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.DiscoveryCriteria.TestSessionInfo.set -> void Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.DiscoveryRequestPayload.TestSessionInfo.get -> Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.TestSessionInfo Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.DiscoveryRequestPayload.TestSessionInfo.set -> void -Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.ITestPlatform.StartTestSession(Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.IRequestData requestData, Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.StartTestSessionCriteria criteria, Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.ITestSessionEventsHandler eventsHandler) -> bool +Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.ITestPlatform.StartTestSession(Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.IRequestData requestData, Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.StartTestSessionCriteria criteria, Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.ITestSessionEventsHandler eventsHandler, System.Collections.Generic.Dictionary sourceToSourceDetailMap) -> bool Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.ITestSessionEventsHandler.HandleStartTestSessionComplete(Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.StartTestSessionCompleteEventArgs eventArgs) -> void Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.ITestSessionEventsHandler.HandleStopTestSessionComplete(Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.StopTestSessionCompleteEventArgs eventArgs) -> void Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.Payloads.StartTestSessionAckPayload.EventArgs.get -> Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.StartTestSessionCompleteEventArgs @@ -933,3 +933,16 @@ Microsoft.VisualStudio.TestPlatform.ObjectModel.SourceDetail.Architecture.get -> Microsoft.VisualStudio.TestPlatform.ObjectModel.SourceDetail.Framework.get -> Microsoft.VisualStudio.TestPlatform.ObjectModel.Framework Microsoft.VisualStudio.TestPlatform.ObjectModel.SourceDetail.Source.get -> string Microsoft.VisualStudio.TestPlatform.ObjectModel.SourceDetail.SourceDetail() -> void +Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.Interfaces.AttachDebuggerInfo +Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.Interfaces.AttachDebuggerInfo.AttachDebuggerInfo() -> void +Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.Interfaces.AttachDebuggerInfo.CancellationToken.get -> System.Threading.CancellationToken +Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.Interfaces.AttachDebuggerInfo.CancellationToken.set -> void +Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.Interfaces.AttachDebuggerInfo.ProcessId.get -> int +Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.Interfaces.AttachDebuggerInfo.ProcessId.set -> void +Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.Interfaces.AttachDebuggerInfo.TargetFramework.get -> Microsoft.VisualStudio.TestPlatform.ObjectModel.Framework +Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.Interfaces.AttachDebuggerInfo.TargetFramework.set -> void +Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.Interfaces.AttachDebuggerInfo.Version.get -> System.Version +Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.Interfaces.AttachDebuggerInfo.Version.set -> void +Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.Interfaces.ITestHostLauncher3 +Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.Interfaces.ITestHostLauncher3.AttachDebuggerToProcess(Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.Interfaces.AttachDebuggerInfo attachDebuggerInfo) -> bool + diff --git a/src/Microsoft.TestPlatform.ObjectModel/PublicAPI/net/PublicAPI.Unshipped.txt b/src/Microsoft.TestPlatform.ObjectModel/PublicAPI/net/PublicAPI.Unshipped.txt index e02abfc9b0..5f282702bb 100644 --- a/src/Microsoft.TestPlatform.ObjectModel/PublicAPI/net/PublicAPI.Unshipped.txt +++ b/src/Microsoft.TestPlatform.ObjectModel/PublicAPI/net/PublicAPI.Unshipped.txt @@ -1 +1 @@ - + \ No newline at end of file diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/CustomTestHostTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/CustomTestHostTests.cs index 14a7b8386d..474b569a44 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/CustomTestHostTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/CustomTestHostTests.cs @@ -258,6 +258,7 @@ public bool AttachDebuggerToProcess(int pid, CancellationToken cancellationToken } } +#pragma warning disable CS0618 // Type or member is obsolete private class TestHostLauncherV3 : ITestHostLauncher3 { public bool IsDebug => true; @@ -304,3 +305,5 @@ public int LaunchTestHost(TestProcessStartInfo defaultTestHostStartInfo, Cancell } } } +#pragma warning restore CS0618 // Type or member is obsolete + diff --git a/test/Microsoft.TestPlatform.CrossPlatEngine.UnitTests/Client/Parallel/ParallelOperationManagerTests.cs b/test/Microsoft.TestPlatform.CrossPlatEngine.UnitTests/Client/Parallel/ParallelOperationManagerTests.cs index 4239fa4083..392be647ad 100644 --- a/test/Microsoft.TestPlatform.CrossPlatEngine.UnitTests/Client/Parallel/ParallelOperationManagerTests.cs +++ b/test/Microsoft.TestPlatform.CrossPlatEngine.UnitTests/Client/Parallel/ParallelOperationManagerTests.cs @@ -1,5 +1,5 @@ -//Copyright(c) Microsoft Corporation.All rights reserved. -//Licensed under the MIT license.See LICENSE file in the project root for full license information. +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. using System; using System.Collections.Generic; diff --git a/test/Microsoft.TestPlatform.CrossPlatEngine.UnitTests/Client/Parallel/ParallelProxyExecutionManagerTests.cs b/test/Microsoft.TestPlatform.CrossPlatEngine.UnitTests/Client/Parallel/ParallelProxyExecutionManagerTests.cs index cfe568b8f2..bb2eda7af9 100644 --- a/test/Microsoft.TestPlatform.CrossPlatEngine.UnitTests/Client/Parallel/ParallelProxyExecutionManagerTests.cs +++ b/test/Microsoft.TestPlatform.CrossPlatEngine.UnitTests/Client/Parallel/ParallelProxyExecutionManagerTests.cs @@ -3,8 +3,6 @@ using System; using System.Collections.Generic; -using System.Collections.ObjectModel; -using System.Linq; using System.Threading; using System.Threading.Tasks; @@ -33,11 +31,7 @@ public class ParallelProxyExecutionManagerTests private readonly List> _usedMockManagers; private readonly Func _createMockManager; private readonly Mock _mockEvenHandler; - private Mock _mockTestHostManager; - private Mock _mockRequestSender; - - private Mock _mockDataCollectionManager; private readonly List _sources; private readonly List _processedSources; private readonly TestRunCriteria _testRunCriteriaWith2Sources; @@ -181,9 +175,9 @@ public void StartTestRunWithSourcesShouldNotSendCompleteUntilAllSourcesAreProces public void HandlePartialRunCompleteShouldCreateNewProxyExecutionManagerIfDataCollectionEnabled() { var completeArgs = new TestRunCompleteEventArgs(null, isCanceled: false, isAborted: false, null, null, null, TimeSpan.Zero); - _mockTestHostManager = new Mock(); - _mockRequestSender = new Mock(); - _mockDataCollectionManager = new Mock(); + var mockTestHostManager = new Mock(); + var mockRequestSender = new Mock(); + var mockDataCollectionManager = new Mock(); var parallelExecutionManager = SetupExecutionManager(_createMockManager, 2, setupTestCases: true); // Trigger discover tests, this will create a manager by calling the _createMockManager func @@ -205,8 +199,8 @@ public void HandlePartialRunCompleteShouldCreateNewProxyExecutionManagerIfDataCo public void HandlePartialRunCompleteShouldCreateNewProxyExecutionManagerIfIsAbortedIsTrue() { var completeArgs = new TestRunCompleteEventArgs(null, true, true, null, null, null, TimeSpan.Zero); - _mockTestHostManager = new Mock(); - _mockRequestSender = new Mock(); + var mockTestHostManager = new Mock(); + var mockRequestSender = new Mock(); var parallelExecutionManager = SetupExecutionManager(_createMockManager, 2, setupTestCases: true); // Trigger discover tests, this will create a manager by calling the _createMockManager func diff --git a/test/Microsoft.TestPlatform.CrossPlatEngine.UnitTests/TestEngineTests.cs b/test/Microsoft.TestPlatform.CrossPlatEngine.UnitTests/TestEngineTests.cs index d2d04fdccc..dfb5fb76c5 100644 --- a/test/Microsoft.TestPlatform.CrossPlatEngine.UnitTests/TestEngineTests.cs +++ b/test/Microsoft.TestPlatform.CrossPlatEngine.UnitTests/TestEngineTests.cs @@ -6,7 +6,6 @@ using System.Linq; using System.Reflection; -using Microsoft.TestPlatform.CrossPlatEngine.UnitTests.TestableImplementations; using Microsoft.TestPlatform.TestUtilities; using Microsoft.VisualStudio.TestPlatform.Common.Telemetry; using Microsoft.VisualStudio.TestPlatform.CrossPlatEngine; @@ -14,7 +13,6 @@ using Microsoft.VisualStudio.TestPlatform.CrossPlatEngine.Client.Parallel; using Microsoft.VisualStudio.TestPlatform.ObjectModel; using Microsoft.VisualStudio.TestPlatform.ObjectModel.Client; -using Microsoft.VisualStudio.TestPlatform.ObjectModel.Engine; using Microsoft.VisualStudio.TestPlatform.ObjectModel.Host; using Microsoft.VisualStudio.TestPlatform.PlatformAbstractions.Interfaces; using Microsoft.VisualStudio.TestTools.UnitTesting; @@ -26,14 +24,12 @@ namespace TestPlatform.CrossPlatEngine.UnitTests; [TestClass] public class TestEngineTests { - private TestableTestEngine _testEngine; + private readonly TestableTestEngine _testEngine; private readonly Mock _mockProcessHelper; private readonly ProtocolConfig _protocolConfig = new() { Version = 1 }; private readonly Mock _mockRequestData; private readonly Mock _mockMetricsCollection; - private ITestRuntimeProvider _testableTestRuntimeProvider; - public TestEngineTests() { TestPluginCacheHelper.SetupMockExtensions(new[] { typeof(TestEngineTests).GetTypeInfo().Assembly.Location }, () => { }); diff --git a/test/Microsoft.TestPlatform.CrossPlatEngine.UnitTests/TestSession/TestSessionPoolTests.cs b/test/Microsoft.TestPlatform.CrossPlatEngine.UnitTests/TestSession/TestSessionPoolTests.cs index 255c4b337d..25abc5b8a9 100644 --- a/test/Microsoft.TestPlatform.CrossPlatEngine.UnitTests/TestSession/TestSessionPoolTests.cs +++ b/test/Microsoft.TestPlatform.CrossPlatEngine.UnitTests/TestSession/TestSessionPoolTests.cs @@ -42,7 +42,7 @@ public void KillSessionShouldSucceedIfTestSessionExists() var mockProxyTestSessionManager = new Mock( new StartTestSessionCriteria(), 1, - (Func)(_ => null), + (Func)(_ => null!), new List()); var mockRequestData = new Mock(); @@ -72,7 +72,7 @@ public void TakeProxyShouldSucceedIfMatchingCriteriaAreCorrect() var mockProxyTestSessionManager = new Mock( new StartTestSessionCriteria(), 1, - (Func)(_ => null), + (Func)(_ => null!), new List()); mockProxyTestSessionManager.SetupSequence(tsm => tsm.DequeueProxy(It.IsAny(), It.IsAny())) @@ -104,7 +104,7 @@ public void ReturnProxyShouldSucceedIfProxyIdIsValid() var mockProxyTestSessionManager = new Mock( new StartTestSessionCriteria(), 1, - (Func)(_ => null), + (Func)(_ => null!), new List()); mockProxyTestSessionManager.SetupSequence(tsm => tsm.EnqueueProxy(It.IsAny())) diff --git a/test/vstest.ProgrammerTests/Fakes/ActionRecord.cs b/test/vstest.ProgrammerTests/Fakes/ActionRecord.cs index fca98e3d89..dede5a031a 100644 --- a/test/vstest.ProgrammerTests/Fakes/ActionRecord.cs +++ b/test/vstest.ProgrammerTests/Fakes/ActionRecord.cs @@ -11,7 +11,7 @@ internal class ActionRecord { public T Value { get; } public string StackTrace { get; } - public ActionRecord (T value) + public ActionRecord(T value) { StackTrace = Environment.StackTrace; Value = value; diff --git a/test/vstest.ProgrammerTests/Fakes/FakeMessage.cs b/test/vstest.ProgrammerTests/Fakes/FakeMessage.cs index 7753d058cc..03a0d9c077 100644 --- a/test/vstest.ProgrammerTests/Fakes/FakeMessage.cs +++ b/test/vstest.ProgrammerTests/Fakes/FakeMessage.cs @@ -53,6 +53,6 @@ public FakeMessage(string messageType, T payload, int version = 0) public override string ToString() { - return $"{MessageType} {{{ Payload }}}"; + return $"{MessageType} {{{Payload}}}"; } } diff --git a/test/vstest.ProgrammerTests/Fakes/FakeTestBatchBuilder.cs b/test/vstest.ProgrammerTests/Fakes/FakeTestBatchBuilder.cs index 54c9e9f3a9..03b1a781d6 100644 --- a/test/vstest.ProgrammerTests/Fakes/FakeTestBatchBuilder.cs +++ b/test/vstest.ProgrammerTests/Fakes/FakeTestBatchBuilder.cs @@ -11,7 +11,7 @@ internal class FakeTestBatchBuilder public TimeSpan Duration { get; private set; } public int BatchSize { get; private set; } public static List> Empty => new(); - public string Source { get; private set; } + public string? Source { get; private set; } public FakeTestBatchBuilder() { diff --git a/test/vstest.ProgrammerTests/Fakes/FakeTestRuntimeProviderManager.cs b/test/vstest.ProgrammerTests/Fakes/FakeTestRuntimeProviderManager.cs index 5a8267a0b4..e19ff8ebba 100644 --- a/test/vstest.ProgrammerTests/Fakes/FakeTestRuntimeProviderManager.cs +++ b/test/vstest.ProgrammerTests/Fakes/FakeTestRuntimeProviderManager.cs @@ -28,7 +28,7 @@ public void AddTestRuntimeProviders(params FakeTestRuntimeProvider[] runtimeProv // so we need to go by order. We also do this resolve twice for each source in parallel run // because we first need to know if the provider is shared. So we add to the queue twice. // This is brittle, but there is no way around this :( - foreach(var provider in runtimeProviders) + foreach (var provider in runtimeProviders) { TestRuntimeProvidersByOrder.Enqueue(provider); TestRuntimeProvidersByOrder.Enqueue(provider); diff --git a/test/vstest.ProgrammerTests/Fakes/FakeTestSessionEventsHandler.cs b/test/vstest.ProgrammerTests/Fakes/FakeTestSessionEventsHandler.cs index 2597cb68ae..a1791e77a4 100644 --- a/test/vstest.ProgrammerTests/Fakes/FakeTestSessionEventsHandler.cs +++ b/test/vstest.ProgrammerTests/Fakes/FakeTestSessionEventsHandler.cs @@ -10,7 +10,7 @@ internal class FakeTestSessionEventsHandler : ITestSessionEventsHandler { private readonly FakeErrorAggregator _fakeErrorAggregator; - public FakeTestSessionEventsHandler (FakeErrorAggregator fakeErrorAggregator) + public FakeTestSessionEventsHandler(FakeErrorAggregator fakeErrorAggregator) { _fakeErrorAggregator = fakeErrorAggregator; } diff --git a/test/vstest.ProgrammerTests/Fakes/Fixture.cs b/test/vstest.ProgrammerTests/Fakes/Fixture.cs index d0890f2c00..60a38bfc35 100644 --- a/test/vstest.ProgrammerTests/Fakes/Fixture.cs +++ b/test/vstest.ProgrammerTests/Fakes/Fixture.cs @@ -28,7 +28,7 @@ internal class Fixture : IDisposable public FakeTestRuntimeProviderManager TestRuntimeProviderManager { get; } public FakeTestRunEventsRegistrar TestRunEventsRegistrar { get; } public FakeEnvironment Environment { get; } - public FakeTestDiscoveryEventsRegistrar TestDiscoveryEventsRegistrar { get; } + public FakeTestDiscoveryEventsRegistrar TestDiscoveryEventsRegistrar { get; } public TestEngine? TestEngine { get; private set; } public TestPlatform? TestPlatform { get; private set; } public TestRunResultAggregator? TestRunResultAggregator { get; private set; } diff --git a/test/vstest.console.UnitTests/CommandLine/InferHelperTests.cs b/test/vstest.console.UnitTests/CommandLine/InferHelperTests.cs index 0148f1cbed..e834673082 100644 --- a/test/vstest.console.UnitTests/CommandLine/InferHelperTests.cs +++ b/test/vstest.console.UnitTests/CommandLine/InferHelperTests.cs @@ -47,7 +47,7 @@ public void AutoDetectArchitectureShouldReturnDefaultArchitectureOnEmptySources( [TestMethod] public void AutoDetectArchitectureShouldReturnDefaultArchitectureOnNullItemInSources() { - Assert.AreEqual(_defaultArchitecture, _inferHelper.AutoDetectArchitecture(new List() { null }, _defaultArchitecture, out _)); + Assert.AreEqual(_defaultArchitecture, _inferHelper.AutoDetectArchitecture(new List() { null! }, _defaultArchitecture, out _)); } [TestMethod] @@ -162,7 +162,7 @@ public void AutoDetectFrameworkShouldReturnDefaultFrameworkOnEmptySources() [TestMethod] public void AutoDetectFrameworkShouldReturnDefaultFrameworkOnNullItemInSources() { - Assert.AreEqual(_defaultFramework, _inferHelper.AutoDetectFramework(new List() { null }, out _)); + Assert.AreEqual(_defaultFramework, _inferHelper.AutoDetectFramework(new List() { null! }, out _)); } [TestMethod] diff --git a/test/vstest.console.UnitTests/TestPlatformHelpers/TestRequestManagerTests.cs b/test/vstest.console.UnitTests/TestPlatformHelpers/TestRequestManagerTests.cs index fead7edcba..226b580b87 100644 --- a/test/vstest.console.UnitTests/TestPlatformHelpers/TestRequestManagerTests.cs +++ b/test/vstest.console.UnitTests/TestPlatformHelpers/TestRequestManagerTests.cs @@ -785,8 +785,8 @@ public void DiscoverTestsShouldNotInferAndUpdateFrameworkAndPlatformInCommandLin _mockAssemblyMetadataProvider.Verify(a => a.GetFrameWork(It.IsAny()), Times.Never); // we don't update the settings - Assert.IsFalse(actualDiscoveryCriteria.RunSettings.Contains(Constants.DotNetFramework46)); - Assert.IsFalse(actualDiscoveryCriteria.RunSettings.Contains(nameof(Architecture.ARM))); + Assert.IsFalse(actualDiscoveryCriteria!.RunSettings.Contains(Constants.DotNetFramework46)); + Assert.IsFalse(actualDiscoveryCriteria!.RunSettings.Contains(nameof(Architecture.ARM))); } [TestMethod] @@ -1448,8 +1448,8 @@ public void RunTestsShouldNotUpdateFrameworkAndPlatformIfSpecifiedInDesignMode() _mockAssemblyMetadataProvider.Verify(a => a.GetFrameWork(It.IsAny()), Times.Never); // don't update them - Assert.IsTrue(actualTestRunCriteria.TestRunSettings.Contains(Constants.DotNetFramework46)); - Assert.IsTrue(actualTestRunCriteria.TestRunSettings.Contains(nameof(Architecture.ARM))); + Assert.IsTrue(actualTestRunCriteria!.TestRunSettings.Contains(Constants.DotNetFramework46)); + Assert.IsTrue(actualTestRunCriteria!.TestRunSettings.Contains(nameof(Architecture.ARM))); } [TestMethod] @@ -1490,7 +1490,7 @@ public void RunTestsShouldNotUpdatePlatformIfSpecifiedInDesignMode(string target _mockAssemblyMetadataProvider.Verify(a => a.GetFrameWork(It.IsAny()), Times.Once); // don't update it - Assert.IsTrue(actualTestRunCriteria.TestRunSettings.Contains(targetPlatform)); + Assert.IsTrue(actualTestRunCriteria!.TestRunSettings.Contains(targetPlatform)); } [TestMethod] @@ -1562,7 +1562,7 @@ public void RunTestsShouldNotpdateFrameworkAndPlatformInRunsettingsIfSpecifiedBy _mockAssemblyMetadataProvider.Verify(a => a.GetFrameWork(It.IsAny()), Times.Never); // don't update them - Assert.IsFalse(actualTestRunCriteria.TestRunSettings.Contains(Constants.DotNetFramework46)); + Assert.IsFalse(actualTestRunCriteria!.TestRunSettings.Contains(Constants.DotNetFramework46)); Assert.IsFalse(actualTestRunCriteria.TestRunSettings.Contains(nameof(Architecture.ARM))); } @@ -1588,9 +1588,9 @@ public void RunTestsWithTestCasesShouldUpdateFrameworkAndPlatformIfNotSpecifiedI List archSources = new(), fxSources = new(); _commandLineOptions.IsDesignMode = true; - _mockAssemblyMetadataProvider.Setup(a => a.GetArchitecture(It.IsAny())).Callback(source => archSources.Add(source)) + _mockAssemblyMetadataProvider.Setup(a => a.GetArchitecture(It.IsAny())).Callback(archSources.Add) .Returns(Architecture.ARM); - _mockAssemblyMetadataProvider.Setup(a => a.GetFrameWork(It.IsAny())).Callback(source => fxSources.Add(source)) + _mockAssemblyMetadataProvider.Setup(a => a.GetFrameWork(It.IsAny())).Callback(fxSources.Add) .Returns(new FrameworkName(Constants.DotNetFramework46)); TestRunCriteria? actualTestRunCriteria = null; var mockTestRunRequest = new Mock(); From da6150c0b135b48794e78e00e5b54632497b01b9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Jare=C5=A1?= Date: Fri, 13 May 2022 15:30:12 +0200 Subject: [PATCH 095/112] Remove test checking that we set custom launcher on default runtime provider, because we don't have any default runtime provider anymore, and we don't figure it out until we run some tests --- .../TestPlatformTests.cs | 21 ------------------- 1 file changed, 21 deletions(-) diff --git a/test/Microsoft.TestPlatform.Client.UnitTests/TestPlatformTests.cs b/test/Microsoft.TestPlatform.Client.UnitTests/TestPlatformTests.cs index 7de499871f..d5b2ec13d6 100644 --- a/test/Microsoft.TestPlatform.Client.UnitTests/TestPlatformTests.cs +++ b/test/Microsoft.TestPlatform.Client.UnitTests/TestPlatformTests.cs @@ -279,27 +279,6 @@ public void CreateTestRunRequestShouldInitializeManagersWithFalseFlagWhenTestPla _executionManager.Verify(dm => dm.Initialize(false), Times.Once); } - [TestMethod] - public void CreateTestRunRequestShouldSetCustomHostLauncherOnEngineDefaultLauncherIfSpecified() - { - var mockCustomLauncher = new Mock(); - _executionManager.Setup(dm => dm.Initialize(false)).Verifiable(); - _testEngine.Setup(te => te.GetExecutionManager(_mockRequestData.Object, It.IsAny(), It.IsAny>())).Returns(_executionManager.Object); - _testEngine.Setup(te => te.GetExtensionManager()).Returns(_extensionManager.Object); - _testEngine.Setup(te => te.GetLoggerManager(_mockRequestData.Object)).Returns(_loggerManager.Object); - - var tp = new TestableTestPlatform(_testEngine.Object, _hostManager.Object); - var testRunCriteria = new TestRunCriteria(new List { "foo" }, 10, false, null, TimeSpan.Zero, mockCustomLauncher.Object); - _hostManager.Setup(hm => hm.GetTestSources(testRunCriteria.Sources)) - .Returns(testRunCriteria.Sources); - - var testRunRequest = tp.CreateTestRunRequest(_mockRequestData.Object, testRunCriteria, new TestPlatformOptions(), new Dictionary()); - - var actualTestRunRequest = testRunRequest as TestRunRequest; - Assert.AreEqual(testRunCriteria, actualTestRunRequest?.TestRunCriteria); - _hostManager.Verify(hl => hl.SetCustomLauncher(mockCustomLauncher.Object), Times.Once); - } - [TestMethod] public void CreateTestRunRequestThrowsIfTestRunCriteriaIsNull() { From f997480117b4687e363dfc33aaa2f4d1d6d841d7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Jare=C5=A1?= Date: Fri, 13 May 2022 17:09:55 +0200 Subject: [PATCH 096/112] Fix NOMERGE unit tests --- .../Parallel/ParallelOperationManager.cs | 66 ++++-- .../ParallelProxyDiscoveryManagerTests.cs | 29 ++- .../ParallelProxyExecutionManagerTests.cs | 219 +++++++++--------- 3 files changed, 168 insertions(+), 146 deletions(-) diff --git a/src/Microsoft.TestPlatform.CrossPlatEngine/Client/Parallel/ParallelOperationManager.cs b/src/Microsoft.TestPlatform.CrossPlatEngine/Client/Parallel/ParallelOperationManager.cs index bd47704944..4a63719368 100644 --- a/src/Microsoft.TestPlatform.CrossPlatEngine/Client/Parallel/ParallelOperationManager.cs +++ b/src/Microsoft.TestPlatform.CrossPlatEngine/Client/Parallel/ParallelOperationManager.cs @@ -24,6 +24,7 @@ internal sealed class ParallelOperationManager? _getEventHandler; private Action? _runWorkload; + private bool _acceptMoreWork; private readonly List> _workloads = new(); private readonly List _managerSlots = new(); @@ -43,13 +44,14 @@ public ParallelOperationManager(Func createNe { _createNewManager = createNewManager; MaxParallelLevel = parallelLevel; - ClearSlots(); + ClearSlots(acceptMoreWork: true); } - private void ClearSlots() + private void ClearSlots(bool acceptMoreWork) { lock (_lock) { + _acceptMoreWork = acceptMoreWork; _managerSlots.Clear(); _managerSlots.AddRange(Enumerable.Range(0, MaxParallelLevel).Select(_ => new Slot())); SetOccupiedSlotCount(); @@ -74,14 +76,11 @@ public void StartWork( _workloads.AddRange(workloads); - lock (_lock) - { - // This creates as many slots as possible even though we might not use them when we get less workloads to process, - // this is not a big issue, and not worth optimizing, because the parallel level is determined by the logical CPU count, - // so it is a small number. - ClearSlots(); - RunWorkInParallel(); - } + // This creates as many slots as possible even though we might not use them when we get less workloads to process, + // this is not a big issue, and not worth optimizing, because the parallel level is determined by the logical CPU count, + // so it is a small number. + ClearSlots(acceptMoreWork: true); + RunWorkInParallel(); } // This does not do anything in parallel, all the workloads we schedule are offloaded to separate Task in the callback. @@ -101,11 +100,20 @@ private bool RunWorkInParallel() if (_runWorkload == null) throw new InvalidOperationException($"{nameof(_runWorkload)} was not provided."); + // Reserve slots and assign them work under the lock so we keep + // the slots consistent. List workToRun = new(); lock (_lock) { if (_workloads.Count == 0) - return true; + return false; + + // When HandlePartialDiscovery or HandlePartialRun are in progress, and we call StopAllManagers, + // it is possible that we will clear all slots, and have RunWorkInParallel waiting on the lock, + // so when it is allowed to enter it will try to add more work, but we already cancelled, + // so we should not start more work. + if (!_acceptMoreWork) + return false; var availableSlots = _managerSlots.Where(slot => slot.IsAvailable).ToList(); var availableWorkloads = _workloads.Where(workload => workload != null).ToList(); @@ -122,11 +130,8 @@ private bool RunWorkInParallel() } SetOccupiedSlotCount(); - } - foreach (var pair in workToRun) - { - try + foreach (var pair in workToRun) { var manager = _createNewManager(pair.Workload.Provider); var eventHandler = _getEventHandler(_eventHandler, manager); @@ -134,8 +139,17 @@ private bool RunWorkInParallel() pair.Slot.Manager = manager; pair.Slot.ManagerInfo = pair.Workload.Provider; pair.Slot.Work = pair.Workload.Work; + } + } - _runWorkload(manager, eventHandler, pair.Workload.Work); + // Kick of the work in parallel outside of the lock so if we have more requests to run + // that come in at the same time we only block them from reserving the same slot at the same time + // but not from starting their assigned work at the same time. + foreach (var pair in workToRun) + { + try + { + _runWorkload(pair.Slot.Manager!, pair.Slot.EventHandler!, pair.Workload.Work); } finally { @@ -144,16 +158,26 @@ private bool RunWorkInParallel() } // Return true when we started more work. Or false, when there was nothing more to do. - // This will propagate to handling of partial discovery / run. + // This will propagate to handling of partial discovery or partial run. return workToRun.Count > 0; } public bool RunNextWork(TManager completedManager!!) + { + ClearCompletedSlot(completedManager); + return RunWorkInParallel(); + } + + private void ClearCompletedSlot(TManager completedManager) { lock (_lock) { var completedSlot = _managerSlots.Where(s => ReferenceEquals(completedManager, s.Manager)).ToList(); - if (completedSlot.Count == 0) + // When HandlePartialDiscovery or HandlePartialRun are in progress, and we call StopAllManagers, + // it is possible that we will clear all slots, while ClearCompletedSlot is waiting on the lock, + // so when it is allowed to enter it will fail to find the respective slot and fail. In this case it is + // okay that the slot is not found because we stopped all work and cleared the slots. + if (completedSlot.Count == 0 && _acceptMoreWork) { throw new InvalidOperationException("The provided manager was not found in any slot."); } @@ -167,8 +191,6 @@ public bool RunNextWork(TManager completedManager!!) slot.IsAvailable = true; SetOccupiedSlotCount(); - - return RunWorkInParallel(); } } @@ -219,12 +241,12 @@ private static void DoManagerAction(Action action) internal void StopAllManagers() { - ClearSlots(); + ClearSlots(acceptMoreWork: false); } public void Dispose() { - ClearSlots(); + ClearSlots(acceptMoreWork: false); } private class Slot diff --git a/test/Microsoft.TestPlatform.CrossPlatEngine.UnitTests/Client/Parallel/ParallelProxyDiscoveryManagerTests.cs b/test/Microsoft.TestPlatform.CrossPlatEngine.UnitTests/Client/Parallel/ParallelProxyDiscoveryManagerTests.cs index cd4cee4c96..e508746831 100644 --- a/test/Microsoft.TestPlatform.CrossPlatEngine.UnitTests/Client/Parallel/ParallelProxyDiscoveryManagerTests.cs +++ b/test/Microsoft.TestPlatform.CrossPlatEngine.UnitTests/Client/Parallel/ParallelProxyDiscoveryManagerTests.cs @@ -286,21 +286,20 @@ public void HandlePartialDiscoveryCompleteShouldCreateANewProxyDiscoveryManagerI Assert.AreEqual(2, _createMockManagerCalled); } - // NOMERGE: fix this test, it is a new test that uses the old approach - //[TestMethod] - //public void DiscoveryTestsWithCompletionMarksAllSourcesAsFullyDiscovered() - //{ - // _testDiscoveryCriteria.TestCaseFilter = "Name~Test"; - // var parallelDiscoveryManager = SetupDiscoveryManager(_proxyManagerFunc, 2, false); - - // Task.Run(() => parallelDiscoveryManager.DiscoverTests(_testDiscoveryCriteria, _mockHandler.Object)); - - // Assert.IsTrue(_discoveryCompleted.Wait(TaskTimeout), "Test discovery not completed."); - // Assert.AreEqual(_sources.Count, _processedSources.Count, "All Sources must be processed."); - // CollectionAssert.AreEquivalent(_sources, _dataAggregator.GetSourcesWithStatus(DiscoveryStatus.FullyDiscovered)); - // Assert.AreEqual(0, _dataAggregator.GetSourcesWithStatus(DiscoveryStatus.PartiallyDiscovered).Count); - // Assert.AreEqual(0, _dataAggregator.GetSourcesWithStatus(DiscoveryStatus.NotDiscovered).Count); - //} + [TestMethod] + public void DiscoveryTestsWithCompletionMarksAllSourcesAsFullyDiscovered() + { + _discoveryCriteriaWith2Sources.TestCaseFilter = "Name~Test"; + var parallelDiscoveryManager = SetupDiscoveryManager(_createMockManager, 2, false); + + Task.Run(() => parallelDiscoveryManager.DiscoverTests(_discoveryCriteriaWith2Sources, _mockEventHandler.Object)); + + Assert.IsTrue(_discoveryCompleted.Wait(Timeout3Seconds), "Test discovery not completed."); + Assert.AreEqual(_sources.Count, _processedSources.Count, "All Sources must be processed."); + CollectionAssert.AreEquivalent(_sources, _dataAggregator.GetSourcesWithStatus(DiscoveryStatus.FullyDiscovered)); + Assert.AreEqual(0, _dataAggregator.GetSourcesWithStatus(DiscoveryStatus.PartiallyDiscovered).Count); + Assert.AreEqual(0, _dataAggregator.GetSourcesWithStatus(DiscoveryStatus.NotDiscovered).Count); + } private IParallelProxyDiscoveryManager SetupDiscoveryManager(Func getProxyManager, int parallelLevel, bool abortDiscovery, int totalTests = 20) { diff --git a/test/Microsoft.TestPlatform.CrossPlatEngine.UnitTests/Client/Parallel/ParallelProxyExecutionManagerTests.cs b/test/Microsoft.TestPlatform.CrossPlatEngine.UnitTests/Client/Parallel/ParallelProxyExecutionManagerTests.cs index bb2eda7af9..c59e836769 100644 --- a/test/Microsoft.TestPlatform.CrossPlatEngine.UnitTests/Client/Parallel/ParallelProxyExecutionManagerTests.cs +++ b/test/Microsoft.TestPlatform.CrossPlatEngine.UnitTests/Client/Parallel/ParallelProxyExecutionManagerTests.cs @@ -3,6 +3,8 @@ using System; using System.Collections.Generic; +using System.Collections.ObjectModel; +using System.Linq; using System.Threading; using System.Threading.Tasks; @@ -30,7 +32,7 @@ public class ParallelProxyExecutionManagerTests private readonly List> _usedMockManagers; private readonly Func _createMockManager; - private readonly Mock _mockEvenHandler; + private readonly Mock _mockEventHandler; private readonly List _sources; private readonly List _processedSources; @@ -64,7 +66,7 @@ public ParallelProxyExecutionManagerTests() _usedMockManagers.Add(manager); return manager.Object; }; - _mockEvenHandler = new Mock(); + _mockEventHandler = new Mock(); // Configure sources _sources = new List() { "1.dll", "2.dll" }; @@ -138,7 +140,7 @@ public void StartTestRunShouldProcessAllSources() { var parallelExecutionManager = SetupExecutionManager(_createMockManager, 2); - parallelExecutionManager.StartTestRun(_testRunCriteriaWith2Sources, _mockEvenHandler.Object); + parallelExecutionManager.StartTestRun(_testRunCriteriaWith2Sources, _mockEventHandler.Object); Assert.IsTrue(_executionCompleted.Wait(Timeout3Seconds), "Test run not completed."); Assert.AreEqual(_sources.Count, _processedSources.Count, "All Sources must be processed."); @@ -152,7 +154,7 @@ public void StartTestRunShouldProcessAllTestCases() { var parallelExecutionManager = SetupExecutionManager(_createMockManager, 3, setupTestCases: true); - parallelExecutionManager.StartTestRun(_testRunCriteriaWithTestsFrom3Dlls, _mockEvenHandler.Object); + parallelExecutionManager.StartTestRun(_testRunCriteriaWithTestsFrom3Dlls, _mockEventHandler.Object); Assert.IsTrue(_executionCompleted.Wait(Timeout3Seconds), "Test run not completed."); Assert.AreEqual(_testCases.Count, _processedTestCases.Count, "All Tests must be processed."); @@ -164,7 +166,7 @@ public void StartTestRunWithSourcesShouldNotSendCompleteUntilAllSourcesAreProces { var parallelExecutionManager = SetupExecutionManager(_createMockManager, 2); - Task.Run(() => parallelExecutionManager.StartTestRun(_testRunCriteriaWith2Sources, _mockEvenHandler.Object)); + Task.Run(() => parallelExecutionManager.StartTestRun(_testRunCriteriaWith2Sources, _mockEventHandler.Object)); Assert.IsTrue(_executionCompleted.Wait(Timeout3Seconds), "Test run not completed."); Assert.AreEqual(_sources.Count, _processedSources.Count, "All Sources must be processed."); @@ -182,7 +184,7 @@ public void HandlePartialRunCompleteShouldCreateNewProxyExecutionManagerIfDataCo // Trigger discover tests, this will create a manager by calling the _createMockManager func // which dequeues it to _usedMockManagers. - parallelExecutionManager.StartTestRun(_testRunCriteriaWith2Sources, _mockEvenHandler.Object); + parallelExecutionManager.StartTestRun(_testRunCriteriaWith2Sources, _mockEventHandler.Object); var completedManager = _usedMockManagers[0]; // act @@ -205,7 +207,7 @@ public void HandlePartialRunCompleteShouldCreateNewProxyExecutionManagerIfIsAbor // Trigger discover tests, this will create a manager by calling the _createMockManager func // which dequeues it to _usedMockManagers. - parallelExecutionManager.StartTestRun(_testRunCriteriaWith2Sources, _mockEvenHandler.Object); + parallelExecutionManager.StartTestRun(_testRunCriteriaWith2Sources, _mockEventHandler.Object); var completedManager = _usedMockManagers[0]; // act @@ -223,7 +225,7 @@ public void StartTestRunWithTestsShouldNotSendCompleteUntilAllTestsAreProcessed( { var parallelExecutionManager = SetupExecutionManager(_createMockManager, 3, setupTestCases: true); - var task = Task.Run(() => parallelExecutionManager.StartTestRun(_testRunCriteriaWithTestsFrom3Dlls, _mockEvenHandler.Object)); + var task = Task.Run(() => parallelExecutionManager.StartTestRun(_testRunCriteriaWithTestsFrom3Dlls, _mockEventHandler.Object)); bool executionCompleted = _executionCompleted.Wait(Timeout3Seconds); @@ -248,7 +250,7 @@ public void StartTestRunShouldNotProcessAllSourcesOnExecutionCancelsForAnySource SetupMockManagers(_processedSources, isCanceled: true, isAborted: false); SetupHandleTestRunComplete(_executionCompleted); - Task.Run(() => parallelExecutionManager.StartTestRun(_testRunCriteriaWith2Sources, _mockEvenHandler.Object)); + Task.Run(() => parallelExecutionManager.StartTestRun(_testRunCriteriaWith2Sources, _mockEventHandler.Object)); Assert.IsTrue(_executionCompleted.Wait(Timeout3Seconds), "Test run not completed."); Assert.AreEqual(1, _processedSources.Count, "Abort should stop all sources execution."); @@ -264,7 +266,7 @@ public void StartTestRunShouldNotProcessAllSourcesOnExecutionAborted() SetupHandleTestRunComplete(_executionCompleted); parallelExecutionManager.Abort(It.IsAny()); - Task.Run(() => parallelExecutionManager.StartTestRun(_testRunCriteriaWith2Sources, _mockEvenHandler.Object)); + Task.Run(() => parallelExecutionManager.StartTestRun(_testRunCriteriaWith2Sources, _mockEventHandler.Object)); Assert.IsTrue(_executionCompleted.Wait(Timeout3Seconds), "Test run not completed."); Assert.AreEqual(1, _processedSources.Count, "Abort should stop all sources execution."); @@ -279,7 +281,7 @@ public void StartTestRunShouldProcessAllSourcesOnExecutionAbortsForAnySource() SetupMockManagers(_processedSources, isCanceled: false, isAborted: true); SetupHandleTestRunComplete(_executionCompleted); - Task.Run(() => parallelExecutionManager.StartTestRun(_testRunCriteriaWith2Sources, _mockEvenHandler.Object)); + Task.Run(() => parallelExecutionManager.StartTestRun(_testRunCriteriaWith2Sources, _mockEventHandler.Object)); Assert.IsTrue(_executionCompleted.Wait(Timeout3Seconds), "Test run not completed."); Assert.AreEqual(2, _processedSources.Count, "Abort should stop all sources execution."); @@ -295,7 +297,7 @@ public void StartTestRunShouldProcessAllSourceIfOneDiscoveryManagerIsStarved() mockManagers[1].Setup(em => em.StartTestRun(It.IsAny(), It.IsAny())) .Throws(); - Task.Run(() => parallelExecutionManager.StartTestRun(_testRunCriteriaWith2Sources, _mockEvenHandler.Object)); + Task.Run(() => parallelExecutionManager.StartTestRun(_testRunCriteriaWith2Sources, _mockEventHandler.Object)); // Processed sources should be 1 since the 2nd source is never discovered Assert.IsTrue(_executionCompleted.Wait(Timeout3Seconds), "Test run not completed."); @@ -311,10 +313,10 @@ public void StartTestRunShouldCatchExceptionAndHandleLogMessageOfError() mockManagers[1].Setup(em => em.StartTestRun(It.IsAny(), It.IsAny())) .Throws(); - Task.Run(() => parallelExecutionManager.StartTestRun(_testRunCriteriaWith2Sources, _mockEvenHandler.Object)); + Task.Run(() => parallelExecutionManager.StartTestRun(_testRunCriteriaWith2Sources, _mockEventHandler.Object)); Assert.IsTrue(_executionCompleted.Wait(Timeout3Seconds), "Test run not completed."); - _mockEvenHandler.Verify(s => s.HandleLogMessage(TestMessageLevel.Error, It.IsAny()), Times.Once); + _mockEventHandler.Verify(s => s.HandleLogMessage(TestMessageLevel.Error, It.IsAny()), Times.Once); } [TestMethod] @@ -326,104 +328,103 @@ public void StartTestRunShouldCatchExceptionAndHandleRawMessageOfTestMessage() mockManagers[1].Setup(em => em.StartTestRun(It.IsAny(), It.IsAny())) .Throws(); - Task.Run(() => parallelExecutionManager.StartTestRun(_testRunCriteriaWith2Sources, _mockEvenHandler.Object)); + Task.Run(() => parallelExecutionManager.StartTestRun(_testRunCriteriaWith2Sources, _mockEventHandler.Object)); Assert.IsTrue(_executionCompleted.Wait(Timeout3Seconds), "Test run not completed."); - _mockEvenHandler.Verify(s => s.HandleRawMessage(It.Is(str => str.Contains(MessageType.TestMessage)))); + _mockEventHandler.Verify(s => s.HandleRawMessage(It.Is(str => str.Contains(MessageType.TestMessage)))); } - // NOMERGE: fix this test, it is a new test that uses the old approach - //[TestMethod] - //public void StartTestRunShouldAggregateRunData() - //{ - // var parallelExecutionManager = new ParallelProxyExecutionManager(_mockRequestData.Object, _createMockManager, 2, _runtimeProviders); - // var syncObject = new object(); - - // foreach (var manager in _preCreatedMockManagers) - // { - // manager.Setup(m => m.StartTestRun(It.IsAny(), It.IsAny())). - // Callback( - // (criteria, handler) => - // { - // lock (syncObject) - // { - // _processedSources.AddRange(criteria.Sources); - // } - - // Task.Delay(100).Wait(); - // var stats = new Dictionary - // { - // { TestOutcome.Passed, 3 }, - // { TestOutcome.Failed, 2 } - // }; - // var runAttachments = new Collection - // { - // new AttachmentSet(new Uri("hello://x/"), "Hello") - // }; - // var executorUris = new List() { "hello1" }; - // bool isCanceled = false; - // bool isAborted = false; - // TimeSpan timespan = TimeSpan.FromMilliseconds(100); - - // if (string.Equals(criteria.Sources?.FirstOrDefault(), "2.dll")) - // { - // isCanceled = true; - // isAborted = true; - // timespan = TimeSpan.FromMilliseconds(200); - // } - - // var completeArgs = new TestRunCompleteEventArgs(new - // TestRunStatistics(5, stats), isCanceled, isAborted, null, runAttachments, new Collection(), timespan); - // handler.HandleTestRunComplete(completeArgs, null, runAttachments, executorUris); - // }); - // } - - // Exception? assertException = null; - // _mockHandler.Setup(m => m.HandleTestRunComplete( - // It.IsAny(), - // It.IsAny(), - // It.IsAny>(), - // It.IsAny>())).Callback - // , ICollection>( - // (completeArgs, runChangedArgs, runAttachments, executorUris) => - // { - // try - // { - // Assert.AreEqual(TimeSpan.FromMilliseconds(200), completeArgs.ElapsedTimeInRunningTests, - // "Time should be max of all"); - // Assert.AreEqual(2, completeArgs.AttachmentSets.Count, - // "All Complete Arg attachments should return"); - // Assert.AreEqual(2, runAttachments.Count, "All RunContextAttachments should return"); - - // Assert.IsTrue(completeArgs.IsAborted, "Aborted value must be OR of all values"); - // Assert.IsTrue(completeArgs.IsCanceled, "Canceled value must be OR of all values"); - - // Assert.AreEqual(10, completeArgs.TestRunStatistics.ExecutedTests, - // "Stats must be aggregated properly"); - - // Assert.AreEqual(6, completeArgs.TestRunStatistics.Stats[TestOutcome.Passed], - // "Stats must be aggregated properly"); - // Assert.AreEqual(4, completeArgs.TestRunStatistics.Stats[TestOutcome.Failed], - // "Stats must be aggregated properly"); - // } - // catch (Exception ex) - // { - // assertException = ex; - // } - // finally - // { - // _executionCompleted.Set(); - // } - // }); - - // Task.Run(() => parallelExecutionManager.StartTestRun(_testRunCriteriaWith2Sources, _mockEvenHandler.Object)); - - // Assert.IsTrue(_executionCompleted.Wait(Timeout3Seconds), "Test run not completed."); - - // Assert.IsNull(assertException, assertException?.ToString()); - // Assert.AreEqual(_sources.Count, _processedSources.Count, "All Sources must be processed."); - // AssertMissingAndDuplicateSources(_processedSources); - //} + [TestMethod] + public void StartTestRunShouldAggregateRunData() + { + var parallelExecutionManager = new ParallelProxyExecutionManager(_mockRequestData.Object, _createMockManager, 2, _runtimeProviders); + var syncObject = new object(); + + foreach (var manager in _preCreatedMockManagers) + { + manager.Setup(m => m.StartTestRun(It.IsAny(), It.IsAny())). + Callback( + (criteria, handler) => + { + lock (syncObject) + { + _processedSources.AddRange(criteria.Sources); + } + + Task.Delay(100).Wait(); + var stats = new Dictionary + { + { TestOutcome.Passed, 3 }, + { TestOutcome.Failed, 2 } + }; + var runAttachments = new Collection + { + new AttachmentSet(new Uri("hello://x/"), "Hello") + }; + var executorUris = new List() { "hello1" }; + bool isCanceled = false; + bool isAborted = false; + TimeSpan timespan = TimeSpan.FromMilliseconds(100); + + if (string.Equals(criteria.Sources?.FirstOrDefault(), "2.dll")) + { + isCanceled = true; + isAborted = true; + timespan = TimeSpan.FromMilliseconds(200); + } + + var completeArgs = new TestRunCompleteEventArgs(new + TestRunStatistics(5, stats), isCanceled, isAborted, null, runAttachments, new Collection(), timespan); + handler.HandleTestRunComplete(completeArgs, null, runAttachments, executorUris); + }); + } + + Exception? assertException = null; + _mockEventHandler.Setup(m => m.HandleTestRunComplete( + It.IsAny(), + It.IsAny(), + It.IsAny>(), + It.IsAny>())).Callback + , ICollection>( + (completeArgs, runChangedArgs, runAttachments, executorUris) => + { + try + { + Assert.AreEqual(TimeSpan.FromMilliseconds(200), completeArgs.ElapsedTimeInRunningTests, + "Time should be max of all"); + Assert.AreEqual(2, completeArgs.AttachmentSets.Count, + "All Complete Arg attachments should return"); + Assert.AreEqual(2, runAttachments.Count, "All RunContextAttachments should return"); + + Assert.IsTrue(completeArgs.IsAborted, "Aborted value must be OR of all values"); + Assert.IsTrue(completeArgs.IsCanceled, "Canceled value must be OR of all values"); + + Assert.AreEqual(10, completeArgs.TestRunStatistics.ExecutedTests, + "Stats must be aggregated properly"); + + Assert.AreEqual(6, completeArgs.TestRunStatistics.Stats[TestOutcome.Passed], + "Stats must be aggregated properly"); + Assert.AreEqual(4, completeArgs.TestRunStatistics.Stats[TestOutcome.Failed], + "Stats must be aggregated properly"); + } + catch (Exception ex) + { + assertException = ex; + } + finally + { + _executionCompleted.Set(); + } + }); + + Task.Run(() => parallelExecutionManager.StartTestRun(_testRunCriteriaWith2Sources, _mockEventHandler.Object)); + + Assert.IsTrue(_executionCompleted.Wait(Timeout3Seconds), "Test run not completed."); + + Assert.IsNull(assertException, assertException?.ToString()); + Assert.AreEqual(_sources.Count, _processedSources.Count, "All Sources must be processed."); + AssertMissingAndDuplicateSources(_processedSources); + } private ParallelProxyExecutionManager SetupExecutionManager(Func proxyManagerFunc, int parallelLevel) { @@ -449,7 +450,7 @@ private ParallelProxyExecutionManager SetupExecutionManager(Func mh.HandleTestRunComplete(It.IsAny(), + _mockEventHandler.Setup(mh => mh.HandleTestRunComplete(It.IsAny(), It.IsAny(), It.IsAny>(), It.IsAny>())) From 8734b3a4ca8421e59e980464979148aaced88fb9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Jare=C5=A1?= Date: Fri, 13 May 2022 17:15:51 +0200 Subject: [PATCH 097/112] Fix runsettings tests --- .../TestRequestManagerTests.cs | 26 +++++++++---------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/test/vstest.console.UnitTests/TestPlatformHelpers/TestRequestManagerTests.cs b/test/vstest.console.UnitTests/TestPlatformHelpers/TestRequestManagerTests.cs index 226b580b87..9e48c0b237 100644 --- a/test/vstest.console.UnitTests/TestPlatformHelpers/TestRequestManagerTests.cs +++ b/test/vstest.console.UnitTests/TestPlatformHelpers/TestRequestManagerTests.cs @@ -780,11 +780,11 @@ public void DiscoverTestsShouldNotInferAndUpdateFrameworkAndPlatformInCommandLin _testRequestManager.DiscoverTests(payload, new Mock().Object, _protocolConfig); - // we don't infer the settings - _mockAssemblyMetadataProvider.Verify(a => a.GetArchitecture(It.IsAny()), Times.Never); - _mockAssemblyMetadataProvider.Verify(a => a.GetFrameWork(It.IsAny()), Times.Never); + // we infer the architecture and framework, so we can print warning when they don't match settings. + _mockAssemblyMetadataProvider.Verify(a => a.GetArchitecture(It.IsAny()), Times.Once); + _mockAssemblyMetadataProvider.Verify(a => a.GetFrameWork(It.IsAny()), Times.Once); - // we don't update the settings + // but we don't update the settings, to keep what user specified Assert.IsFalse(actualDiscoveryCriteria!.RunSettings.Contains(Constants.DotNetFramework46)); Assert.IsFalse(actualDiscoveryCriteria!.RunSettings.Contains(nameof(Architecture.ARM))); } @@ -1443,11 +1443,11 @@ public void RunTestsShouldNotUpdateFrameworkAndPlatformIfSpecifiedInDesignMode() _testRequestManager.RunTests(payload, new Mock().Object, new Mock().Object, _protocolConfig); - // don't infer them - _mockAssemblyMetadataProvider.Verify(a => a.GetArchitecture(It.IsAny()), Times.Never); - _mockAssemblyMetadataProvider.Verify(a => a.GetFrameWork(It.IsAny()), Times.Never); + // infer them so we can print warning when dlls are not compatible with runsettings + _mockAssemblyMetadataProvider.Verify(a => a.GetArchitecture(It.IsAny()), Times.Once); + _mockAssemblyMetadataProvider.Verify(a => a.GetFrameWork(It.IsAny()), Times.Once); - // don't update them + // but don't update runsettings because we want to keep what user specified Assert.IsTrue(actualTestRunCriteria!.TestRunSettings.Contains(Constants.DotNetFramework46)); Assert.IsTrue(actualTestRunCriteria!.TestRunSettings.Contains(nameof(Architecture.ARM))); } @@ -1484,7 +1484,7 @@ public void RunTestsShouldNotUpdatePlatformIfSpecifiedInDesignMode(string target _testRequestManager.RunTests(payload, new Mock().Object, new Mock().Object, _protocolConfig); - // don't infer it + // Infer _mockAssemblyMetadataProvider.Verify(a => a.GetArchitecture(It.IsAny()), Times.Never); // but infer framework _mockAssemblyMetadataProvider.Verify(a => a.GetFrameWork(It.IsAny()), Times.Once); @@ -1557,11 +1557,11 @@ public void RunTestsShouldNotpdateFrameworkAndPlatformInRunsettingsIfSpecifiedBy _testRequestManager.RunTests(payload, new Mock().Object, new Mock().Object, _protocolConfig); - // don't infer them - _mockAssemblyMetadataProvider.Verify(a => a.GetArchitecture(It.IsAny()), Times.Never); - _mockAssemblyMetadataProvider.Verify(a => a.GetFrameWork(It.IsAny()), Times.Never); + // infer them so we can print warnings when the assemblies are not compatible + _mockAssemblyMetadataProvider.Verify(a => a.GetArchitecture(It.IsAny()), Times.Once); + _mockAssemblyMetadataProvider.Verify(a => a.GetFrameWork(It.IsAny()), Times.Once); - // don't update them + // but don't update them in runsettings so we keep what user specified Assert.IsFalse(actualTestRunCriteria!.TestRunSettings.Contains(Constants.DotNetFramework46)); Assert.IsFalse(actualTestRunCriteria.TestRunSettings.Contains(nameof(Architecture.ARM))); } From 2980a29180be56a0014d7b671a788dde75d2e132 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Jare=C5=A1?= Date: Fri, 13 May 2022 17:31:42 +0200 Subject: [PATCH 098/112] More unit tests --- .../TestEngineTests.cs | 25 ------------------- .../TestRequestManagerTests.cs | 7 +++--- 2 files changed, 3 insertions(+), 29 deletions(-) diff --git a/test/Microsoft.TestPlatform.CrossPlatEngine.UnitTests/TestEngineTests.cs b/test/Microsoft.TestPlatform.CrossPlatEngine.UnitTests/TestEngineTests.cs index dfb5fb76c5..6a62a2657d 100644 --- a/test/Microsoft.TestPlatform.CrossPlatEngine.UnitTests/TestEngineTests.cs +++ b/test/Microsoft.TestPlatform.CrossPlatEngine.UnitTests/TestEngineTests.cs @@ -549,31 +549,6 @@ public void GetExtensionManagerShouldCollectMetrics() _mockMetricsCollection.Verify(mc => mc.Add(TelemetryDataConstants.ParallelEnabledDuringExecution, It.IsAny()), Times.Once); } - [TestMethod] - public void ProxyDataCollectionManagerShouldBeInitialzedWithCorrectTestSourcesWhenTestRunCriteriaContainsSourceList() - { - var settingXml = @""; - - var testRunCriteria = new TestRunCriteria(new List { "1.dll", "2.dll" }, 100, false, settingXml); - - var executionManager = _testEngine.GetExecutionManager(_mockRequestData.Object, testRunCriteria, new Dictionary()); - - Assert.IsTrue(((ProxyExecutionManagerWithDataCollection)executionManager).ProxyDataCollectionManager.Sources.Contains("1.dll")); - } - - [TestMethod] - public void ProxyDataCollectionManagerShouldBeInitialzedWithCorrectTestSourcesWhenTestRunCriteriaContainsTestCaseList() - { - var settingXml = @""; - - var testCaseList = new List { new TestCase("x.y.z", new Uri("uri://dummy"), "x.dll") }; - var testRunCriteria = new TestRunCriteria(testCaseList, 100, false, settingXml); - - var executionManager = _testEngine.GetExecutionManager(_mockRequestData.Object, testRunCriteria, new Dictionary()); - - Assert.IsTrue(((ProxyExecutionManagerWithDataCollection)executionManager).ProxyDataCollectionManager.Sources.Contains("x.dll")); - } - /// /// GetLoggerManager should return a non null instance. /// diff --git a/test/vstest.console.UnitTests/TestPlatformHelpers/TestRequestManagerTests.cs b/test/vstest.console.UnitTests/TestPlatformHelpers/TestRequestManagerTests.cs index 9e48c0b237..fcbe03c2fd 100644 --- a/test/vstest.console.UnitTests/TestPlatformHelpers/TestRequestManagerTests.cs +++ b/test/vstest.console.UnitTests/TestPlatformHelpers/TestRequestManagerTests.cs @@ -1484,12 +1484,11 @@ public void RunTestsShouldNotUpdatePlatformIfSpecifiedInDesignMode(string target _testRequestManager.RunTests(payload, new Mock().Object, new Mock().Object, _protocolConfig); - // Infer - _mockAssemblyMetadataProvider.Verify(a => a.GetArchitecture(It.IsAny()), Times.Never); - // but infer framework + // infer platform and framework so we can print warnings when dlls are not compatible with runsettings + _mockAssemblyMetadataProvider.Verify(a => a.GetArchitecture(It.IsAny()), Times.Once); _mockAssemblyMetadataProvider.Verify(a => a.GetFrameWork(It.IsAny()), Times.Once); - // don't update it + // don't update it in runsettings to keep what user provided Assert.IsTrue(actualTestRunCriteria!.TestRunSettings.Contains(targetPlatform)); } From 94bc0f1bf00f8d222eee94290a5e799845284a3e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Jare=C5=A1?= Date: Fri, 13 May 2022 17:57:54 +0200 Subject: [PATCH 099/112] Fix more unit tests --- .../netstandard/Tracing/PlatformEqtTrace.cs | 20 +++++++------------ .../Client/ProxyTestSessionManagerTests.cs | 3 ++- 2 files changed, 9 insertions(+), 14 deletions(-) diff --git a/src/Microsoft.TestPlatform.PlatformAbstractions/netstandard/Tracing/PlatformEqtTrace.cs b/src/Microsoft.TestPlatform.PlatformAbstractions/netstandard/Tracing/PlatformEqtTrace.cs index 4d1d220e4e..a82354f462 100644 --- a/src/Microsoft.TestPlatform.PlatformAbstractions/netstandard/Tracing/PlatformEqtTrace.cs +++ b/src/Microsoft.TestPlatform.PlatformAbstractions/netstandard/Tracing/PlatformEqtTrace.cs @@ -33,38 +33,32 @@ public class PlatformEqtTrace : IPlatformEqtTrace public void WriteLine(PlatformTraceLevel traceLevel, string message) { - if (!ShouldTrace(traceLevel)) - { - return; - } - var level = Enum.GetName(typeof(PlatformTraceLevel), traceLevel); +# if NETSTANDARD1_3 + System.IO.File.AppendAllText("C:\\temp\\log.txt", $"[{level}] {message}"); +#endif Debug.WriteLine($"[{level}] {message}"); } public bool InitializeVerboseTrace(string customLogFile) { -#if DEBUG // We don't have access to System.Diagnostics.Trace on netstandard1.3 // so we write to Debug. No need to initialize for non-debug builds. return true; -#else - return false; -#endif } public bool InitializeTrace(string customLogFile, PlatformTraceLevel traceLevel) { _traceLevel = traceLevel; -#if DEBUG // We don't have access to System.Diagnostics.Trace on netstandard1.3 // so we write to Debug. No need to initialize for non-debug builds. - return true; -#else - return false; +# if NETSTANDARD1_3 + + System.IO.Directory.CreateDirectory("C:\\temp\\"); #endif + return true; } public bool ShouldTrace(PlatformTraceLevel traceLevel) diff --git a/test/Microsoft.TestPlatform.CrossPlatEngine.UnitTests/Client/ProxyTestSessionManagerTests.cs b/test/Microsoft.TestPlatform.CrossPlatEngine.UnitTests/Client/ProxyTestSessionManagerTests.cs index dc9ba10394..1b8e508c8e 100644 --- a/test/Microsoft.TestPlatform.CrossPlatEngine.UnitTests/Client/ProxyTestSessionManagerTests.cs +++ b/test/Microsoft.TestPlatform.CrossPlatEngine.UnitTests/Client/ProxyTestSessionManagerTests.cs @@ -528,11 +528,12 @@ private ProxyTestSessionManager CreateProxy( StartTestSessionCriteria testSessionCriteria, ProxyOperationManager proxyOperationManager) { + var runSettings = testSessionCriteria.RunSettings ?? _fakeRunSettings; var runtimeProviderInfo = new TestRuntimeProviderInfo ( typeof(ITestRuntimeProvider), shared: false, - _fakeRunSettings, + runSettings, testSessionCriteria.Sources.Select(s => new SourceDetail { Source = s, From 2f6c67aeea2d95c8e89e1795a082c1086069c042 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Jare=C5=A1?= Date: Fri, 13 May 2022 20:50:20 +0200 Subject: [PATCH 100/112] Fix unit test --- .../Client/Parallel/ParallelOperationManager.cs | 13 ++++++++++--- .../Parallel/ParallelProxyExecutionManagerTests.cs | 1 + 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/src/Microsoft.TestPlatform.CrossPlatEngine/Client/Parallel/ParallelOperationManager.cs b/src/Microsoft.TestPlatform.CrossPlatEngine/Client/Parallel/ParallelOperationManager.cs index 4a63719368..c8c5228c49 100644 --- a/src/Microsoft.TestPlatform.CrossPlatEngine/Client/Parallel/ParallelOperationManager.cs +++ b/src/Microsoft.TestPlatform.CrossPlatEngine/Client/Parallel/ParallelOperationManager.cs @@ -176,10 +176,17 @@ private void ClearCompletedSlot(TManager completedManager) // When HandlePartialDiscovery or HandlePartialRun are in progress, and we call StopAllManagers, // it is possible that we will clear all slots, while ClearCompletedSlot is waiting on the lock, // so when it is allowed to enter it will fail to find the respective slot and fail. In this case it is - // okay that the slot is not found because we stopped all work and cleared the slots. - if (completedSlot.Count == 0 && _acceptMoreWork) + // okay that the slot is not found, and we do nothing, because we already stopped all work and cleared the slots. + if (completedSlot.Count == 0) { - throw new InvalidOperationException("The provided manager was not found in any slot."); + if (_acceptMoreWork) + { + throw new InvalidOperationException("The provided manager was not found in any slot."); + } + else + { + return; + } } if (completedSlot.Count > 1) diff --git a/test/Microsoft.TestPlatform.CrossPlatEngine.UnitTests/Client/Parallel/ParallelProxyExecutionManagerTests.cs b/test/Microsoft.TestPlatform.CrossPlatEngine.UnitTests/Client/Parallel/ParallelProxyExecutionManagerTests.cs index c59e836769..219b595dea 100644 --- a/test/Microsoft.TestPlatform.CrossPlatEngine.UnitTests/Client/Parallel/ParallelProxyExecutionManagerTests.cs +++ b/test/Microsoft.TestPlatform.CrossPlatEngine.UnitTests/Client/Parallel/ParallelProxyExecutionManagerTests.cs @@ -419,6 +419,7 @@ public void StartTestRunShouldAggregateRunData() Task.Run(() => parallelExecutionManager.StartTestRun(_testRunCriteriaWith2Sources, _mockEventHandler.Object)); + // If you are debugging this, maybe it is good idea to set this timeout higher. Assert.IsTrue(_executionCompleted.Wait(Timeout3Seconds), "Test run not completed."); Assert.IsNull(assertException, assertException?.ToString()); From 08506a8ea602c0c3836ce9202635d59d765dbdb4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Jare=C5=A1?= Date: Mon, 16 May 2022 12:15:57 +0200 Subject: [PATCH 101/112] Update resources --- .../Resources/xlf/CommonResources.cs.xlf | 8 ++++---- .../Resources/xlf/CommonResources.de.xlf | 8 ++++---- .../Resources/xlf/CommonResources.es.xlf | 8 ++++---- .../Resources/xlf/CommonResources.fr.xlf | 8 ++++---- .../Resources/xlf/CommonResources.it.xlf | 8 ++++---- .../Resources/xlf/CommonResources.ja.xlf | 8 ++++---- .../Resources/xlf/CommonResources.ko.xlf | 8 ++++---- .../Resources/xlf/CommonResources.pl.xlf | 8 ++++---- .../Resources/xlf/CommonResources.pt-BR.xlf | 8 ++++---- .../Resources/xlf/CommonResources.ru.xlf | 8 ++++---- .../Resources/xlf/CommonResources.tr.xlf | 8 ++++---- .../Resources/xlf/CommonResources.xlf | 4 ++-- .../Resources/xlf/CommonResources.zh-Hans.xlf | 8 ++++---- .../Resources/xlf/CommonResources.zh-Hant.xlf | 8 ++++---- 14 files changed, 54 insertions(+), 54 deletions(-) diff --git a/src/Microsoft.TestPlatform.ObjectModel/Resources/xlf/CommonResources.cs.xlf b/src/Microsoft.TestPlatform.ObjectModel/Resources/xlf/CommonResources.cs.xlf index 5e6f6bda56..a97e4c557b 100644 --- a/src/Microsoft.TestPlatform.ObjectModel/Resources/xlf/CommonResources.cs.xlf +++ b/src/Microsoft.TestPlatform.ObjectModel/Resources/xlf/CommonResources.cs.xlf @@ -8,9 +8,9 @@ Parametr nemůže být null nebo prázdný. - Test run detected DLL(s) which were built for different framework and platform versions. Following DLL(s) do not match current settings, which are {0} framework and {1} platform.{2}Go to {3} for more details on managing these settings. + Test run detected DLL(s) which would use different framework and platform versions. Following DLL(s) do not match current settings, which are {0} framework and {1} platform.{2}Go to {3} for more details on managing these settings. - Otestujte zjištěné knihovny DLL, které se vytvořily pro různé verze architektury a platformy. Následující knihovny DLL neodpovídají aktuálnímu nastavení, tedy architektuře {0} a platformě {1}.{2}Další podrobnosti o správě těchto nastavení najdete v {3}. + Otestujte zjištěné knihovny DLL, které se vytvořily pro různé verze architektury a platformy. Následující knihovny DLL neodpovídají aktuálnímu nastavení, tedy architektuře {0} a platformě {1}.{2}Další podrobnosti o správě těchto nastavení najdete v {3}. None of the provided test containers match the Platform Architecture and .Net Framework settings for the test run. Platform: {0} .Net Framework: {1}. Go to http://go.microsoft.com/fwlink/?LinkID=330428 for more details on managing these settings. @@ -23,8 +23,8 @@ - {0} is built for Framework {1} and Platform {2}. - Produkt {0} je sestaven pro rozhraní {1} a platformu {2}. + {0} would use Framework {1} and Platform {2}. + Produkt {0} je sestaven pro rozhraní {1} a platformu {2}. diff --git a/src/Microsoft.TestPlatform.ObjectModel/Resources/xlf/CommonResources.de.xlf b/src/Microsoft.TestPlatform.ObjectModel/Resources/xlf/CommonResources.de.xlf index 0904b4e6a4..3d2e7b6df9 100644 --- a/src/Microsoft.TestPlatform.ObjectModel/Resources/xlf/CommonResources.de.xlf +++ b/src/Microsoft.TestPlatform.ObjectModel/Resources/xlf/CommonResources.de.xlf @@ -8,9 +8,9 @@ Der Parameter darf nicht NULL oder leer sein. - Test run detected DLL(s) which were built for different framework and platform versions. Following DLL(s) do not match current settings, which are {0} framework and {1} platform.{2}Go to {3} for more details on managing these settings. + Test run detected DLL(s) which would use different framework and platform versions. Following DLL(s) do not match current settings, which are {0} framework and {1} platform.{2}Go to {3} for more details on managing these settings. - Beim Testlauf wurden DLLs erkannt, die für verschiedene Framework- und Plattformversionen erstellt wurden. Die folgenden DLLs entsprechen nicht den aktuellen Einstellungen: Framework {0} und Plattform {1}.{2}Wechseln Sie zu {3}, um weitere Informationen zum Verwalten dieser Einstellungen zu erhalten. + Beim Testlauf wurden DLLs erkannt, die für verschiedene Framework- und Plattformversionen erstellt wurden. Die folgenden DLLs entsprechen nicht den aktuellen Einstellungen: Framework {0} und Plattform {1}.{2}Wechseln Sie zu {3}, um weitere Informationen zum Verwalten dieser Einstellungen zu erhalten. None of the provided test containers match the Platform Architecture and .Net Framework settings for the test run. Platform: {0} .Net Framework: {1}. Go to http://go.microsoft.com/fwlink/?LinkID=330428 for more details on managing these settings. @@ -23,8 +23,8 @@ - {0} is built for Framework {1} and Platform {2}. - "{0}" wurde für das Framework {1} und die Plattform {2} erstellt. + {0} would use Framework {1} and Platform {2}. + "{0}" wurde für das Framework {1} und die Plattform {2} erstellt. diff --git a/src/Microsoft.TestPlatform.ObjectModel/Resources/xlf/CommonResources.es.xlf b/src/Microsoft.TestPlatform.ObjectModel/Resources/xlf/CommonResources.es.xlf index ea9dd037da..3375c797af 100644 --- a/src/Microsoft.TestPlatform.ObjectModel/Resources/xlf/CommonResources.es.xlf +++ b/src/Microsoft.TestPlatform.ObjectModel/Resources/xlf/CommonResources.es.xlf @@ -8,9 +8,9 @@ El parámetro no puede ser nulo ni estar vacío. - Test run detected DLL(s) which were built for different framework and platform versions. Following DLL(s) do not match current settings, which are {0} framework and {1} platform.{2}Go to {3} for more details on managing these settings. + Test run detected DLL(s) which would use different framework and platform versions. Following DLL(s) do not match current settings, which are {0} framework and {1} platform.{2}Go to {3} for more details on managing these settings. - La serie de pruebas detectó archivos DLL que se compilaron para diferentes versiones de marco de trabajo y plataforma. Los siguientes archivos DLL no coinciden con la configuración actual, que es el marco de trabajo {0} y la plataforma {1}. {2}Vaya a {3} para obtener más información sobre cómo administrar esta configuración. + La serie de pruebas detectó archivos DLL que se compilaron para diferentes versiones de marco de trabajo y plataforma. Los siguientes archivos DLL no coinciden con la configuración actual, que es el marco de trabajo {0} y la plataforma {1}. {2}Vaya a {3} para obtener más información sobre cómo administrar esta configuración. None of the provided test containers match the Platform Architecture and .Net Framework settings for the test run. Platform: {0} .Net Framework: {1}. Go to http://go.microsoft.com/fwlink/?LinkID=330428 for more details on managing these settings. @@ -23,8 +23,8 @@ - {0} is built for Framework {1} and Platform {2}. - {0} se ha compilado para el marco {1} y la plataforma {2}. + {0} would use Framework {1} and Platform {2}. + {0} se ha compilado para el marco {1} y la plataforma {2}. diff --git a/src/Microsoft.TestPlatform.ObjectModel/Resources/xlf/CommonResources.fr.xlf b/src/Microsoft.TestPlatform.ObjectModel/Resources/xlf/CommonResources.fr.xlf index 0c9b75f180..a433ea6503 100644 --- a/src/Microsoft.TestPlatform.ObjectModel/Resources/xlf/CommonResources.fr.xlf +++ b/src/Microsoft.TestPlatform.ObjectModel/Resources/xlf/CommonResources.fr.xlf @@ -8,9 +8,9 @@ Le paramètre ne peut pas avoir une valeur null ou être vide. - Test run detected DLL(s) which were built for different framework and platform versions. Following DLL(s) do not match current settings, which are {0} framework and {1} platform.{2}Go to {3} for more details on managing these settings. + Test run detected DLL(s) which would use different framework and platform versions. Following DLL(s) do not match current settings, which are {0} framework and {1} platform.{2}Go to {3} for more details on managing these settings. - La série de tests a détecté une ou plusieurs DLL qui ont été générées pour d'autres versions de framework et de plateforme. La ou les DLL suivantes ne correspondent pas aux paramètres actuels : framework {0} et plateforme {1}.{2}Accédez à {3} pour plus d'informations sur la gestion de ces paramètres. + La série de tests a détecté une ou plusieurs DLL qui ont été générées pour d'autres versions de framework et de plateforme. La ou les DLL suivantes ne correspondent pas aux paramètres actuels : framework {0} et plateforme {1}.{2}Accédez à {3} pour plus d'informations sur la gestion de ces paramètres. None of the provided test containers match the Platform Architecture and .Net Framework settings for the test run. Platform: {0} .Net Framework: {1}. Go to http://go.microsoft.com/fwlink/?LinkID=330428 for more details on managing these settings. @@ -23,8 +23,8 @@ - {0} is built for Framework {1} and Platform {2}. - {0} est généré pour le framework {1} et la plateforme {2}. + {0} would use Framework {1} and Platform {2}. + {0} est généré pour le framework {1} et la plateforme {2}. diff --git a/src/Microsoft.TestPlatform.ObjectModel/Resources/xlf/CommonResources.it.xlf b/src/Microsoft.TestPlatform.ObjectModel/Resources/xlf/CommonResources.it.xlf index cde112bdb8..4e25bc663c 100644 --- a/src/Microsoft.TestPlatform.ObjectModel/Resources/xlf/CommonResources.it.xlf +++ b/src/Microsoft.TestPlatform.ObjectModel/Resources/xlf/CommonResources.it.xlf @@ -8,9 +8,9 @@ Il parametro non può essere vuoto o Null. - Test run detected DLL(s) which were built for different framework and platform versions. Following DLL(s) do not match current settings, which are {0} framework and {1} platform.{2}Go to {3} for more details on managing these settings. + Test run detected DLL(s) which would use different framework and platform versions. Following DLL(s) do not match current settings, which are {0} framework and {1} platform.{2}Go to {3} for more details on managing these settings. - L'esecuzione dei test ha rilevato DLL compilate per versioni diverse del framework e della piattaforma. Le DLL seguenti non corrispondono alle impostazioni correnti, ovvero al framework {0} e alla piattaforma {1}.{2}Per altri dettagli sulla gestione di queste impostazioni, vedere {3}. + L'esecuzione dei test ha rilevato DLL compilate per versioni diverse del framework e della piattaforma. Le DLL seguenti non corrispondono alle impostazioni correnti, ovvero al framework {0} e alla piattaforma {1}.{2}Per altri dettagli sulla gestione di queste impostazioni, vedere {3}. None of the provided test containers match the Platform Architecture and .Net Framework settings for the test run. Platform: {0} .Net Framework: {1}. Go to http://go.microsoft.com/fwlink/?LinkID=330428 for more details on managing these settings. @@ -23,8 +23,8 @@ - {0} is built for Framework {1} and Platform {2}. - {0} viene compilato per il framework {1} e la piattaforma {2}. + {0} would use Framework {1} and Platform {2}. + {0} viene compilato per il framework {1} e la piattaforma {2}. diff --git a/src/Microsoft.TestPlatform.ObjectModel/Resources/xlf/CommonResources.ja.xlf b/src/Microsoft.TestPlatform.ObjectModel/Resources/xlf/CommonResources.ja.xlf index 5e8f037b7a..33c7312fd5 100644 --- a/src/Microsoft.TestPlatform.ObjectModel/Resources/xlf/CommonResources.ja.xlf +++ b/src/Microsoft.TestPlatform.ObjectModel/Resources/xlf/CommonResources.ja.xlf @@ -8,9 +8,9 @@ パラメーターを null または空にすることはできません。 - Test run detected DLL(s) which were built for different framework and platform versions. Following DLL(s) do not match current settings, which are {0} framework and {1} platform.{2}Go to {3} for more details on managing these settings. + Test run detected DLL(s) which would use different framework and platform versions. Following DLL(s) do not match current settings, which are {0} framework and {1} platform.{2}Go to {3} for more details on managing these settings. - テストの実行で、別のフレームワークとプラットフォームのバージョン用にビルドされた DLL が検出されました。以下の DLL は、現在の設定と一致しません。現在のフレームワーク: {0}、プラットフォーム: {1}。{2}これらの設定を管理することについて詳しくは、{3} にアクセスしてください。 + テストの実行で、別のフレームワークとプラットフォームのバージョン用にビルドされた DLL が検出されました。以下の DLL は、現在の設定と一致しません。現在のフレームワーク: {0}、プラットフォーム: {1}。{2}これらの設定を管理することについて詳しくは、{3} にアクセスしてください。 None of the provided test containers match the Platform Architecture and .Net Framework settings for the test run. Platform: {0} .Net Framework: {1}. Go to http://go.microsoft.com/fwlink/?LinkID=330428 for more details on managing these settings. @@ -23,8 +23,8 @@ - {0} is built for Framework {1} and Platform {2}. - {0} はフレームワーク {1} およびプラットフォーム {2} 向けにビルドされます。 + {0} would use Framework {1} and Platform {2}. + {0} はフレームワーク {1} およびプラットフォーム {2} 向けにビルドされます。 diff --git a/src/Microsoft.TestPlatform.ObjectModel/Resources/xlf/CommonResources.ko.xlf b/src/Microsoft.TestPlatform.ObjectModel/Resources/xlf/CommonResources.ko.xlf index ee4b96f092..727fe5f09d 100644 --- a/src/Microsoft.TestPlatform.ObjectModel/Resources/xlf/CommonResources.ko.xlf +++ b/src/Microsoft.TestPlatform.ObjectModel/Resources/xlf/CommonResources.ko.xlf @@ -8,9 +8,9 @@ 매개 변수는 null이거나 비워 둘 수 없습니다. - Test run detected DLL(s) which were built for different framework and platform versions. Following DLL(s) do not match current settings, which are {0} framework and {1} platform.{2}Go to {3} for more details on managing these settings. + Test run detected DLL(s) which would use different framework and platform versions. Following DLL(s) do not match current settings, which are {0} framework and {1} platform.{2}Go to {3} for more details on managing these settings. - 테스트 실행에서 다른 프레임워크 및 플랫폼 버전용으로 빌드된 DLL을 감지했습니다. 다음 DLL은 현재 설정({0} 프레임워크 및 {1} 플랫폼)과 일치하지 않습니다.{2}이 설정을 관리하는 방법에 대한 자세한 내용을 보려면 {3}(으)로 이동하세요. + 테스트 실행에서 다른 프레임워크 및 플랫폼 버전용으로 빌드된 DLL을 감지했습니다. 다음 DLL은 현재 설정({0} 프레임워크 및 {1} 플랫폼)과 일치하지 않습니다.{2}이 설정을 관리하는 방법에 대한 자세한 내용을 보려면 {3}(으)로 이동하세요. None of the provided test containers match the Platform Architecture and .Net Framework settings for the test run. Platform: {0} .Net Framework: {1}. Go to http://go.microsoft.com/fwlink/?LinkID=330428 for more details on managing these settings. @@ -23,8 +23,8 @@ - {0} is built for Framework {1} and Platform {2}. - {0}은(는) 프레임워크 {1} 및 플랫폼 {2}을(를) 대상으로 빌드됩니다. + {0} would use Framework {1} and Platform {2}. + {0}은(는) 프레임워크 {1} 및 플랫폼 {2}을(를) 대상으로 빌드됩니다. diff --git a/src/Microsoft.TestPlatform.ObjectModel/Resources/xlf/CommonResources.pl.xlf b/src/Microsoft.TestPlatform.ObjectModel/Resources/xlf/CommonResources.pl.xlf index e4e0d81fb4..21d972a1ec 100644 --- a/src/Microsoft.TestPlatform.ObjectModel/Resources/xlf/CommonResources.pl.xlf +++ b/src/Microsoft.TestPlatform.ObjectModel/Resources/xlf/CommonResources.pl.xlf @@ -8,9 +8,9 @@ Parametr nie może mieć wartości null ani być pusty. - Test run detected DLL(s) which were built for different framework and platform versions. Following DLL(s) do not match current settings, which are {0} framework and {1} platform.{2}Go to {3} for more details on managing these settings. + Test run detected DLL(s) which would use different framework and platform versions. Following DLL(s) do not match current settings, which are {0} framework and {1} platform.{2}Go to {3} for more details on managing these settings. - Przebieg testowy wykrył biblioteki DLL utworzone dla innych wersji struktury i platformy. Następujące biblioteki DLL są niezgodne z bieżącymi ustawieniami (określającymi strukturę {0} i platformę {1}).{2}Aby uzyskać więcej informacji o zarządzaniu tymi ustawieniami, przejdź do: {3}. + Przebieg testowy wykrył biblioteki DLL utworzone dla innych wersji struktury i platformy. Następujące biblioteki DLL są niezgodne z bieżącymi ustawieniami (określającymi strukturę {0} i platformę {1}).{2}Aby uzyskać więcej informacji o zarządzaniu tymi ustawieniami, przejdź do: {3}. None of the provided test containers match the Platform Architecture and .Net Framework settings for the test run. Platform: {0} .Net Framework: {1}. Go to http://go.microsoft.com/fwlink/?LinkID=330428 for more details on managing these settings. @@ -23,8 +23,8 @@ - {0} is built for Framework {1} and Platform {2}. - Element {0} został skompilowany dla struktury {1} i platformy {2}. + {0} would use Framework {1} and Platform {2}. + Element {0} został skompilowany dla struktury {1} i platformy {2}. diff --git a/src/Microsoft.TestPlatform.ObjectModel/Resources/xlf/CommonResources.pt-BR.xlf b/src/Microsoft.TestPlatform.ObjectModel/Resources/xlf/CommonResources.pt-BR.xlf index d36d02f785..54be5010de 100644 --- a/src/Microsoft.TestPlatform.ObjectModel/Resources/xlf/CommonResources.pt-BR.xlf +++ b/src/Microsoft.TestPlatform.ObjectModel/Resources/xlf/CommonResources.pt-BR.xlf @@ -8,9 +8,9 @@ O parâmetro não pode ser nulo ou estar vazio. - Test run detected DLL(s) which were built for different framework and platform versions. Following DLL(s) do not match current settings, which are {0} framework and {1} platform.{2}Go to {3} for more details on managing these settings. + Test run detected DLL(s) which would use different framework and platform versions. Following DLL(s) do not match current settings, which are {0} framework and {1} platform.{2}Go to {3} for more details on managing these settings. - A execução de teste detectou DLLs que foram criadas para diferentes versões de estrutura e de plataforma. As DLLs a seguir não correspondem às configurações atuais, que são a estrutura {0} e a plataforma {1}.{2}Acesse {3} para obter mais detalhes sobre o gerenciamento dessas configurações. + A execução de teste detectou DLLs que foram criadas para diferentes versões de estrutura e de plataforma. As DLLs a seguir não correspondem às configurações atuais, que são a estrutura {0} e a plataforma {1}.{2}Acesse {3} para obter mais detalhes sobre o gerenciamento dessas configurações. None of the provided test containers match the Platform Architecture and .Net Framework settings for the test run. Platform: {0} .Net Framework: {1}. Go to http://go.microsoft.com/fwlink/?LinkID=330428 for more details on managing these settings. @@ -23,8 +23,8 @@ - {0} is built for Framework {1} and Platform {2}. - {0} é compilado para Framework {1} e Platform {2}. + {0} would use Framework {1} and Platform {2}. + {0} é compilado para Framework {1} e Platform {2}. diff --git a/src/Microsoft.TestPlatform.ObjectModel/Resources/xlf/CommonResources.ru.xlf b/src/Microsoft.TestPlatform.ObjectModel/Resources/xlf/CommonResources.ru.xlf index 10fda594fd..1b033235c6 100644 --- a/src/Microsoft.TestPlatform.ObjectModel/Resources/xlf/CommonResources.ru.xlf +++ b/src/Microsoft.TestPlatform.ObjectModel/Resources/xlf/CommonResources.ru.xlf @@ -8,9 +8,9 @@ Параметр не может быть пустым или иметь значение null. - Test run detected DLL(s) which were built for different framework and platform versions. Following DLL(s) do not match current settings, which are {0} framework and {1} platform.{2}Go to {3} for more details on managing these settings. + Test run detected DLL(s) which would use different framework and platform versions. Following DLL(s) do not match current settings, which are {0} framework and {1} platform.{2}Go to {3} for more details on managing these settings. - Тестовый запуск обнаружил библиотеки DLL, которые были созданы для различных версий инфраструктуры и платформы. Следующие библиотеки DLL не соответствуют текущим параметрам (инфраструктура {0} и платформа {1}).{2}Перейдите по ссылке {3} для получения дополнительных сведений об управлении этими параметрами. + Тестовый запуск обнаружил библиотеки DLL, которые были созданы для различных версий инфраструктуры и платформы. Следующие библиотеки DLL не соответствуют текущим параметрам (инфраструктура {0} и платформа {1}).{2}Перейдите по ссылке {3} для получения дополнительных сведений об управлении этими параметрами. None of the provided test containers match the Platform Architecture and .Net Framework settings for the test run. Platform: {0} .Net Framework: {1}. Go to http://go.microsoft.com/fwlink/?LinkID=330428 for more details on managing these settings. @@ -23,8 +23,8 @@ - {0} is built for Framework {1} and Platform {2}. - {0} создается для программной платформы {1} и аппаратной платформы {2}. + {0} would use Framework {1} and Platform {2}. + {0} создается для программной платформы {1} и аппаратной платформы {2}. diff --git a/src/Microsoft.TestPlatform.ObjectModel/Resources/xlf/CommonResources.tr.xlf b/src/Microsoft.TestPlatform.ObjectModel/Resources/xlf/CommonResources.tr.xlf index d8243a5dff..9d70720de4 100644 --- a/src/Microsoft.TestPlatform.ObjectModel/Resources/xlf/CommonResources.tr.xlf +++ b/src/Microsoft.TestPlatform.ObjectModel/Resources/xlf/CommonResources.tr.xlf @@ -8,9 +8,9 @@ Parametre null veya boş olamaz. - Test run detected DLL(s) which were built for different framework and platform versions. Following DLL(s) do not match current settings, which are {0} framework and {1} platform.{2}Go to {3} for more details on managing these settings. + Test run detected DLL(s) which would use different framework and platform versions. Following DLL(s) do not match current settings, which are {0} framework and {1} platform.{2}Go to {3} for more details on managing these settings. - Test çalıştırması, farklı çerçeve ve platform sürümleri için oluşturulmuş DLL'ler buldu. Aşağıdaki DLL'ler, {0} çerçevesi ve {1} platformu olan geçerli ayarlarla eşleşmiyor. {2}Bu ayarları yönetme hakkında daha fazla ayrıntı için {3} bölümüne gidin. + Test çalıştırması, farklı çerçeve ve platform sürümleri için oluşturulmuş DLL'ler buldu. Aşağıdaki DLL'ler, {0} çerçevesi ve {1} platformu olan geçerli ayarlarla eşleşmiyor. {2}Bu ayarları yönetme hakkında daha fazla ayrıntı için {3} bölümüne gidin. None of the provided test containers match the Platform Architecture and .Net Framework settings for the test run. Platform: {0} .Net Framework: {1}. Go to http://go.microsoft.com/fwlink/?LinkID=330428 for more details on managing these settings. @@ -23,8 +23,8 @@ - {0} is built for Framework {1} and Platform {2}. - {0}, {1} Çerçevesi ve {2} Platformu için derlenmiştir. + {0} would use Framework {1} and Platform {2}. + {0}, {1} Çerçevesi ve {2} Platformu için derlenmiştir. diff --git a/src/Microsoft.TestPlatform.ObjectModel/Resources/xlf/CommonResources.xlf b/src/Microsoft.TestPlatform.ObjectModel/Resources/xlf/CommonResources.xlf index 29ceff30df..9b140afbe6 100644 --- a/src/Microsoft.TestPlatform.ObjectModel/Resources/xlf/CommonResources.xlf +++ b/src/Microsoft.TestPlatform.ObjectModel/Resources/xlf/CommonResources.xlf @@ -7,7 +7,7 @@ - Test run detected DLL(s) which were built for different framework and platform versions. Following DLL(s) do not match current settings, which are {0} framework and {1} platform.{2}Go to {3} for more details on managing these settings. + Test run detected DLL(s) which would use different framework and platform versions. Following DLL(s) do not match current settings, which are {0} framework and {1} platform.{2}Go to {3} for more details on managing these settings. @@ -20,7 +20,7 @@ - {0} is built for Framework {1} and Platform {2}. + {0} would use Framework {1} and Platform {2}. {0} is built for Framework {1} and Platform {2}. diff --git a/src/Microsoft.TestPlatform.ObjectModel/Resources/xlf/CommonResources.zh-Hans.xlf b/src/Microsoft.TestPlatform.ObjectModel/Resources/xlf/CommonResources.zh-Hans.xlf index 91aa3fa748..e5d069d375 100644 --- a/src/Microsoft.TestPlatform.ObjectModel/Resources/xlf/CommonResources.zh-Hans.xlf +++ b/src/Microsoft.TestPlatform.ObjectModel/Resources/xlf/CommonResources.zh-Hans.xlf @@ -8,9 +8,9 @@ 参数不能为 NULL 或为空。 - Test run detected DLL(s) which were built for different framework and platform versions. Following DLL(s) do not match current settings, which are {0} framework and {1} platform.{2}Go to {3} for more details on managing these settings. + Test run detected DLL(s) which would use different framework and platform versions. Following DLL(s) do not match current settings, which are {0} framework and {1} platform.{2}Go to {3} for more details on managing these settings. - 测试运行检测到为不同框架和平台版本构建的 DLL。后面的 DLL 与当前设置(即 {0} 框架和 {1} 平台)不匹配。{2}如需了解如何管理这些设置，请转到 {3}。 + 测试运行检测到为不同框架和平台版本构建的 DLL。后面的 DLL 与当前设置(即 {0} 框架和 {1} 平台)不匹配。{2}如需了解如何管理这些设置，请转到 {3}。 None of the provided test containers match the Platform Architecture and .Net Framework settings for the test run. Platform: {0} .Net Framework: {1}. Go to http://go.microsoft.com/fwlink/?LinkID=330428 for more details on managing these settings. @@ -23,8 +23,8 @@ - {0} is built for Framework {1} and Platform {2}. - {0} 适用于 {1} 框架和 {2} 平台。 + {0} would use Framework {1} and Platform {2}. + {0} 适用于 {1} 框架和 {2} 平台。 diff --git a/src/Microsoft.TestPlatform.ObjectModel/Resources/xlf/CommonResources.zh-Hant.xlf b/src/Microsoft.TestPlatform.ObjectModel/Resources/xlf/CommonResources.zh-Hant.xlf index faa0382b32..5de22766e6 100644 --- a/src/Microsoft.TestPlatform.ObjectModel/Resources/xlf/CommonResources.zh-Hant.xlf +++ b/src/Microsoft.TestPlatform.ObjectModel/Resources/xlf/CommonResources.zh-Hant.xlf @@ -8,9 +8,9 @@ 參數不可為 null 或空白。 - Test run detected DLL(s) which were built for different framework and platform versions. Following DLL(s) do not match current settings, which are {0} framework and {1} platform.{2}Go to {3} for more details on managing these settings. + Test run detected DLL(s) which would use different framework and platform versions. Following DLL(s) do not match current settings, which are {0} framework and {1} platform.{2}Go to {3} for more details on managing these settings. - 測試回合偵測到為不同架構和平台版本所建置的 DLL。下列 DLL 與目前的設定不相符，亦即 {0} 架構及 {1} 平台。{2}如需管理這些設定的詳細資料，請前往 {3}。 + 測試回合偵測到為不同架構和平台版本所建置的 DLL。下列 DLL 與目前的設定不相符，亦即 {0} 架構及 {1} 平台。{2}如需管理這些設定的詳細資料，請前往 {3}。 None of the provided test containers match the Platform Architecture and .Net Framework settings for the test run. Platform: {0} .Net Framework: {1}. Go to http://go.microsoft.com/fwlink/?LinkID=330428 for more details on managing these settings. @@ -23,8 +23,8 @@ - {0} is built for Framework {1} and Platform {2}. - {0} 專為架構 {1} 及平台 {2} 建置。 + {0} would use Framework {1} and Platform {2}. + {0} 專為架構 {1} 及平台 {2} 建置。 From 750c8bbc24d7534300cfb8934d0b614dae73d95b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Jare=C5=A1?= Date: Mon, 16 May 2022 15:40:32 +0200 Subject: [PATCH 102/112] Fix collection modified exception --- .../TestPlatformHelpers/TestRequestManager.cs | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/vstest.console/TestPlatformHelpers/TestRequestManager.cs b/src/vstest.console/TestPlatformHelpers/TestRequestManager.cs index 718d753eef..0d70ce6dee 100644 --- a/src/vstest.console/TestPlatformHelpers/TestRequestManager.cs +++ b/src/vstest.console/TestPlatformHelpers/TestRequestManager.cs @@ -767,7 +767,9 @@ private bool UpdateRunSettingsIfRequired( // The sourceToArchitectureMap contains the real architecture, overwrite it by the value chosen by runsettings, to force one unified platform to be used. if (disableMultiTfm || platformSetByRunsettings) { - foreach (var key in sourceToArchitectureMap.Keys) + // Copy the list of key, otherwise we will get collection changed exception. + var keys = sourceToArchitectureMap.Keys.ToList(); + foreach (var key in keys) { sourceToArchitectureMap[key] = chosenPlatform; } @@ -776,7 +778,9 @@ private bool UpdateRunSettingsIfRequired( // The sourceToFrameworkMap contains the real framework, overwrite it by the value chosen by runsettings, to force one unified framework to be used. if (disableMultiTfm || frameworkSetByRunsettings) { - foreach (var key in sourceToFrameworkMap.Keys) + // Copy the list of key, otherwise we will get collection changed exception. + var keys = sourceToFrameworkMap.Keys.ToList(); + foreach (var key in keys) { sourceToFrameworkMap[key] = chosenFramework; } From fe4218e314878d06efc2dd37a3bab5c524eca0a0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Jare=C5=A1?= Date: Mon, 16 May 2022 20:57:38 +0200 Subject: [PATCH 103/112] Don't return true where you should not --- .../Client/Parallel/ParallelProxyExecutionManager.cs | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/Microsoft.TestPlatform.CrossPlatEngine/Client/Parallel/ParallelProxyExecutionManager.cs b/src/Microsoft.TestPlatform.CrossPlatEngine/Client/Parallel/ParallelProxyExecutionManager.cs index b732a865fc..4598274861 100644 --- a/src/Microsoft.TestPlatform.CrossPlatEngine/Client/Parallel/ParallelProxyExecutionManager.cs +++ b/src/Microsoft.TestPlatform.CrossPlatEngine/Client/Parallel/ParallelProxyExecutionManager.cs @@ -160,10 +160,11 @@ public bool HandlePartialRunComplete( if (!testRunCompleteArgs.IsCanceled && !_abortRequested) { var hadMoreWork = _parallelOperationManager.RunNextWork(proxyExecutionManager); - if (!hadMoreWork) - { - return true; - } + // ugh huh????! don't return true, or else stuff will fail! + //if (!hadMoreWork) + //{ + // return true; + //} } return false; From a6dbcf277dd3b70e615b7628e7678f3d8f0c249a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Jare=C5=A1?= Date: Tue, 17 May 2022 10:01:13 +0200 Subject: [PATCH 104/112] Fix assignment --- .../Client/Parallel/ParallelProxyExecutionManager.cs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/Microsoft.TestPlatform.CrossPlatEngine/Client/Parallel/ParallelProxyExecutionManager.cs b/src/Microsoft.TestPlatform.CrossPlatEngine/Client/Parallel/ParallelProxyExecutionManager.cs index 4598274861..e78b86ac29 100644 --- a/src/Microsoft.TestPlatform.CrossPlatEngine/Client/Parallel/ParallelProxyExecutionManager.cs +++ b/src/Microsoft.TestPlatform.CrossPlatEngine/Client/Parallel/ParallelProxyExecutionManager.cs @@ -159,7 +159,8 @@ public bool HandlePartialRunComplete( // and queue another test run. if (!testRunCompleteArgs.IsCanceled && !_abortRequested) { - var hadMoreWork = _parallelOperationManager.RunNextWork(proxyExecutionManager); + _parallelOperationManager.RunNextWork(proxyExecutionManager); + //var hadMoreWork = _parallelOperationManager.RunNextWork(proxyExecutionManager); // ugh huh????! don't return true, or else stuff will fail! //if (!hadMoreWork) //{ From dbb04f6415b7f447d51e7c29327e77fffe806385 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Jare=C5=A1?= Date: Tue, 17 May 2022 13:56:25 +0200 Subject: [PATCH 105/112] Fix tests --- src/AttachVS/AttachVs.cs | 11 ++----- .../TestPlatform.cs | 3 -- .../Parallel/ParallelProxyDiscoveryManager.cs | 2 -- .../TestSession/ProxyTestSessionManager.cs | 2 +- .../netstandard/Tracing/PlatformEqtTrace.cs | 20 ++++++++----- .../ExecutionTests.cs | 8 ++--- .../Properties/AssemblyInfo.cs | 2 +- .../RunsettingsTests.cs | 30 +++++++++++++++---- .../CustomTestHostTests.cs | 2 ++ .../TranslationLayerTests/DiscoverTests.cs | 4 +-- .../EventHandler/DiscoveryEventHandler.cs | 6 +++- .../Fakes/FakeCommunicationChannel.cs | 8 ++--- 12 files changed, 58 insertions(+), 40 deletions(-) diff --git a/src/AttachVS/AttachVs.cs b/src/AttachVS/AttachVs.cs index acbcfabe52..407c1a5045 100644 --- a/src/AttachVS/AttachVs.cs +++ b/src/AttachVS/AttachVs.cs @@ -138,19 +138,12 @@ private static bool AttachVs(Process vs, int pid) } } } - catch (COMException ex) + // Catch the exception if it is COMException coming directly, or coming from methodInvocation, otherwise just let it be. + catch (Exception ex) when (ex is COMException || (ex is TargetInvocationException tie && tie.InnerException is COMException)) { Trace($"ComException: Retrying in 250ms.\n{ex}"); Thread.Sleep(250); } - catch (TargetInvocationException ex) - { - if (ex.InnerException is not COMException) - throw; - - Trace($"ComException: Retrying in 250ms.\n{ex}"); - Thread.Sleep(250); - } } Marshal.ReleaseComObject(moniker[0]); diff --git a/src/Microsoft.TestPlatform.Client/TestPlatform.cs b/src/Microsoft.TestPlatform.Client/TestPlatform.cs index cb5b37e842..d029abb5d9 100644 --- a/src/Microsoft.TestPlatform.Client/TestPlatform.cs +++ b/src/Microsoft.TestPlatform.Client/TestPlatform.cs @@ -75,9 +75,6 @@ protected internal TestPlatform( _testHostProviderManager = testHostProviderManager; } - /// - /// Gets or sets the test engine instance. - /// private readonly ITestEngine _testEngine; /// diff --git a/src/Microsoft.TestPlatform.CrossPlatEngine/Client/Parallel/ParallelProxyDiscoveryManager.cs b/src/Microsoft.TestPlatform.CrossPlatEngine/Client/Parallel/ParallelProxyDiscoveryManager.cs index d9837880a2..23f8f69574 100644 --- a/src/Microsoft.TestPlatform.CrossPlatEngine/Client/Parallel/ParallelProxyDiscoveryManager.cs +++ b/src/Microsoft.TestPlatform.CrossPlatEngine/Client/Parallel/ParallelProxyDiscoveryManager.cs @@ -15,8 +15,6 @@ using Microsoft.VisualStudio.TestPlatform.ObjectModel.Engine; using Microsoft.VisualStudio.TestPlatform.ObjectModel.Logging; -#nullable disable - namespace Microsoft.VisualStudio.TestPlatform.CrossPlatEngine.Client.Parallel; /// diff --git a/src/Microsoft.TestPlatform.CrossPlatEngine/TestSession/ProxyTestSessionManager.cs b/src/Microsoft.TestPlatform.CrossPlatEngine/TestSession/ProxyTestSessionManager.cs index 56b029e91c..4bc8f33b17 100644 --- a/src/Microsoft.TestPlatform.CrossPlatEngine/TestSession/ProxyTestSessionManager.cs +++ b/src/Microsoft.TestPlatform.CrossPlatEngine/TestSession/ProxyTestSessionManager.cs @@ -112,7 +112,7 @@ public virtual bool StartSession(ITestSessionEventsHandler eventsHandler, IReque // testhosts as we have sources. In the future we will have a maxParallelLevel set to the actual parallel level // (which might be lower than the number of sources) and we should do some kind of thinking here to figure out how to split the sources. // To follow the way parallel execution and discovery is (supposed to be) working, there should be as many testhosts - // as the maxParallel level pre-started,and marked with the Shared, and configuration that they can run. + // as the maxParallel level pre-started, and marked with the Shared, and configuration that they can run. // Create all the proxies in parallel, one task per proxy. var taskList = new Task[_maxTesthostCount]; diff --git a/src/Microsoft.TestPlatform.PlatformAbstractions/netstandard/Tracing/PlatformEqtTrace.cs b/src/Microsoft.TestPlatform.PlatformAbstractions/netstandard/Tracing/PlatformEqtTrace.cs index a82354f462..4d1d220e4e 100644 --- a/src/Microsoft.TestPlatform.PlatformAbstractions/netstandard/Tracing/PlatformEqtTrace.cs +++ b/src/Microsoft.TestPlatform.PlatformAbstractions/netstandard/Tracing/PlatformEqtTrace.cs @@ -33,32 +33,38 @@ public class PlatformEqtTrace : IPlatformEqtTrace public void WriteLine(PlatformTraceLevel traceLevel, string message) { + if (!ShouldTrace(traceLevel)) + { + return; + } + var level = Enum.GetName(typeof(PlatformTraceLevel), traceLevel); -# if NETSTANDARD1_3 - System.IO.File.AppendAllText("C:\\temp\\log.txt", $"[{level}] {message}"); -#endif Debug.WriteLine($"[{level}] {message}"); } public bool InitializeVerboseTrace(string customLogFile) { +#if DEBUG // We don't have access to System.Diagnostics.Trace on netstandard1.3 // so we write to Debug. No need to initialize for non-debug builds. return true; +#else + return false; +#endif } public bool InitializeTrace(string customLogFile, PlatformTraceLevel traceLevel) { _traceLevel = traceLevel; +#if DEBUG // We don't have access to System.Diagnostics.Trace on netstandard1.3 // so we write to Debug. No need to initialize for non-debug builds. -# if NETSTANDARD1_3 - - System.IO.Directory.CreateDirectory("C:\\temp\\"); -#endif return true; +#else + return false; +#endif } public bool ShouldTrace(PlatformTraceLevel traceLevel) diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/ExecutionTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/ExecutionTests.cs index c5230aa9f7..ea9294d8eb 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/ExecutionTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/ExecutionTests.cs @@ -112,7 +112,7 @@ public void RunMultipleTestAssembliesWithoutTestAdapterPath(RunnerInfo runnerInf // and after --arch feature implementation we won't find correct muxer on CI. [TestCategory("Windows")] [TestMethod] - [MSTestCompatibilityDataSource] + [MSTestCompatibilityDataSource(DebugVSTestConsole = true)] public void RunMultipleTestAssembliesInParallel(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); @@ -259,7 +259,7 @@ public void IncompatibleSourcesWarningShouldBeDisplayedInTheConsoleWhenGivenInco { SetTestEnvironment(_testEnvironment, runnerInfo); - var expectedWarningContains = @"Following DLL(s) do not match current settings, which are .NETFramework,Version=v4.5.1 framework and X86 platform. SimpleTestProject3.dll is built for Framework .NETFramework,Version=v4.5.1 and Platform X64"; + var expectedWarningContains = @"Following DLL(s) do not match current settings, which are .NETFramework,Version=v4.5.1 framework and X86 platform. SimpleTestProject3.dll would use Framework .NETFramework,Version=v4.5.1 and Platform X64"; var assemblyPaths = BuildMultipleAssemblyPath("SimpleTestProject3.dll", "SimpleTestProjectx86.dll"); var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: TempDirectory.Path); @@ -281,7 +281,7 @@ public void NoIncompatibleSourcesWarningShouldBeDisplayedInTheConsoleWhenGivenSi { SetTestEnvironment(_testEnvironment, runnerInfo); - var expectedWarningContains = @"Following DLL(s) do not match current settings, which are .NETFramework,Version=v4.5.1 framework and X86 platform. SimpleTestProjectx86 is built for Framework .NETFramework,Version=v4.5.1 and Platform X86"; + var expectedWarningContains = @"Following DLL(s) do not match current settings, which are .NETFramework,Version=v4.5.1 framework and X86 platform. SimpleTestProjectx86 would use Framework .NETFramework,Version=v4.5.1 and Platform X86"; var assemblyPaths = BuildMultipleAssemblyPath("SimpleTestProjectx86.dll"); var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: TempDirectory.Path); @@ -301,7 +301,7 @@ public void IncompatibleSourcesWarningShouldBeDisplayedInTheConsoleOnlyWhenRunni { SetTestEnvironment(_testEnvironment, runnerInfo); - var expectedWarningContains = @"Following DLL(s) do not match current settings, which are .NETFramework,Version=v4.5.1 framework and X86 platform. SimpleTestProject2.dll is built for Framework .NETFramework,Version=v4.5.1 and Platform X64"; + var expectedWarningContains = @"Following DLL(s) do not match current settings, which are .NETFramework,Version=v4.5.1 framework and X86 platform. SimpleTestProject2.dll would use Framework .NETFramework,Version=v4.5.1 and Platform X64"; var assemblyPaths = BuildMultipleAssemblyPath("SimpleTestProject2.dll"); var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: TempDirectory.Path); diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/Properties/AssemblyInfo.cs b/test/Microsoft.TestPlatform.AcceptanceTests/Properties/AssemblyInfo.cs index a31c6c40a5..71781d0b4a 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/Properties/AssemblyInfo.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/Properties/AssemblyInfo.cs @@ -23,4 +23,4 @@ [assembly: Guid("755996fa-672a-4272-9776-7f707a520058")] // Enable IAP at class level with as many threads as possible based on CPU and core count. -[assembly: Parallelize(Workers = 0, Scope = ExecutionScope.MethodLevel)] +[assembly: Parallelize(Workers = 0, Scope = ExecutionScope.ClassLevel)] diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/RunsettingsTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/RunsettingsTests.cs index 2633e3f715..7b6ddfa5f2 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/RunsettingsTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/RunsettingsTests.cs @@ -31,7 +31,10 @@ public void CommandLineRunSettingsShouldWinAmongAllOptions(RunnerInfo runnerInfo // We pass 2 dlls in RunTestWithRunSettings, for .NET Framework they run in // 1 hosts because that host is Shared. - var expectedNumOfProcessCreated = runnerInfo.IsNetFrameworkTarget ? 1 : 2; + // + // Testhosts are no longer shared, we should see 2 of them always. + // var expectedNumOfProcessCreated = runnerInfo.IsNetFrameworkTarget ? 1 : 2; + var expectedNumOfProcessCreated = 2; // passing parallel var runConfigurationDictionary = new Dictionary @@ -71,7 +74,10 @@ public void CLIRunsettingsShouldWinBetweenCLISwitchesAndCLIRunsettings(RunnerInf // We pass 2 dlls in RunTestWithRunSettings, for .NET Framework they run in // 1 hosts because that host is Shared. - var expectedNumOfProcessCreated = runnerInfo.IsNetFrameworkTarget ? 1 : 2; + // + // Testhosts are no longer shared, we should see 2 of them always. + // var expectedNumOfProcessCreated = runnerInfo.IsNetFrameworkTarget ? 1 : 2; + var expectedNumOfProcessCreated = 2; // Pass parallel var additionalArgs = "/Parallel"; @@ -107,7 +113,10 @@ public void CommandLineSwitchesShouldWinBetweenSettingsFileAndCommandLineSwitche // We pass 2 dlls in RunTestWithRunSettings, for .NET Framework they run in // 1 hosts because that host is Shared. - var expectedNumOfProcessCreated = runnerInfo.IsNetFrameworkTarget ? 1 : 2; + // + // Testhosts are no longer shared, we should see 2 of them always. + // var expectedNumOfProcessCreated = runnerInfo.IsNetFrameworkTarget ? 1 : 2; + var expectedNumOfProcessCreated = 2; // passing different platform var runConfigurationDictionary = new Dictionary @@ -136,7 +145,10 @@ public void RunSettingsWithoutParallelAndPlatformX86(RunnerInfo runnerInfo) // We pass 2 dlls in RunTestWithRunSettings, for .NET Framework they run in // 1 hosts because that host is Shared. - var expectedNumOfProcessCreated = runnerInfo.IsNetFrameworkTarget ? 1 : 2; + // + // Testhosts are no longer shared. We should always see 2. + // var expectedNumOfProcessCreated = runnerInfo.IsNetFrameworkTarget ? 1 : 2; + var expectedNumOfProcessCreated = 2; var runConfigurationDictionary = new Dictionary { @@ -160,7 +172,10 @@ public void RunSettingsParamsAsArguments(RunnerInfo runnerInfo) // We pass 2 dlls in RunTestWithRunSettings, for .NET Framework they run in // 1 hosts because that host is Shared. - var expectedNumOfProcessCreated = runnerInfo.IsNetFrameworkTarget ? 1 : 2; + // + // Testhosts are no longer shared, we should see 2 of them always. + // var expectedNumOfProcessCreated = runnerInfo.IsNetFrameworkTarget ? 1 : 2; + var expectedNumOfProcessCreated = 2; var runSettingsArgs = string.Join( " ", @@ -187,7 +202,10 @@ public void RunSettingsAndRunSettingsParamsAsArguments(RunnerInfo runnerInfo) // We pass 2 dlls in RunTestWithRunSettings, for .NET Framework they run in // 1 hosts because that host is Shared. - var expectedNumOfProcessCreated = runnerInfo.IsNetFrameworkTarget ? 1 : 2; + // + // Testhosts are no longer shared, we should see 2 of them always. + // var expectedNumOfProcessCreated = runnerInfo.IsNetFrameworkTarget ? 1 : 2; + var expectedNumOfProcessCreated = 2; var runConfigurationDictionary = new Dictionary { diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/CustomTestHostTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/CustomTestHostTests.cs index 474b569a44..b08964a690 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/CustomTestHostTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/CustomTestHostTests.cs @@ -142,6 +142,7 @@ public void RunTestsWithCustomTestHostLauncherUsesLaunchWhenGivenAnOutdatedITest [TestMethod] [TestCategory("Windows-Review")] [TestCategory("Feature")] + [Ignore("This is for debugger v3 and does not work yet.")] [RunnerCompatibilityDataSource(AfterFeature = Features.MULTI_TFM, JustRow = 1)] public void RunAllTestsWithMixedTFMsWillProvideAdditionalInformationToTheDebugger(RunnerInfo runnerInfo) { @@ -171,6 +172,7 @@ public void RunAllTestsWithMixedTFMsWillProvideAdditionalInformationToTheDebugge [TestMethod] [TestCategory("Windows-Review")] [TestCategory("BackwardCompatibilityWithRunner")] + [Ignore("This is for debugger v3 and does not work yet.")] [RunnerCompatibilityDataSource(BeforeFeature = Features.MULTI_TFM, JustRow = 1)] public void RunAllTestsWithMixedTFMsCallsBackToTestHostLauncherV3EvenWhenRunnerDoesNotSupportItYet(RunnerInfo runnerInfo) { diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/DiscoverTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/DiscoverTests.cs index 220ece0ba2..385cf931e9 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/DiscoverTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/DiscoverTests.cs @@ -124,8 +124,8 @@ public void DiscoverTestsUsingEventHandler2AndBatchSize(RunnerInfo runnerInfo) discoveryEventHandlerForBatchSize); // Assert. - Assert.AreEqual(6, discoveryEventHandlerForBatchSize.DiscoveredTestCases.Count); - Assert.AreEqual(3, discoveryEventHandlerForBatchSize.BatchSize); + Assert.AreEqual(6, discoveryEventHandlerForBatchSize.DiscoveredTestCases.Count, "discovered test cases"); + Assert.AreEqual(3, discoveryEventHandlerForBatchSize.BatchSize, "batch size"); } [TestMethod] diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/EventHandler/DiscoveryEventHandler.cs b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/EventHandler/DiscoveryEventHandler.cs index 4a9d890b41..102d47465c 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/EventHandler/DiscoveryEventHandler.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/EventHandler/DiscoveryEventHandler.cs @@ -1,6 +1,7 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT license. See LICENSE file in the project root for full license information. +using System; using System.Collections.Generic; using System.Linq; @@ -146,7 +147,10 @@ public void HandleRawMessage(string rawMessage) public void HandleLogMessage(TestMessageLevel level, string message) { - // No Op + if (level == TestMessageLevel.Error) + { + Console.WriteLine($"ERROR:{message}"); + }; } public void HandleDiscoveryComplete(DiscoveryCompleteEventArgs discoveryCompleteEventArgs, IEnumerable lastChunk) diff --git a/test/vstest.ProgrammerTests/Fakes/FakeCommunicationChannel.cs b/test/vstest.ProgrammerTests/Fakes/FakeCommunicationChannel.cs index 9774ed07eb..2f689fd722 100644 --- a/test/vstest.ProgrammerTests/Fakes/FakeCommunicationChannel.cs +++ b/test/vstest.ProgrammerTests/Fakes/FakeCommunicationChannel.cs @@ -78,7 +78,7 @@ internal class FakeCommunicationChannel : FakeCommunicationChannel, IC /// public Queue> NextResponses { get; } = new(); public FakeErrorAggregator FakeErrorAggregator { get; } - public FakeMessage? OutgoingMessage { get; private set; } + public FakeMessage? PendingMessage { get; private set; } public TContext? Context { get; private set; } public List> ProcessedMessages { get; } = new(); public Task? ProcessIncomingMessagesTask { get; private set; } @@ -105,9 +105,9 @@ private void ProcessOutgoingMessages() try { // TODO: better name for the property? This is message that we are currently trying to send. - OutgoingMessage = OutQueue.Take(token); - OnMessageReceived(this, new MessageReceivedEventArgs { Data = OutgoingMessage.SerializedMessage }); - OutgoingMessage = null; + PendingMessage = OutQueue.Take(token); + OnMessageReceived(this, new MessageReceivedEventArgs { Data = PendingMessage.SerializedMessage }); + PendingMessage = null; } catch (OperationCanceledException) { } catch (Exception ex) From d6d47a52cc9c75c26303d475de47105284ff8ad6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Jare=C5=A1?= Date: Tue, 17 May 2022 14:20:19 +0200 Subject: [PATCH 106/112] up versions for matrix --- scripts/build/TestPlatform.Dependencies.props | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/scripts/build/TestPlatform.Dependencies.props b/scripts/build/TestPlatform.Dependencies.props index c1c05c5f9b..0f78ef53b2 100644 --- a/scripts/build/TestPlatform.Dependencies.props +++ b/scripts/build/TestPlatform.Dependencies.props @@ -33,9 +33,9 @@ Exact versions are used to avoid Nuget substituting them by closest match, if we make a typo. These versions need to be "statically" readable because we read this file as xml in our build and tests. --> - [2.2.9-preview-20220210-07] - [2.2.8] - [2.2.7] + [2.2.10-preview-20220414-01] + [2.2.10] + [2.2.8] [2.1.0] [2.1.0] [1.4.0] @@ -45,9 +45,9 @@ See Invoke-TestAssetsBuild in scripts/build.ps1. Exact versions are used to avoid Nuget substituting them by closest match, if we make a typo. These versions need to be "statically" readable because we read this file as xml in our build and tests. --> - [17.2.0-preview-20220131-20] - [17.1.0] - [17.0.0] + [17.2.0-preview-20220401-08] + [17.2.0] + [17.1.0] [16.6.1] [16.11.0] [15.9.2] From 902e4359ab6e6e3ccc121667b778583f5b94064e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Jare=C5=A1?= Date: Tue, 17 May 2022 15:10:59 +0200 Subject: [PATCH 107/112] More PR feedbacks --- .../FeatureFlag/FeatureFlag.cs | 2 ++ src/Microsoft.TestPlatform.CrossPlatEngine/TestEngine.cs | 8 ++++++-- test/TestAssets/NuGet.config | 4 ---- 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/src/Microsoft.TestPlatform.CoreUtilities/FeatureFlag/FeatureFlag.cs b/src/Microsoft.TestPlatform.CoreUtilities/FeatureFlag/FeatureFlag.cs index 14cfceb4a4..26ab30b2c9 100644 --- a/src/Microsoft.TestPlatform.CoreUtilities/FeatureFlag/FeatureFlag.cs +++ b/src/Microsoft.TestPlatform.CoreUtilities/FeatureFlag/FeatureFlag.cs @@ -48,6 +48,8 @@ private FeatureFlag() { } // Faster JSON serialization relies on less internals of NewtonsoftJson, and on some additional caching. public const string DISABLE_FASTER_JSON_SERIALIZATION = VSTEST_ + nameof(DISABLE_FASTER_JSON_SERIALIZATION); + // Forces vstest.console to run all sources using the same target framework (TFM) and architecture, instead of allowing + // multiple different tfms and architectures to run at the same time. public const string DISABLE_MULTI_TFM_RUN = VSTEST_ + nameof(DISABLE_MULTI_TFM_RUN); [Obsolete("Only use this in tests.")] diff --git a/src/Microsoft.TestPlatform.CrossPlatEngine/TestEngine.cs b/src/Microsoft.TestPlatform.CrossPlatEngine/TestEngine.cs index 3e2d4ef16b..f86cae546d 100644 --- a/src/Microsoft.TestPlatform.CrossPlatEngine/TestEngine.cs +++ b/src/Microsoft.TestPlatform.CrossPlatEngine/TestEngine.cs @@ -89,11 +89,15 @@ public IProxyDiscoveryManager GetDiscoveryManager( if (ShouldRunInProcess(discoveryCriteria.RunSettings, isParallelRun, isDataCollectorEnabled: false, testHostManagers)) { // We are running in process, so whatever the architecture and framework that was figured out is, it must be compatible. If we have more - // changes that we want to to runsettings in the future, based on SourceDetail then it will depend on those details. But in general + // changes that we want to do to runsettings in the future, based on SourceDetail then it will depend on those details. But in general // we will have to check that all source details are the same. Otherwise we for sure cannot run in process. // E.g. if we get list of sources where one of them has different architecture we for sure cannot run in process, because the current // process can handle only single runsettings. - var testHostManagerInfo = testHostManagers.Single(); + if (testHostManagers.Count != 1) + { + throw new InvalidOperationException($"Exactly 1 testhost manager must be provided when running in process, but there {testHostManagers.Count} were provided."); + } + var testHostManagerInfo = testHostManagers[0]; testHostManager.Initialize(TestSessionMessageLogger.Instance, testHostManagerInfo.RunSettings); var isTelemetryOptedIn = requestData.IsTelemetryOptedIn; diff --git a/test/TestAssets/NuGet.config b/test/TestAssets/NuGet.config index 78c8ceebcb..d4a9cb8272 100644 --- a/test/TestAssets/NuGet.config +++ b/test/TestAssets/NuGet.config @@ -5,10 +5,6 @@ - - - - From 5e2f9b09db9411c4b6a0870adf7c796196a432d7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Jare=C5=A1?= Date: Tue, 17 May 2022 16:50:11 +0200 Subject: [PATCH 108/112] up internal version and remove debug attribute --- test/Microsoft.TestPlatform.AcceptanceTests/ExecutionTests.cs | 2 +- .../Extension/Features.cs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/ExecutionTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/ExecutionTests.cs index ea9294d8eb..9575467707 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/ExecutionTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/ExecutionTests.cs @@ -112,7 +112,7 @@ public void RunMultipleTestAssembliesWithoutTestAdapterPath(RunnerInfo runnerInf // and after --arch feature implementation we won't find correct muxer on CI. [TestCategory("Windows")] [TestMethod] - [MSTestCompatibilityDataSource(DebugVSTestConsole = true)] + [MSTestCompatibilityDataSource)] public void RunMultipleTestAssembliesInParallel(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/Features.cs b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/Features.cs index f235879f46..bbfb7104e3 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/Features.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/Features.cs @@ -15,7 +15,7 @@ public static class Features public static IImmutableDictionary TestPlatformFeatures { get; } = new Dictionary { [ATTACH_DEBUGGER_FLOW] = new(version: "v16.7.0-preview-20200519-01", issue: "https://github.com/microsoft/vstest/pull/2325"), - [MULTI_TFM] = new(version: "17.2.0-dev", issue: "https://github.com/microsoft/vstest/pull/3412") + [MULTI_TFM] = new(version: "17.3.0-dev", issue: "https://github.com/microsoft/vstest/pull/3412") }.ToImmutableDictionary(); public static IImmutableDictionary AdapterFeatures { get; internal set; } = new Dictionary From 287b4e651d61683dcc1f61c88c41dc0121f62716 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Jare=C5=A1?= Date: Tue, 17 May 2022 16:57:38 +0200 Subject: [PATCH 109/112] yah for the extra parent --- test/Microsoft.TestPlatform.AcceptanceTests/ExecutionTests.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/ExecutionTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/ExecutionTests.cs index 9575467707..a83129b78d 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/ExecutionTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/ExecutionTests.cs @@ -112,7 +112,7 @@ public void RunMultipleTestAssembliesWithoutTestAdapterPath(RunnerInfo runnerInf // and after --arch feature implementation we won't find correct muxer on CI. [TestCategory("Windows")] [TestMethod] - [MSTestCompatibilityDataSource)] + [MSTestCompatibilityDataSource] public void RunMultipleTestAssembliesInParallel(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); From 7dcf780813f23b64b1a3478b066204c97bcc7012 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Jare=C5=A1?= Date: Wed, 18 May 2022 18:25:17 +0200 Subject: [PATCH 110/112] Fix testhost startup performance and choosing x86 for mixed runs. --- .../Parallel/ParallelOperationManager.cs | 2 +- .../Hosting/DefaultTestHostManager.cs | 7 +++++-- .../Hosting/DotnetTestHostManager.cs | 8 ++++++-- .../TestPlatformHelpers/TestRequestManager.cs | 19 +++++++++++++++---- 4 files changed, 27 insertions(+), 9 deletions(-) diff --git a/src/Microsoft.TestPlatform.CrossPlatEngine/Client/Parallel/ParallelOperationManager.cs b/src/Microsoft.TestPlatform.CrossPlatEngine/Client/Parallel/ParallelOperationManager.cs index c8c5228c49..a1723e98d2 100644 --- a/src/Microsoft.TestPlatform.CrossPlatEngine/Client/Parallel/ParallelOperationManager.cs +++ b/src/Microsoft.TestPlatform.CrossPlatEngine/Client/Parallel/ParallelOperationManager.cs @@ -83,7 +83,7 @@ public void StartWork( RunWorkInParallel(); } - // This does not do anything in parallel, all the workloads we schedule are offloaded to separate Task in the callback. + // This does not do anything in parallel, all the workloads we schedule are offloaded to separate Task in the _runWorkload callback. // I did not want to change that, yet but this is the correct place to do that offloading. Not each manager. private bool RunWorkInParallel() { diff --git a/src/Microsoft.TestPlatform.TestHostProvider/Hosting/DefaultTestHostManager.cs b/src/Microsoft.TestPlatform.TestHostProvider/Hosting/DefaultTestHostManager.cs index a522c7c150..be1a4c1ac3 100644 --- a/src/Microsoft.TestPlatform.TestHostProvider/Hosting/DefaultTestHostManager.cs +++ b/src/Microsoft.TestPlatform.TestHostProvider/Hosting/DefaultTestHostManager.cs @@ -124,9 +124,12 @@ public TestHostConnectionInfo GetTestHostConnectionInfo() } /// - public async Task LaunchTestHostAsync(TestProcessStartInfo testHostStartInfo, CancellationToken cancellationToken) + public Task LaunchTestHostAsync(TestProcessStartInfo testHostStartInfo, CancellationToken cancellationToken) { - return await Task.Run(() => LaunchHost(testHostStartInfo, cancellationToken), cancellationToken); + // Do NOT offload this to thread pool using Task.Run, we already are on thread pool + // and this would go into a queue after all the other startup tasks. Meaning we will start + // testhost much later, and not immediately. + return Task.FromResult(LaunchHost(testHostStartInfo, cancellationToken)); } /// diff --git a/src/Microsoft.TestPlatform.TestHostProvider/Hosting/DotnetTestHostManager.cs b/src/Microsoft.TestPlatform.TestHostProvider/Hosting/DotnetTestHostManager.cs index fc5ff062ca..5700e51a7e 100644 --- a/src/Microsoft.TestPlatform.TestHostProvider/Hosting/DotnetTestHostManager.cs +++ b/src/Microsoft.TestPlatform.TestHostProvider/Hosting/DotnetTestHostManager.cs @@ -193,9 +193,12 @@ public TestHostConnectionInfo GetTestHostConnectionInfo() } /// - public async Task LaunchTestHostAsync(TestProcessStartInfo testHostStartInfo, CancellationToken cancellationToken) + public Task LaunchTestHostAsync(TestProcessStartInfo testHostStartInfo, CancellationToken cancellationToken) { - return await Task.Run(() => LaunchHost(testHostStartInfo, cancellationToken), cancellationToken); + // Do NOT offload this to thread pool using Task.Run, we already are on thread pool + // and this would go into a queue after all the other startup tasks. Meaning we will start + // testhost much later, and not immediately. + return Task.FromResult(LaunchHost(testHostStartInfo, cancellationToken)); } /// @@ -664,6 +667,7 @@ private bool LaunchHost(TestProcessStartInfo testHostStartInfo, CancellationToke EqtTrace.Verbose("DotnetTestHostManager: Starting process '{0}' with command line '{1}'", testHostStartInfo.FileName, testHostStartInfo.Arguments); cancellationToken.ThrowIfCancellationRequested(); + _testHostProcess = _processHelper.LaunchProcess( testHostStartInfo.FileName, testHostStartInfo.Arguments, diff --git a/src/vstest.console/TestPlatformHelpers/TestRequestManager.cs b/src/vstest.console/TestPlatformHelpers/TestRequestManager.cs index 0d70ce6dee..09565da14a 100644 --- a/src/vstest.console/TestPlatformHelpers/TestRequestManager.cs +++ b/src/vstest.console/TestPlatformHelpers/TestRequestManager.cs @@ -687,10 +687,22 @@ private bool UpdateRunSettingsIfRequired( settingsUpdated |= frameworkWasAutodetected; var frameworkSetByRunsettings = !frameworkWasAutodetected; + // Before MULTI_TFM feature the sourceToArchitectureMap and sourceToFrameworkMap were only used as informational + // to be able to do this compatibility check and print warning. And in the later steps only chosenPlatform, chosenFramework + // were used, that represented the single architecture and framework to be used. + // + // After MULTI_TFM sourceToArchitectureMap and sourceToFrameworkMap are the source of truth, and are propagated forward, + // so when we want to revert to the older behavior we need to re-enable the check, and unify all the architecture and + // framework entries to the same chosen value. + var disableMultiTfm = FeatureFlag.Instance.IsSet(FeatureFlag.DISABLE_MULTI_TFM_RUN); + // Choose default architecture based on the framework. - // For .NET core, the default platform architecture should be based on the process. + // For a run with mixed tfms enabled, or .NET "Core", the default platform architecture should be based on the process. + // This will choose x64 by default for both .NET and .NET Framework, and avoids choosing x86 for a mixed + // run, so we will run via .NET testhost.exe, and not via dotnet testhost.dll. Architecture defaultArchitecture = Architecture.X86; - if (chosenFramework.Name.IndexOf("netstandard", StringComparison.OrdinalIgnoreCase) >= 0 + if (!disableMultiTfm + || chosenFramework.Name.IndexOf("netstandard", StringComparison.OrdinalIgnoreCase) >= 0 || chosenFramework.Name.IndexOf("netcoreapp", StringComparison.OrdinalIgnoreCase) >= 0 // This is a special case for 1 version of Nuget.Frameworks that was shipped with using identifier NET5 instead of NETCoreApp5 for .NET 5. || chosenFramework.Name.IndexOf("net5", StringComparison.OrdinalIgnoreCase) >= 0) @@ -722,7 +734,7 @@ private bool UpdateRunSettingsIfRequired( defaultArchitecture = GetDefaultArchitecture(runConfiguration); } - // For all other scenario we keep the old default Architecture.X86. + // For all other scenarios we keep the old default Architecture.X86. } EqtTrace.Verbose($"TestRequestManager.UpdateRunSettingsIfRequired: Default architecture: {defaultArchitecture} IsDefaultTargetArchitecture: {RunSettingsHelper.Instance.IsDefaultTargetArchitecture}, Current process architecture: {_processHelper.GetCurrentProcessArchitecture()} OperatingSystem: {_environment.OperatingSystem}."); @@ -748,7 +760,6 @@ private bool UpdateRunSettingsIfRequired( // After MULTI_TFM sourceToArchitectureMap and sourceToFrameworkMap are the source of truth, and are propagated forward, // so when we want to revert to the older behavior we need to re-enable the check, and unify all the architecture and // framework entries to the same chosen value. - var disableMultiTfm = FeatureFlag.Instance.IsSet(FeatureFlag.DISABLE_MULTI_TFM_RUN); // Do the check only when we enable MULTI_TFM and platform or framework are forced by settings, because then we maybe have some sources // that are not compatible with the chosen settings. And do the check always when MULTI_TFM is disabled, because then we want to warn every From da0cbdf6017e8160e14cc5bf7f669cb17c95bd9a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Jare=C5=A1?= Date: Thu, 19 May 2022 09:15:46 +0200 Subject: [PATCH 111/112] Clean up HandlePartialRunComplete, and fix unit tests --- .../Parallel/ParallelProxyExecutionManager.cs | 21 ++++++++++++------- .../Hosting/DefaultTestHostManagerTests.cs | 2 +- .../Hosting/DotnetTestHostManagerTests.cs | 2 +- 3 files changed, 16 insertions(+), 9 deletions(-) diff --git a/src/Microsoft.TestPlatform.CrossPlatEngine/Client/Parallel/ParallelProxyExecutionManager.cs b/src/Microsoft.TestPlatform.CrossPlatEngine/Client/Parallel/ParallelProxyExecutionManager.cs index e78b86ac29..682f8ace01 100644 --- a/src/Microsoft.TestPlatform.CrossPlatEngine/Client/Parallel/ParallelProxyExecutionManager.cs +++ b/src/Microsoft.TestPlatform.CrossPlatEngine/Client/Parallel/ParallelProxyExecutionManager.cs @@ -159,13 +159,20 @@ public bool HandlePartialRunComplete( // and queue another test run. if (!testRunCompleteArgs.IsCanceled && !_abortRequested) { - _parallelOperationManager.RunNextWork(proxyExecutionManager); - //var hadMoreWork = _parallelOperationManager.RunNextWork(proxyExecutionManager); - // ugh huh????! don't return true, or else stuff will fail! - //if (!hadMoreWork) - //{ - // return true; - //} + // Do NOT return true here, there should be only one place where this method returns true, + // and cancellation or success or any other other combination or timing should result in only one true. + // This is largely achieved by returning true above when "allRunsCompleted" is true. That variable is true + // when we cancel all sources or when we complete all sources. + // + // But we can also start a source, and cancel right after, which will remove all managers, and RunNextWork returns + // false, because we had no more work to do. If we check that result here and return true, then the whole logic is + // broken and we end up calling RunComplete handlers twice and writing logger output to screen twice. So don't do it. + // var hadMoreWork = _parallelOperationManager.RunNextWork(proxyExecutionManager); + // if (!hadMoreWork) + // { + // return true; + // } + var _ = _parallelOperationManager.RunNextWork(proxyExecutionManager); } return false; diff --git a/test/Microsoft.TestPlatform.TestHostProvider.UnitTests/Hosting/DefaultTestHostManagerTests.cs b/test/Microsoft.TestPlatform.TestHostProvider.UnitTests/Hosting/DefaultTestHostManagerTests.cs index e3d4bb198e..ca482cc9bf 100644 --- a/test/Microsoft.TestPlatform.TestHostProvider.UnitTests/Hosting/DefaultTestHostManagerTests.cs +++ b/test/Microsoft.TestPlatform.TestHostProvider.UnitTests/Hosting/DefaultTestHostManagerTests.cs @@ -354,7 +354,7 @@ public void LaunchTestHostAsyncShouldNotStartHostProcessIfCancellationTokenIsSet CancellationTokenSource cancellationTokenSource = new(); cancellationTokenSource.Cancel(); - Assert.ThrowsException(() => _testableTestHostManager.LaunchTestHostAsync(GetDefaultStartInfo(), cancellationTokenSource.Token).Wait()); + Assert.ThrowsException(() => _testableTestHostManager.LaunchTestHostAsync(GetDefaultStartInfo(), cancellationTokenSource.Token).Wait()); } [TestMethod] diff --git a/test/Microsoft.TestPlatform.TestHostProvider.UnitTests/Hosting/DotnetTestHostManagerTests.cs b/test/Microsoft.TestPlatform.TestHostProvider.UnitTests/Hosting/DotnetTestHostManagerTests.cs index 289e0799e4..581c26ad93 100644 --- a/test/Microsoft.TestPlatform.TestHostProvider.UnitTests/Hosting/DotnetTestHostManagerTests.cs +++ b/test/Microsoft.TestPlatform.TestHostProvider.UnitTests/Hosting/DotnetTestHostManagerTests.cs @@ -456,7 +456,7 @@ public void LaunchTestHostAsyncShouldNotStartHostProcessIfCancellationTokenIsSet CancellationTokenSource cancellationTokenSource = new(); cancellationTokenSource.Cancel(); - Assert.ThrowsException(() => _dotnetHostManager.LaunchTestHostAsync(startInfo, cancellationTokenSource.Token).Wait()); + Assert.ThrowsException(() => _dotnetHostManager.LaunchTestHostAsync(startInfo, cancellationTokenSource.Token).Wait()); } [TestMethod] From 6e49ebe32f80ce94945ef19161496892fc4e641b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Jare=C5=A1?= Date: Thu, 19 May 2022 15:28:49 +0200 Subject: [PATCH 112/112] Fix acceptance tests --- .../ExecutionTests.cs | 2 +- .../FrameworkTests.cs | 2 +- .../TranslationLayerTests/DiscoverTests.cs | 26 ++++++++++++------- .../EventHandler/DiscoveryEventHandler.cs | 4 +-- 4 files changed, 20 insertions(+), 14 deletions(-) diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/ExecutionTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/ExecutionTests.cs index a83129b78d..01cbe7a8c2 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/ExecutionTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/ExecutionTests.cs @@ -259,7 +259,7 @@ public void IncompatibleSourcesWarningShouldBeDisplayedInTheConsoleWhenGivenInco { SetTestEnvironment(_testEnvironment, runnerInfo); - var expectedWarningContains = @"Following DLL(s) do not match current settings, which are .NETFramework,Version=v4.5.1 framework and X86 platform. SimpleTestProject3.dll would use Framework .NETFramework,Version=v4.5.1 and Platform X64"; + var expectedWarningContains = @"Following DLL(s) do not match current settings, which are .NETFramework,Version=v4.5.1 framework and X64 platform. SimpleTestProjectx86.dll would use Framework .NETFramework,Version=v4.5.1 and Platform X86"; var assemblyPaths = BuildMultipleAssemblyPath("SimpleTestProject3.dll", "SimpleTestProjectx86.dll"); var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: TempDirectory.Path); diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/FrameworkTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/FrameworkTests.cs index 394db87a80..452c60510b 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/FrameworkTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/FrameworkTests.cs @@ -92,7 +92,7 @@ public void RunSpecificTestsShouldWorkWithFrameworkInCompatibleWarning(RunnerInf } else { - StdOutputContains("Following DLL(s) do not match current settings, which are .NETFramework,Version=v4.0 framework and X86 platform."); + StdOutputContains("Following DLL(s) do not match current settings, which are .NETFramework,Version=v4.0 framework and X64 platform."); ValidateSummaryStatus(1, 0, 0); } } diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/DiscoverTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/DiscoverTests.cs index 385cf931e9..3364c3dc9f 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/DiscoverTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/DiscoverTests.cs @@ -7,6 +7,8 @@ using System.Linq; using System.Threading.Tasks; +using FluentAssertions; + using Microsoft.TestPlatform.TestUtilities; using Microsoft.TestPlatform.VsTestConsole.TranslationLayer.Interfaces; using Microsoft.VisualStudio.TestPlatform.Common.Telemetry; @@ -109,11 +111,11 @@ public void DiscoverTestsUsingEventHandler2AndBatchSize(RunnerInfo runnerInfo) Setup(); var discoveryEventHandlerForBatchSize = new DiscoveryEventHandlerForBatchSize(); - - string runSettingsXml = @" + var batchSize = 2; + string runSettingsXml = $@" - 3 + {batchSize} "; @@ -124,8 +126,10 @@ public void DiscoverTestsUsingEventHandler2AndBatchSize(RunnerInfo runnerInfo) discoveryEventHandlerForBatchSize); // Assert. - Assert.AreEqual(6, discoveryEventHandlerForBatchSize.DiscoveredTestCases.Count, "discovered test cases"); - Assert.AreEqual(3, discoveryEventHandlerForBatchSize.BatchSize, "batch size"); + discoveryEventHandlerForBatchSize.DiscoveredTestCases.Should().HaveCount(6, "we found 6 tests in total"); + // Batching happens based on size and time interva. The middle batch should almost always be 2, + // if the discovery is fast enough, but the only requirement we can reliably check and enforce is that no batch is bigger than the expected size. + discoveryEventHandlerForBatchSize.Batches.Should().OnlyContain(v => v <= batchSize, "all batches should be the same size or smaller than the batch size"); } [TestMethod] @@ -137,11 +141,11 @@ public void DiscoverTestsUsingEventHandler1AndBatchSize(RunnerInfo runnerInfo) Setup(); var discoveryEventHandlerForBatchSize = new DiscoveryEventHandlerForBatchSize(); - - string runSettingsXml = @" + var batchSize = 2; + string runSettingsXml = $@" - 3 + {batchSize} "; @@ -151,8 +155,10 @@ public void DiscoverTestsUsingEventHandler1AndBatchSize(RunnerInfo runnerInfo) discoveryEventHandlerForBatchSize); // Assert. - Assert.AreEqual(6, discoveryEventHandlerForBatchSize.DiscoveredTestCases.Count); - Assert.AreEqual(3, discoveryEventHandlerForBatchSize.BatchSize); + discoveryEventHandlerForBatchSize.DiscoveredTestCases.Should().HaveCount(6, "we found 6 tests in total"); + // Batching happens based on size and time interva. The middle batch should almost always be 2, + // if the discovery is fast enough, but the only requirement we can reliably check and enforce is that no batch is bigger than the expected size. + discoveryEventHandlerForBatchSize.Batches.Should().OnlyContain(v => v <= batchSize, "all batches should be the same size or smaller than the batch size"); } [TestMethod] diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/EventHandler/DiscoveryEventHandler.cs b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/EventHandler/DiscoveryEventHandler.cs index 102d47465c..5f7b825de2 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/EventHandler/DiscoveryEventHandler.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/EventHandler/DiscoveryEventHandler.cs @@ -128,7 +128,7 @@ public class DiscoveryEventHandlerForBatchSize : ITestDiscoveryEventsHandler2, I /// /// Gets the batch size. /// - public long BatchSize { get; private set; } + public List Batches { get; } = new List(); /// /// Gets the discovered test cases. @@ -174,7 +174,7 @@ public void HandleDiscoveredTests(IEnumerable discoveredTestCases) if (discoveredTestCases != null && discoveredTestCases.Any()) { DiscoveredTestCases.AddRange(discoveredTestCases); - BatchSize = discoveredTestCases.Count(); + Batches.Add(discoveredTestCases.Count()); } } }