From cf19aa2e7a4acd19a4db1becc12f9e203e69b31d Mon Sep 17 00:00:00 2001 From: nohwnd Date: Mon, 7 Feb 2022 10:11:20 +0100 Subject: [PATCH 01/32] use fakes over mocks --- TestPlatform.sln | 45 ++++ scripts/build/TestPlatform.Settings.targets | 2 +- src/Microsoft.TestPlatform.Client/Friends.cs | 1 + .../TestPlatform.cs | 6 +- src/Microsoft.TestPlatform.Common/Friends.cs | 2 + .../Hosting/ITestRuntimeProviderManager.cs | 11 + .../Hosting/TestRunTimeProviderManager.cs | 2 +- .../Friends.cs | 1 + .../TestEngine.cs | 6 +- .../CommandLine/TestRunResultAggregator.cs | 2 +- src/vstest.console/Friends.cs | 1 + test/Intent.Primitives/ExcludeAttribute.cs | 9 + test/Intent.Primitives/IRunLogger.cs | 14 ++ .../Intent.Primitives.csproj | 9 + test/Intent.Primitives/TestResult.cs | 12 + test/Intent/ConsoleLogger.cs | 44 ++++ test/Intent/Extensions.cs | 36 +++ test/Intent/Intent.csproj | 14 ++ test/Intent/Program.cs | 12 + test/Intent/Runner.cs | 52 ++++ test/vstest.ProgrammerTests/Program.cs | 13 + .../Properties/launchSettings.json | 8 + test/vstest.ProgrammerTests/UnitTest1.cs | 238 ++++++++++++++++++ test/vstest.ProgrammerTests/VstestConsole.cs | 47 ++++ .../vstest.ProgrammerTests.csproj | 39 +++ 25 files changed, 617 insertions(+), 9 deletions(-) create mode 100644 src/Microsoft.TestPlatform.Common/Hosting/ITestRuntimeProviderManager.cs create mode 100644 test/Intent.Primitives/ExcludeAttribute.cs create mode 100644 test/Intent.Primitives/IRunLogger.cs create mode 100644 test/Intent.Primitives/Intent.Primitives.csproj create mode 100644 test/Intent.Primitives/TestResult.cs create mode 100644 test/Intent/ConsoleLogger.cs create mode 100644 test/Intent/Extensions.cs create mode 100644 test/Intent/Intent.csproj create mode 100644 test/Intent/Program.cs create mode 100644 test/Intent/Runner.cs create mode 100644 test/vstest.ProgrammerTests/Program.cs create mode 100644 test/vstest.ProgrammerTests/Properties/launchSettings.json create mode 100644 test/vstest.ProgrammerTests/UnitTest1.cs create mode 100644 test/vstest.ProgrammerTests/VstestConsole.cs create mode 100644 test/vstest.ProgrammerTests/vstest.ProgrammerTests.csproj diff --git a/TestPlatform.sln b/TestPlatform.sln index 3701df2db4..0a1d3c2fbc 100644 --- a/TestPlatform.sln +++ b/TestPlatform.sln @@ -179,6 +179,12 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "MSTest1", "playground\MSTes EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "AttachmentProcessorDataCollector", "test\TestAssets\AttachmentProcessorDataCollector\AttachmentProcessorDataCollector.csproj", "{B6AF6BCD-64C6-4F4E-ABCA-C8AA2AA66B7B}" EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "vstest.ProgrammerTests", "test\vstest.ProgrammerTests\vstest.ProgrammerTests.csproj", "{B1F84FD8-6150-4ECA-9AD7-C316E04E17D8}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Intent", "test\Intent\Intent.csproj", "{BFBB35C9-6437-480A-8DCC-AE3700110E7D}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Intent.Primitives", "test\Intent.Primitives\Intent.Primitives.csproj", "{29270853-90DC-4C39-9621-F47AE40A79B6}" +EndProject Global GlobalSection(SharedMSBuildProjectFiles) = preSolution src\Microsoft.TestPlatform.Execution.Shared\Microsoft.TestPlatform.Execution.Shared.projitems*{10b6ade1-f808-4612-801d-4452f5b52242}*SharedItemsImports = 5 @@ -880,6 +886,42 @@ Global {B6AF6BCD-64C6-4F4E-ABCA-C8AA2AA66B7B}.Release|x64.Build.0 = Release|Any CPU {B6AF6BCD-64C6-4F4E-ABCA-C8AA2AA66B7B}.Release|x86.ActiveCfg = Release|Any CPU {B6AF6BCD-64C6-4F4E-ABCA-C8AA2AA66B7B}.Release|x86.Build.0 = Release|Any CPU + {B1F84FD8-6150-4ECA-9AD7-C316E04E17D8}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {B1F84FD8-6150-4ECA-9AD7-C316E04E17D8}.Debug|Any CPU.Build.0 = Debug|Any CPU + {B1F84FD8-6150-4ECA-9AD7-C316E04E17D8}.Debug|x64.ActiveCfg = Debug|Any CPU + {B1F84FD8-6150-4ECA-9AD7-C316E04E17D8}.Debug|x64.Build.0 = Debug|Any CPU + {B1F84FD8-6150-4ECA-9AD7-C316E04E17D8}.Debug|x86.ActiveCfg = Debug|Any CPU + {B1F84FD8-6150-4ECA-9AD7-C316E04E17D8}.Debug|x86.Build.0 = Debug|Any CPU + {B1F84FD8-6150-4ECA-9AD7-C316E04E17D8}.Release|Any CPU.ActiveCfg = Release|Any CPU + {B1F84FD8-6150-4ECA-9AD7-C316E04E17D8}.Release|Any CPU.Build.0 = Release|Any CPU + {B1F84FD8-6150-4ECA-9AD7-C316E04E17D8}.Release|x64.ActiveCfg = Release|Any CPU + {B1F84FD8-6150-4ECA-9AD7-C316E04E17D8}.Release|x64.Build.0 = Release|Any CPU + {B1F84FD8-6150-4ECA-9AD7-C316E04E17D8}.Release|x86.ActiveCfg = Release|Any CPU + {B1F84FD8-6150-4ECA-9AD7-C316E04E17D8}.Release|x86.Build.0 = Release|Any CPU + {BFBB35C9-6437-480A-8DCC-AE3700110E7D}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {BFBB35C9-6437-480A-8DCC-AE3700110E7D}.Debug|Any CPU.Build.0 = Debug|Any CPU + {BFBB35C9-6437-480A-8DCC-AE3700110E7D}.Debug|x64.ActiveCfg = Debug|Any CPU + {BFBB35C9-6437-480A-8DCC-AE3700110E7D}.Debug|x64.Build.0 = Debug|Any CPU + {BFBB35C9-6437-480A-8DCC-AE3700110E7D}.Debug|x86.ActiveCfg = Debug|Any CPU + {BFBB35C9-6437-480A-8DCC-AE3700110E7D}.Debug|x86.Build.0 = Debug|Any CPU + {BFBB35C9-6437-480A-8DCC-AE3700110E7D}.Release|Any CPU.ActiveCfg = Release|Any CPU + {BFBB35C9-6437-480A-8DCC-AE3700110E7D}.Release|Any CPU.Build.0 = Release|Any CPU + {BFBB35C9-6437-480A-8DCC-AE3700110E7D}.Release|x64.ActiveCfg = Release|Any CPU + {BFBB35C9-6437-480A-8DCC-AE3700110E7D}.Release|x64.Build.0 = Release|Any CPU + {BFBB35C9-6437-480A-8DCC-AE3700110E7D}.Release|x86.ActiveCfg = Release|Any CPU + {BFBB35C9-6437-480A-8DCC-AE3700110E7D}.Release|x86.Build.0 = Release|Any CPU + {29270853-90DC-4C39-9621-F47AE40A79B6}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {29270853-90DC-4C39-9621-F47AE40A79B6}.Debug|Any CPU.Build.0 = Debug|Any CPU + {29270853-90DC-4C39-9621-F47AE40A79B6}.Debug|x64.ActiveCfg = Debug|Any CPU + {29270853-90DC-4C39-9621-F47AE40A79B6}.Debug|x64.Build.0 = Debug|Any CPU + {29270853-90DC-4C39-9621-F47AE40A79B6}.Debug|x86.ActiveCfg = Debug|Any CPU + {29270853-90DC-4C39-9621-F47AE40A79B6}.Debug|x86.Build.0 = Debug|Any CPU + {29270853-90DC-4C39-9621-F47AE40A79B6}.Release|Any CPU.ActiveCfg = Release|Any CPU + {29270853-90DC-4C39-9621-F47AE40A79B6}.Release|Any CPU.Build.0 = Release|Any CPU + {29270853-90DC-4C39-9621-F47AE40A79B6}.Release|x64.ActiveCfg = Release|Any CPU + {29270853-90DC-4C39-9621-F47AE40A79B6}.Release|x64.Build.0 = Release|Any CPU + {29270853-90DC-4C39-9621-F47AE40A79B6}.Release|x86.ActiveCfg = Release|Any CPU + {29270853-90DC-4C39-9621-F47AE40A79B6}.Release|x86.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -955,6 +997,9 @@ Global {545A88D3-1AE2-4D39-9B7C-C691768AD17F} = {6CE2F530-582B-4695-A209-41065E103426} {57A61A09-10AD-44BE-8DF4-A6FD108F7DF7} = {6CE2F530-582B-4695-A209-41065E103426} {B6AF6BCD-64C6-4F4E-ABCA-C8AA2AA66B7B} = {D9A30E32-D466-4EC5-B4F2-62E17562279B} + {B1F84FD8-6150-4ECA-9AD7-C316E04E17D8} = {B27FAFDF-2DBA-4AB0-BA85-FD5F21D359D6} + {BFBB35C9-6437-480A-8DCC-AE3700110E7D} = {B27FAFDF-2DBA-4AB0-BA85-FD5F21D359D6} + {29270853-90DC-4C39-9621-F47AE40A79B6} = {B27FAFDF-2DBA-4AB0-BA85-FD5F21D359D6} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {0541B30C-FF51-4E28-B172-83F5F3934BCD} diff --git a/scripts/build/TestPlatform.Settings.targets b/scripts/build/TestPlatform.Settings.targets index 2539dc660b..34c8262b47 100644 --- a/scripts/build/TestPlatform.Settings.targets +++ b/scripts/build/TestPlatform.Settings.targets @@ -20,7 +20,7 @@ serialization. This is also defined in build script. --> 15.0.0 - true + false true true diff --git a/src/Microsoft.TestPlatform.Client/Friends.cs b/src/Microsoft.TestPlatform.Client/Friends.cs index 90a5d9db6f..81da967ae4 100644 --- a/src/Microsoft.TestPlatform.Client/Friends.cs +++ b/src/Microsoft.TestPlatform.Client/Friends.cs @@ -10,5 +10,6 @@ #region Test Assemblies [assembly: InternalsVisibleTo("Microsoft.TestPlatform.Client.UnitTests, PublicKey=002400000480000094000000060200000024000052534131000400000100010007d1fa57c4aed9f0a32e84aa0faefd0de9e8fd6aec8f87fb03766c834c99921eb23be79ad9d5dcc1dd9ad236132102900b723cf980957fc4e177108fc607774f29e8320e92ea05ece4e821c0a5efe8f1645c4c0c93c1ab99285d622caa652c1dfad63d745d6f2de5f17e5eaf0fc4963d261c8a12436518206dc093344d5ad293")] +[assembly: InternalsVisibleTo("vstest.ProgrammerTests, PublicKey=002400000480000094000000060200000024000052534131000400000100010007d1fa57c4aed9f0a32e84aa0faefd0de9e8fd6aec8f87fb03766c834c99921eb23be79ad9d5dcc1dd9ad236132102900b723cf980957fc4e177108fc607774f29e8320e92ea05ece4e821c0a5efe8f1645c4c0c93c1ab99285d622caa652c1dfad63d745d6f2de5f17e5eaf0fc4963d261c8a12436518206dc093344d5ad293")] #endregion diff --git a/src/Microsoft.TestPlatform.Client/TestPlatform.cs b/src/Microsoft.TestPlatform.Client/TestPlatform.cs index 1065759bd7..43b849639f 100644 --- a/src/Microsoft.TestPlatform.Client/TestPlatform.cs +++ b/src/Microsoft.TestPlatform.Client/TestPlatform.cs @@ -34,7 +34,7 @@ namespace Microsoft.VisualStudio.TestPlatform.Client; /// internal class TestPlatform : ITestPlatform { - private readonly TestRuntimeProviderManager _testHostProviderManager; + private readonly ITestRuntimeProviderManager _testHostProviderManager; private readonly IFileHelper _fileHelper; @@ -64,10 +64,10 @@ public TestPlatform() /// The test engine. /// The file helper. /// The data. - protected TestPlatform( + internal TestPlatform( ITestEngine testEngine, IFileHelper filehelper, - TestRuntimeProviderManager testHostProviderManager) + ITestRuntimeProviderManager testHostProviderManager) { TestEngine = testEngine; _fileHelper = filehelper; diff --git a/src/Microsoft.TestPlatform.Common/Friends.cs b/src/Microsoft.TestPlatform.Common/Friends.cs index ea3f0a478d..23875a52a0 100644 --- a/src/Microsoft.TestPlatform.Common/Friends.cs +++ b/src/Microsoft.TestPlatform.Common/Friends.cs @@ -25,4 +25,6 @@ [assembly: InternalsVisibleTo("Microsoft.TestPlatform.Extensions.BlameDataCollector.UnitTests, PublicKey=002400000480000094000000060200000024000052534131000400000100010007d1fa57c4aed9f0a32e84aa0faefd0de9e8fd6aec8f87fb03766c834c99921eb23be79ad9d5dcc1dd9ad236132102900b723cf980957fc4e177108fc607774f29e8320e92ea05ece4e821c0a5efe8f1645c4c0c93c1ab99285d622caa652c1dfad63d745d6f2de5f17e5eaf0fc4963d261c8a12436518206dc093344d5ad293")] [assembly: InternalsVisibleTo("Microsoft.TestPlatform.TestUtilities, PublicKey=002400000480000094000000060200000024000052534131000400000100010007d1fa57c4aed9f0a32e84aa0faefd0de9e8fd6aec8f87fb03766c834c99921eb23be79ad9d5dcc1dd9ad236132102900b723cf980957fc4e177108fc607774f29e8320e92ea05ece4e821c0a5efe8f1645c4c0c93c1ab99285d622caa652c1dfad63d745d6f2de5f17e5eaf0fc4963d261c8a12436518206dc093344d5ad293")] [assembly: InternalsVisibleTo("Microsoft.TestPlatform.AcceptanceTests, PublicKey=002400000480000094000000060200000024000052534131000400000100010007d1fa57c4aed9f0a32e84aa0faefd0de9e8fd6aec8f87fb03766c834c99921eb23be79ad9d5dcc1dd9ad236132102900b723cf980957fc4e177108fc607774f29e8320e92ea05ece4e821c0a5efe8f1645c4c0c93c1ab99285d622caa652c1dfad63d745d6f2de5f17e5eaf0fc4963d261c8a12436518206dc093344d5ad293")] +[assembly: InternalsVisibleTo("vstest.ProgrammerTests, PublicKey=002400000480000094000000060200000024000052534131000400000100010007d1fa57c4aed9f0a32e84aa0faefd0de9e8fd6aec8f87fb03766c834c99921eb23be79ad9d5dcc1dd9ad236132102900b723cf980957fc4e177108fc607774f29e8320e92ea05ece4e821c0a5efe8f1645c4c0c93c1ab99285d622caa652c1dfad63d745d6f2de5f17e5eaf0fc4963d261c8a12436518206dc093344d5ad293")] + #endregion diff --git a/src/Microsoft.TestPlatform.Common/Hosting/ITestRuntimeProviderManager.cs b/src/Microsoft.TestPlatform.Common/Hosting/ITestRuntimeProviderManager.cs new file mode 100644 index 0000000000..023edd18b8 --- /dev/null +++ b/src/Microsoft.TestPlatform.Common/Hosting/ITestRuntimeProviderManager.cs @@ -0,0 +1,11 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using Microsoft.VisualStudio.TestPlatform.ObjectModel.Host; + +namespace Microsoft.VisualStudio.TestPlatform.Common.Hosting; +internal interface ITestRuntimeProviderManager +{ + ITestRuntimeProvider GetTestHostManagerByRunConfiguration(string runConfiguration); + ITestRuntimeProvider GetTestHostManagerByUri(string hostUri); +} diff --git a/src/Microsoft.TestPlatform.Common/Hosting/TestRunTimeProviderManager.cs b/src/Microsoft.TestPlatform.Common/Hosting/TestRunTimeProviderManager.cs index 3c2bd3c0b1..752b2111e0 100644 --- a/src/Microsoft.TestPlatform.Common/Hosting/TestRunTimeProviderManager.cs +++ b/src/Microsoft.TestPlatform.Common/Hosting/TestRunTimeProviderManager.cs @@ -12,7 +12,7 @@ namespace Microsoft.VisualStudio.TestPlatform.Common.Hosting; /// /// Responsible for managing TestRuntimeProviderManager extensions /// -public class TestRuntimeProviderManager +public class TestRuntimeProviderManager : ITestRuntimeProviderManager { #region Fields diff --git a/src/Microsoft.TestPlatform.CrossPlatEngine/Friends.cs b/src/Microsoft.TestPlatform.CrossPlatEngine/Friends.cs index b52f4adf02..25f5726567 100644 --- a/src/Microsoft.TestPlatform.CrossPlatEngine/Friends.cs +++ b/src/Microsoft.TestPlatform.CrossPlatEngine/Friends.cs @@ -11,5 +11,6 @@ [assembly: InternalsVisibleTo("testhost, PublicKey = 002400000480000094000000060200000024000052534131000400000100010007d1fa57c4aed9f0a32e84aa0faefd0de9e8fd6aec8f87fb03766c834c99921eb23be79ad9d5dcc1dd9ad236132102900b723cf980957fc4e177108fc607774f29e8320e92ea05ece4e821c0a5efe8f1645c4c0c93c1ab99285d622caa652c1dfad63d745d6f2de5f17e5eaf0fc4963d261c8a12436518206dc093344d5ad293")] [assembly: InternalsVisibleTo("testhost.x86, PublicKey = 002400000480000094000000060200000024000052534131000400000100010007d1fa57c4aed9f0a32e84aa0faefd0de9e8fd6aec8f87fb03766c834c99921eb23be79ad9d5dcc1dd9ad236132102900b723cf980957fc4e177108fc607774f29e8320e92ea05ece4e821c0a5efe8f1645c4c0c93c1ab99285d622caa652c1dfad63d745d6f2de5f17e5eaf0fc4963d261c8a12436518206dc093344d5ad293")] [assembly: InternalsVisibleTo("vstest.console, PublicKey = 002400000480000094000000060200000024000052534131000400000100010007d1fa57c4aed9f0a32e84aa0faefd0de9e8fd6aec8f87fb03766c834c99921eb23be79ad9d5dcc1dd9ad236132102900b723cf980957fc4e177108fc607774f29e8320e92ea05ece4e821c0a5efe8f1645c4c0c93c1ab99285d622caa652c1dfad63d745d6f2de5f17e5eaf0fc4963d261c8a12436518206dc093344d5ad293")] +[assembly: InternalsVisibleTo("vstest.ProgrammerTests, PublicKey=002400000480000094000000060200000024000052534131000400000100010007d1fa57c4aed9f0a32e84aa0faefd0de9e8fd6aec8f87fb03766c834c99921eb23be79ad9d5dcc1dd9ad236132102900b723cf980957fc4e177108fc607774f29e8320e92ea05ece4e821c0a5efe8f1645c4c0c93c1ab99285d622caa652c1dfad63d745d6f2de5f17e5eaf0fc4963d261c8a12436518206dc093344d5ad293")] [assembly: InternalsVisibleTo("DynamicProxyGenAssembly2, PublicKey=0024000004800000940000000602000000240000525341310004000001000100c547cac37abd99c8db225ef2f6c8a3602f3b3606cc9891605d02baa56104f4cfc0734aa39b93bf7852f7d9266654753cc297e7d2edfe0bac1cdcf9f717241550e0a7b191195b7667bb4f64bcb8e2121380fd1d9d46ad2d92d2d15605093924cceaf74c4861eff62abf69b9291ed0a340e113be11e6a7d3113e92484cf7045cc7")] diff --git a/src/Microsoft.TestPlatform.CrossPlatEngine/TestEngine.cs b/src/Microsoft.TestPlatform.CrossPlatEngine/TestEngine.cs index 8914e19e65..95afa677a3 100644 --- a/src/Microsoft.TestPlatform.CrossPlatEngine/TestEngine.cs +++ b/src/Microsoft.TestPlatform.CrossPlatEngine/TestEngine.cs @@ -34,7 +34,7 @@ public class TestEngine : ITestEngine { #region Private Fields - private readonly TestRuntimeProviderManager _testHostProviderManager; + private readonly ITestRuntimeProviderManager _testHostProviderManager; private ITestExtensionManager _testExtensionManager; private readonly IProcessHelper _processHelper; @@ -44,8 +44,8 @@ public class TestEngine : ITestEngine { } - protected TestEngine( - TestRuntimeProviderManager testHostProviderManager, + internal TestEngine( + ITestRuntimeProviderManager testHostProviderManager, IProcessHelper processHelper) { _testHostProviderManager = testHostProviderManager; diff --git a/src/vstest.console/CommandLine/TestRunResultAggregator.cs b/src/vstest.console/CommandLine/TestRunResultAggregator.cs index 09c221d084..852a7b396c 100644 --- a/src/vstest.console/CommandLine/TestRunResultAggregator.cs +++ b/src/vstest.console/CommandLine/TestRunResultAggregator.cs @@ -20,7 +20,7 @@ internal class TestRunResultAggregator /// Initializes the TestRunResultAggregator /// /// Constructor is private since the factory method should be used to get the instance. - protected TestRunResultAggregator() + internal TestRunResultAggregator() { // Outcome is passed until we see a failure. Outcome = TestOutcome.Passed; diff --git a/src/vstest.console/Friends.cs b/src/vstest.console/Friends.cs index 9b2a48fabe..9dd880e408 100644 --- a/src/vstest.console/Friends.cs +++ b/src/vstest.console/Friends.cs @@ -10,6 +10,7 @@ #region Test Assemblies [assembly: InternalsVisibleTo("vstest.console.UnitTests, PublicKey=002400000480000094000000060200000024000052534131000400000100010007d1fa57c4aed9f0a32e84aa0faefd0de9e8fd6aec8f87fb03766c834c99921eb23be79ad9d5dcc1dd9ad236132102900b723cf980957fc4e177108fc607774f29e8320e92ea05ece4e821c0a5efe8f1645c4c0c93c1ab99285d622caa652c1dfad63d745d6f2de5f17e5eaf0fc4963d261c8a12436518206dc093344d5ad293")] +[assembly: InternalsVisibleTo("vstest.ProgrammerTests, PublicKey=002400000480000094000000060200000024000052534131000400000100010007d1fa57c4aed9f0a32e84aa0faefd0de9e8fd6aec8f87fb03766c834c99921eb23be79ad9d5dcc1dd9ad236132102900b723cf980957fc4e177108fc607774f29e8320e92ea05ece4e821c0a5efe8f1645c4c0c93c1ab99285d622caa652c1dfad63d745d6f2de5f17e5eaf0fc4963d261c8a12436518206dc093344d5ad293")] [assembly: InternalsVisibleTo("vstest.console.PlatformTests, PublicKey=002400000480000094000000060200000024000052534131000400000100010007d1fa57c4aed9f0a32e84aa0faefd0de9e8fd6aec8f87fb03766c834c99921eb23be79ad9d5dcc1dd9ad236132102900b723cf980957fc4e177108fc607774f29e8320e92ea05ece4e821c0a5efe8f1645c4c0c93c1ab99285d622caa652c1dfad63d745d6f2de5f17e5eaf0fc4963d261c8a12436518206dc093344d5ad293")] [assembly: InternalsVisibleTo("DynamicProxyGenAssembly2, PublicKey=0024000004800000940000000602000000240000525341310004000001000100c547cac37abd99c8db225ef2f6c8a3602f3b3606cc9891605d02baa56104f4cfc0734aa39b93bf7852f7d9266654753cc297e7d2edfe0bac1cdcf9f717241550e0a7b191195b7667bb4f64bcb8e2121380fd1d9d46ad2d92d2d15605093924cceaf74c4861eff62abf69b9291ed0a340e113be11e6a7d3113e92484cf7045cc7")] diff --git a/test/Intent.Primitives/ExcludeAttribute.cs b/test/Intent.Primitives/ExcludeAttribute.cs new file mode 100644 index 0000000000..4fe517e902 --- /dev/null +++ b/test/Intent.Primitives/ExcludeAttribute.cs @@ -0,0 +1,9 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +namespace Intent; + +[AttributeUsage(AttributeTargets.Assembly | AttributeTargets.Class | AttributeTargets.Constructor | AttributeTargets.Method)] +public class ExcludeAttribute : Attribute +{ +} diff --git a/test/Intent.Primitives/IRunLogger.cs b/test/Intent.Primitives/IRunLogger.cs new file mode 100644 index 0000000000..20891f65af --- /dev/null +++ b/test/Intent.Primitives/IRunLogger.cs @@ -0,0 +1,14 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using System.Reflection; + +namespace Intent; + +public interface IRunLogger +{ + void WriteTestPassed(MethodInfo m); + void WriteTestInconclusive(MethodInfo m); + void WriteTestFailure(MethodInfo m, Exception ex); + void WriteFrameworkError(Exception ex); +} diff --git a/test/Intent.Primitives/Intent.Primitives.csproj b/test/Intent.Primitives/Intent.Primitives.csproj new file mode 100644 index 0000000000..132c02c59c --- /dev/null +++ b/test/Intent.Primitives/Intent.Primitives.csproj @@ -0,0 +1,9 @@ + + + + net6.0 + enable + enable + + + diff --git a/test/Intent.Primitives/TestResult.cs b/test/Intent.Primitives/TestResult.cs new file mode 100644 index 0000000000..5d6408d736 --- /dev/null +++ b/test/Intent.Primitives/TestResult.cs @@ -0,0 +1,12 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +namespace Intent; + +public enum TestResult +{ + None = 0, + Passed, + Failed, + Error, +} diff --git a/test/Intent/ConsoleLogger.cs b/test/Intent/ConsoleLogger.cs new file mode 100644 index 0000000000..af1edb5543 --- /dev/null +++ b/test/Intent/ConsoleLogger.cs @@ -0,0 +1,44 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using System.Reflection; + +using static System.Console; +using static System.ConsoleColor; + +namespace Intent.Console; + +internal class ConsoleLogger : IRunLogger +{ + public void WriteTestInconclusive(MethodInfo m) + { + var currentColor = ForegroundColor; + ForegroundColor = Yellow; + WriteLine($"[?] {m.Name} inconclusive"); + ForegroundColor = currentColor; + } + + public void WriteTestPassed(MethodInfo m) + { + var currentColor = ForegroundColor; + ForegroundColor = Green; + WriteLine($"[+] {m.Name} passed"); + ForegroundColor = currentColor; + } + + public void WriteTestFailure(MethodInfo m, Exception ex) + { + var currentColor = ForegroundColor; + ForegroundColor = Red; + WriteLine($"[-] {m.Name} failed{Environment.NewLine}{ex}"); + ForegroundColor = currentColor; + } + + public void WriteFrameworkError(Exception ex) + { + var currentColor = ForegroundColor; + ForegroundColor = DarkRed; + WriteLine($"[-] framework failed{Environment.NewLine}{ex}{Environment.NewLine}{Environment.NewLine}"); + ForegroundColor = currentColor; + } +} diff --git a/test/Intent/Extensions.cs b/test/Intent/Extensions.cs new file mode 100644 index 0000000000..9e777c3162 --- /dev/null +++ b/test/Intent/Extensions.cs @@ -0,0 +1,36 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using System.Reflection; + +namespace Intent; + +public static class Extensions +{ + public static bool IsExcluded(this Assembly asm) + { + return asm.CustomAttributes.Any(a => a.AttributeType == typeof(ExcludeAttribute)); + } + + public static List SkipExcluded(this IEnumerable e) + { + return e.Where(i => i.GetCustomAttribute() == null).ToList(); + } + + public static List SkipNonPublic(this IEnumerable e) + { + return e.Where(i => i.IsPublic).ToList(); + } + + public static List SkipExcluded(this IEnumerable e) + { + return e.Where(i => + i.Name != nameof(object.ToString) + && i.Name != nameof(object.GetType) + && i.Name != nameof(object.GetHashCode) + && i.Name != nameof(object.Equals) + && i.GetCustomAttribute() == null).ToList(); + } + + +} diff --git a/test/Intent/Intent.csproj b/test/Intent/Intent.csproj new file mode 100644 index 0000000000..bb711c9256 --- /dev/null +++ b/test/Intent/Intent.csproj @@ -0,0 +1,14 @@ + + + + Exe + net6.0 + enable + enable + + + + + + + diff --git a/test/Intent/Program.cs b/test/Intent/Program.cs new file mode 100644 index 0000000000..75d50702b1 --- /dev/null +++ b/test/Intent/Program.cs @@ -0,0 +1,12 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +namespace Intent.Console; + +public class Program +{ + public static void Main(string[] path) + { + Runner.Run(path, new ConsoleLogger()); + } +} diff --git a/test/Intent/Runner.cs b/test/Intent/Runner.cs new file mode 100644 index 0000000000..3dbf68ca49 --- /dev/null +++ b/test/Intent/Runner.cs @@ -0,0 +1,52 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using System.Reflection; + +namespace Intent; + +public class Runner +{ + public static void Run(IEnumerable path, IRunLogger logger) + { + foreach (var p in path) + { + try + { + var asm = Assembly.LoadFrom(p); + if (asm.IsExcluded()) + continue; + + var ts = asm.GetTypes().SkipNonPublic().SkipExcluded(); + foreach (var t in ts) + { + var ms = t.GetMethods().SkipExcluded(); + foreach (var m in ms) + { + try + { + var i = Activator.CreateInstance(t); + m.Invoke(i, Array.Empty()); + logger.WriteTestPassed(m); + } + catch (Exception ex) + { + if (ex is TargetInvocationException tex && tex.InnerException != null) + { + logger.WriteTestFailure(m, tex.InnerException); + } + else + { + logger.WriteTestFailure(m, ex); + } + } + } + } + } + catch (Exception ex) + { + logger.WriteFrameworkError(ex); + } + } + } +} diff --git a/test/vstest.ProgrammerTests/Program.cs b/test/vstest.ProgrammerTests/Program.cs new file mode 100644 index 0000000000..6de575dca8 --- /dev/null +++ b/test/vstest.ProgrammerTests/Program.cs @@ -0,0 +1,13 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using System.Reflection; + +namespace vstest.ProgrammerTests; +internal class Program +{ + static void Main() + { + Intent.Console.Program.Main(new[] { Assembly.GetExecutingAssembly().Location }); + } +} diff --git a/test/vstest.ProgrammerTests/Properties/launchSettings.json b/test/vstest.ProgrammerTests/Properties/launchSettings.json new file mode 100644 index 0000000000..33504c948a --- /dev/null +++ b/test/vstest.ProgrammerTests/Properties/launchSettings.json @@ -0,0 +1,8 @@ +{ + "profiles": { + "WSL": { + "commandName": "WSL2", + "distributionName": "" + } + } +} \ No newline at end of file diff --git a/test/vstest.ProgrammerTests/UnitTest1.cs b/test/vstest.ProgrammerTests/UnitTest1.cs new file mode 100644 index 0000000000..df55767bb9 --- /dev/null +++ b/test/vstest.ProgrammerTests/UnitTest1.cs @@ -0,0 +1,238 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using System.Text; + +using FluentAssertions; + +using Microsoft.VisualStudio.TestPlatform.Client; +using Microsoft.VisualStudio.TestPlatform.CommandLine; +using Microsoft.VisualStudio.TestPlatform.CommandLine.Processors; +using Microsoft.VisualStudio.TestPlatform.CommandLine.Publisher; +using Microsoft.VisualStudio.TestPlatform.CommandLine.TestPlatformHelpers; +using Microsoft.VisualStudio.TestPlatform.CommandLineUtilities; +using Microsoft.VisualStudio.TestPlatform.Common.Hosting; +using Microsoft.VisualStudio.TestPlatform.CoreUtilities.Tracing.Interfaces; +using Microsoft.VisualStudio.TestPlatform.CrossPlatEngine; +using Microsoft.VisualStudio.TestPlatform.ObjectModel.Client; +using Microsoft.VisualStudio.TestPlatform.ObjectModel.Engine; +using Microsoft.VisualStudio.TestPlatform.PlatformAbstractions.Interfaces; +using Microsoft.VisualStudio.TestPlatform.Utilities; +using Microsoft.VisualStudio.TestPlatform.Utilities.Helpers.Interfaces; + +#pragma warning disable IDE1006 // Naming Styles +namespace vstest.ProgrammerTests.CommandLine; +#pragma warning restore IDE1006 // Naming Styles + +// exluded from run +internal class InlineRunSettingsTests +{ + public void GivenInlineRunsettingsWhenCallingVstestConsoleThenTheyPropagateToTestHost() + { + using Fixture fixture = new(); + fixture.VstestConsole + .WithSource(TestDlls.MSTest1) + .WithArguments($" -- {RunConfiguration.MaxParallelLevel.InlinePath}=3") + .Execute(); + + fixture.Processes.Should().HaveCount(1); + var process = fixture.Processes.First(); + process.Should().BeAssignableTo(); + var testhost = (FakeTestHostProcess)process; + testhost.RunSettings.Should().NotBeNull(); + testhost.RunSettings!.MaxParallelLevel.Should().Be(3); + } +} + +public class TestDiscoveryTests +{ + public void GivenAnMSTestAssemblyWith5Tests_WhenTestsAreDiscovered_Then5TestsAreFound() + { + var commandLineOptions = CommandLineOptions.Instance; + + var fakeTestRuntimeProviderManager = new FakeTestRuntimeProviderManager(); + var fakeProcessHelper = new FakeProcessHelper(); + var testEngine = new TestEngine(fakeTestRuntimeProviderManager, fakeProcessHelper); + var fakeFileHelper = new FakeFileHelper(); + var testPlatform = new TestPlatform(testEngine, fakeFileHelper, fakeTestRuntimeProviderManager); + + var testRunResultAggregator = new TestRunResultAggregator(); + var fakeTestPlatformEventSource = new FakeTestPlatformEventSource(); + + var fakeAssemblyMetadataProvider = new FakeAssemblyMetadataProvider(); + var inferHelper = new InferHelper(fakeAssemblyMetadataProvider); + + Task fakeMetricsPublisherTask = + TestRequestManager trm = new( + commandLineOptions, + testPlatform, + testRunResultAggregator, + fakeTestPlatformEventSource, + inferHelper, + metricsPublisherTask, + + ); + + + } + +} + +internal class FakeAssemblyMetadataProvider : IAssemblyMetadataProvider +{ + public FakeAssemblyMetadataProvider() + { + } +} + +internal class FakeTestPlatformEventSource : ITestPlatformEventSource +{ + public FakeTestPlatformEventSource() + { + } +} + +internal class FakeFileHelper : IFileHelper +{ + public FakeFileHelper() + { + } +} + +internal class FakeProcessHelper : IProcessHelper +{ + public FakeProcessHelper() + { + } +} + +internal class FakeTestRuntimeProviderManager : ITestRuntimeProviderManager +{ + public FakeTestRuntimeProviderManager() + { + } +} + +internal class FakeTestHostProcess : FakeProcess +{ + public FakeTestHostProcess(string commandLine) : base(commandLine) + { + } + + public CapturedRunSettings? RunSettings { get; internal set; } +} + +internal class Fixture : IDisposable +{ + public Fixture() + { + + } + + public List Processes { get; } = new(); + + public VstestConsole VstestConsole { get; } = new(); + + public FakeTestExtensionManager TestExtensionManager { get; } = new(); + + public void Dispose() + { + + } +} + +internal class FakeTestExtensionManager : ITestExtensionManager +{ + public void ClearExtensions() + { + throw new NotImplementedException(); + } + + public void UseAdditionalExtensions(IEnumerable pathToAdditionalExtensions, bool skipExtensionFilters) + { + throw new NotImplementedException(); + } +} + +internal class FakeProcess +{ + public string CommandLine { get; } + + public FakeProcess(string commandLine) + { + CommandLine = commandLine; + } +} + + + +internal class CapturedRunSettings +{ + public int MaxParallelLevel { get; internal set; } +} + +internal class FakeOutput : IOutput +{ + public FakeOutput() + { + } + + public List Messages { get; } = new(); + public StringBuilder CurrentLine { get; } = new(); + public List Lines { get; } = new(); + + public void Write(string message, OutputLevel level) + { + Messages.Add(new OutputMessage(message, level, isNewLine: false)); + CurrentLine.Append(message); + } + + public void WriteLine(string message, OutputLevel level) + { + Lines.Add(CurrentLine + message); + CurrentLine.Clear(); + } +} + +internal class OutputMessage +{ + public OutputMessage(string message, OutputLevel level, bool isNewLine) + { + Message = message; + Level = level; + IsNewLine = isNewLine; + } + + public string Message { get; } + public OutputLevel Level { get; } + public bool IsNewLine { get; } +} + +internal static class RunConfiguration +{ + public static ConfigurationEntry MaxParallelLevel { get; } = new(nameof(MaxParallelLevel)); +} + +internal class ConfigurationEntry +{ + public ConfigurationEntry(string name) + { + Name = name; + } + + public string Name { get; } + + public string InlinePath => $"RunConfiguration.{Name}"; + + public string FullPath => $"RunSettings.{InlinePath}"; + + public override string ToString() + { + return Name; + } +} + +internal class TestDlls +{ + public static string MSTest1 { get; } = $"{nameof(MSTest1)}.dll"; +} diff --git a/test/vstest.ProgrammerTests/VstestConsole.cs b/test/vstest.ProgrammerTests/VstestConsole.cs new file mode 100644 index 0000000000..ce2cd698e5 --- /dev/null +++ b/test/vstest.ProgrammerTests/VstestConsole.cs @@ -0,0 +1,47 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using Microsoft.VisualStudio.TestPlatform.CommandLine; + +namespace vstest.ProgrammerTests.CommandLine; + +internal class VstestConsole +{ + public List Sources { get; } = new(); + + public List Arguments { get; } = new(); + + internal FakeOutput Output { get; } = new(); + + internal VstestConsole WithSource(params string[] sources) + { + Sources.AddRange(sources); + return this; + } + + internal VstestConsole WithArguments(params string[] arguments) + { + Arguments.AddRange(arguments); + return this; + } + + internal void Execute() + { + var commandLine = new[] { Sources, Arguments }.SelectMany(s => s).JoinBySpace(); + // vstest.console + var console = new Executor(Output).Execute(commandLine); + } +} + +internal static class StringExtensions +{ + public static string Join(this IEnumerable value, string separator) + { + return string.Join(separator, value); + } + + public static string JoinBySpace(this IEnumerable value) + { + return string.Join(" ", value); + } +} diff --git a/test/vstest.ProgrammerTests/vstest.ProgrammerTests.csproj b/test/vstest.ProgrammerTests/vstest.ProgrammerTests.csproj new file mode 100644 index 0000000000..648d985bc9 --- /dev/null +++ b/test/vstest.ProgrammerTests/vstest.ProgrammerTests.csproj @@ -0,0 +1,39 @@ + + + + ..\..\ + false + false + + + + enable + true + preview + net6.0 + netcoreapp3.1 + Exe + Exe + + + + + + + + true + + + true + + + + + + + + + + + + From 8df385423d2ee870535408e9f564612aeb262144 Mon Sep 17 00:00:00 2001 From: nohwnd Date: Mon, 7 Feb 2022 17:59:55 +0100 Subject: [PATCH 02/32] Isolating --- .../TestRequestSender.cs | 14 +- .../Client/TestRunCriteria.cs | 2 +- .../TestServiceLocator.cs | 27 ++ .../TestPlatformHelpers/TestRequestManager.cs | 2 + .../Fakes/EventRecord.cs | 19 ++ .../Fakes/FakeAssemblyMetadataProvider.cs | 32 +++ .../Fakes/FakeCommunicationEndpoint.cs | 27 ++ ...taCollectorAttachmentsProcessorsFactory.cs | 17 ++ .../Fakes/FakeDllFile.cs | 22 ++ test/vstest.ProgrammerTests/Fakes/FakeFile.cs | 14 ++ .../Fakes/FakeFileHelper.cs | 104 ++++++++ .../Fakes/FakeMetricsPublisher.cs | 20 ++ .../Fakes/FakeOutput.cs | 32 +++ .../Fakes/FakeProcess.cs | 38 +++ .../Fakes/FakeProcessHelper.cs | 106 ++++++++ .../Fakes/FakeTestExtensionManager.cs | 20 ++ .../Fakes/FakeTestHostLauncher.cs | 31 +++ .../Fakes/FakeTestHostProcess.cs | 15 ++ .../Fakes/FakeTestPlatformEventSource.cs | 234 ++++++++++++++++++ .../Fakes/FakeTestRunEventsRegistrar.cs | 79 ++++++ .../Fakes/FakeTestRuntimeProvider.cs | 81 ++++++ .../Fakes/FakeTestRuntimeProviderManager.cs | 28 +++ test/vstest.ProgrammerTests/OutputMessage.cs | 21 ++ .../Properties/launchSettings.json | 3 + .../StringExtensions.cs | 22 ++ test/vstest.ProgrammerTests/UnitTest1.cs | 162 ++++-------- test/vstest.ProgrammerTests/VstestConsole.cs | 15 +- 27 files changed, 1051 insertions(+), 136 deletions(-) create mode 100644 src/Microsoft.TestPlatform.ObjectModel/TestServiceLocator.cs create mode 100644 test/vstest.ProgrammerTests/Fakes/EventRecord.cs create mode 100644 test/vstest.ProgrammerTests/Fakes/FakeAssemblyMetadataProvider.cs create mode 100644 test/vstest.ProgrammerTests/Fakes/FakeCommunicationEndpoint.cs create mode 100644 test/vstest.ProgrammerTests/Fakes/FakeDataCollectorAttachmentsProcessorsFactory.cs create mode 100644 test/vstest.ProgrammerTests/Fakes/FakeDllFile.cs create mode 100644 test/vstest.ProgrammerTests/Fakes/FakeFile.cs create mode 100644 test/vstest.ProgrammerTests/Fakes/FakeFileHelper.cs create mode 100644 test/vstest.ProgrammerTests/Fakes/FakeMetricsPublisher.cs create mode 100644 test/vstest.ProgrammerTests/Fakes/FakeOutput.cs create mode 100644 test/vstest.ProgrammerTests/Fakes/FakeProcess.cs create mode 100644 test/vstest.ProgrammerTests/Fakes/FakeProcessHelper.cs create mode 100644 test/vstest.ProgrammerTests/Fakes/FakeTestExtensionManager.cs create mode 100644 test/vstest.ProgrammerTests/Fakes/FakeTestHostLauncher.cs create mode 100644 test/vstest.ProgrammerTests/Fakes/FakeTestHostProcess.cs create mode 100644 test/vstest.ProgrammerTests/Fakes/FakeTestPlatformEventSource.cs create mode 100644 test/vstest.ProgrammerTests/Fakes/FakeTestRunEventsRegistrar.cs create mode 100644 test/vstest.ProgrammerTests/Fakes/FakeTestRuntimeProvider.cs create mode 100644 test/vstest.ProgrammerTests/Fakes/FakeTestRuntimeProviderManager.cs create mode 100644 test/vstest.ProgrammerTests/OutputMessage.cs create mode 100644 test/vstest.ProgrammerTests/StringExtensions.cs diff --git a/src/Microsoft.TestPlatform.CommunicationUtilities/TestRequestSender.cs b/src/Microsoft.TestPlatform.CommunicationUtilities/TestRequestSender.cs index 5d5b70e999..eaf114df66 100644 --- a/src/Microsoft.TestPlatform.CommunicationUtilities/TestRequestSender.cs +++ b/src/Microsoft.TestPlatform.CommunicationUtilities/TestRequestSender.cs @@ -78,7 +78,6 @@ public TestRequestSender(ProtocolConfig protocolConfig, ITestRuntimeProvider run protocolConfig, ClientProcessExitWaitTimeout) { - SetCommunicationEndPoint(); } internal TestRequestSender( @@ -99,7 +98,12 @@ internal TestRequestSender( // The connectionInfo here is that of RuntimeProvider, so reverse the role of runner. _runtimeProvider = runtimeProvider; - _communicationEndpoint = communicationEndPoint; + + // TODO: In various places TestRequest sender is instantiated, and we can't easily inject the factory, so this is last + // resort of getting the dependency into the execution flow. + // TODO: I am not sure if we need multiple instances of ICommunicationEndpoint, in that case we should register + // and resolve Func and invoke that. + _communicationEndpoint = communicationEndPoint ?? TestServiceLocator.Get() ?? SetCommunicationEndPoint(); _connectionInfo.Endpoint = connectionInfo.Endpoint; _connectionInfo.Role = connectionInfo.Role == ConnectionRole.Host ? ConnectionRole.Client @@ -748,24 +752,24 @@ private void SetOperationComplete() Interlocked.CompareExchange(ref _operationCompleted, 1, 0); } - private void SetCommunicationEndPoint() + private ICommunicationEndPoint SetCommunicationEndPoint() { // TODO: Use factory to get the communication endpoint. It will abstract out the type of communication endpoint like socket, shared memory or named pipe etc., if (_connectionInfo.Role == ConnectionRole.Client) { - _communicationEndpoint = new SocketClient(); if (EqtTrace.IsVerboseEnabled) { EqtTrace.Verbose("TestRequestSender is acting as client"); } + return new SocketClient(); } else { - _communicationEndpoint = new SocketServer(); if (EqtTrace.IsVerboseEnabled) { EqtTrace.Verbose("TestRequestSender is acting as server"); } + return new SocketServer(); } } } diff --git a/src/Microsoft.TestPlatform.ObjectModel/Client/TestRunCriteria.cs b/src/Microsoft.TestPlatform.ObjectModel/Client/TestRunCriteria.cs index 2629058517..1c2f9288d4 100644 --- a/src/Microsoft.TestPlatform.ObjectModel/Client/TestRunCriteria.cs +++ b/src/Microsoft.TestPlatform.ObjectModel/Client/TestRunCriteria.cs @@ -480,7 +480,7 @@ public TestRunCriteria( runStatsChangeEventTimeout, testHostLauncher) { - var testCases = tests as IList ?? tests.ToList(); + var testCases = tests as IList ?? tests?.ToList(); ValidateArg.NotNullOrEmpty(testCases, nameof(tests)); Tests = testCases; diff --git a/src/Microsoft.TestPlatform.ObjectModel/TestServiceLocator.cs b/src/Microsoft.TestPlatform.ObjectModel/TestServiceLocator.cs new file mode 100644 index 0000000000..c99af67eaa --- /dev/null +++ b/src/Microsoft.TestPlatform.ObjectModel/TestServiceLocator.cs @@ -0,0 +1,27 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using System; +using System.Collections.Generic; + +namespace Microsoft.VisualStudio.TestPlatform.ObjectModel; + +// TODO: Make this internal, I am just trying to have easier time trying this out. +public static class TestServiceLocator +{ + public static Dictionary Instances { get; } = new Dictionary(); + + public static void Register(TRegistration instance) + { + Instances.Add(typeof(TRegistration), instance); + } + + public static TRegistration Get() + { + if (!Instances.TryGetValue(typeof(TRegistration), out var instance)) + throw new InvalidOperationException($"Cannot find instance for type {typeof(TRegistration)}."); + + return (TRegistration)instance; + } +} + diff --git a/src/vstest.console/TestPlatformHelpers/TestRequestManager.cs b/src/vstest.console/TestPlatformHelpers/TestRequestManager.cs index 6911fb5fba..8808615b67 100644 --- a/src/vstest.console/TestPlatformHelpers/TestRequestManager.cs +++ b/src/vstest.console/TestPlatformHelpers/TestRequestManager.cs @@ -49,6 +49,8 @@ internal class TestRequestManager : ITestRequestManager private readonly ITestPlatform _testPlatform; private readonly ITestPlatformEventSource _testPlatformEventSource; + // TODO: No idea what is Task supposed to buy us, Tasks start immediately on instantiation + // and the work done to produce the metrics publisher is minimal. private readonly Task _metricsPublisher; private readonly object _syncObject = new(); diff --git a/test/vstest.ProgrammerTests/Fakes/EventRecord.cs b/test/vstest.ProgrammerTests/Fakes/EventRecord.cs new file mode 100644 index 0000000000..9a8dfa91da --- /dev/null +++ b/test/vstest.ProgrammerTests/Fakes/EventRecord.cs @@ -0,0 +1,19 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + + +#pragma warning disable IDE1006 // Naming Styles +namespace vstest.ProgrammerTests.CommandLine.Fakes; + +internal class EventRecord +{ + public object? Sender { get; } + + public T Data { get; } + + public EventRecord(object? sender, T data) + { + Sender = sender; + Data = data; + } +} diff --git a/test/vstest.ProgrammerTests/Fakes/FakeAssemblyMetadataProvider.cs b/test/vstest.ProgrammerTests/Fakes/FakeAssemblyMetadataProvider.cs new file mode 100644 index 0000000000..b6a723652a --- /dev/null +++ b/test/vstest.ProgrammerTests/Fakes/FakeAssemblyMetadataProvider.cs @@ -0,0 +1,32 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using System.Runtime.Versioning; + +using Microsoft.VisualStudio.TestPlatform.CommandLine.Processors; +using Microsoft.VisualStudio.TestPlatform.ObjectModel; + +#pragma warning disable IDE1006 // Naming Styles +namespace vstest.ProgrammerTests.CommandLine.Fakes; + +internal class FakeAssemblyMetadataProvider : IAssemblyMetadataProvider +{ + private readonly FakeFileHelper _fakeFileHelper; + + public FakeAssemblyMetadataProvider(FakeFileHelper fakeFileHelper) + { + _fakeFileHelper = fakeFileHelper; + } + + public Architecture GetArchitecture(string filePath) + { + var file = (FakeDllFile)_fakeFileHelper.Files.Single(f => f.Path == filePath); + return file.Architecture; + } + + public FrameworkName GetFrameWork(string filePath) + { + var file = (FakeDllFile)_fakeFileHelper.Files.Single(f => f.Path == filePath); + return file.FrameworkName; + } +} diff --git a/test/vstest.ProgrammerTests/Fakes/FakeCommunicationEndpoint.cs b/test/vstest.ProgrammerTests/Fakes/FakeCommunicationEndpoint.cs new file mode 100644 index 0000000000..279f3142e8 --- /dev/null +++ b/test/vstest.ProgrammerTests/Fakes/FakeCommunicationEndpoint.cs @@ -0,0 +1,27 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using Microsoft.VisualStudio.TestPlatform.CommunicationUtilities.Interfaces; + +namespace vstest.ProgrammerTests.CommandLine; + +internal class FakeCommunicationEndpoint : ICommunicationEndPoint +{ + public FakeCommunicationEndpoint() + { + } + + public event EventHandler Connected; + public event EventHandler Disconnected; + + public string Start(string endPoint) + { + Connected?.Invoke(this, new ConnectedEventArgs()); + return endPoint; + } + + public void Stop() + { + Disconnected?.Invoke(this, new DisconnectedEventArgs()); + } +} diff --git a/test/vstest.ProgrammerTests/Fakes/FakeDataCollectorAttachmentsProcessorsFactory.cs b/test/vstest.ProgrammerTests/Fakes/FakeDataCollectorAttachmentsProcessorsFactory.cs new file mode 100644 index 0000000000..bd9b2fe3cc --- /dev/null +++ b/test/vstest.ProgrammerTests/Fakes/FakeDataCollectorAttachmentsProcessorsFactory.cs @@ -0,0 +1,17 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using Microsoft.VisualStudio.TestPlatform.ObjectModel; +using Microsoft.VisualStudio.TestPlatform.ObjectModel.Engine; +using Microsoft.VisualStudio.TestPlatform.ObjectModel.Logging; + +#pragma warning disable IDE1006 // Naming Styles +namespace vstest.ProgrammerTests.CommandLine.Fakes; + +internal class FakeDataCollectorAttachmentsProcessorsFactory : IDataCollectorAttachmentsProcessorsFactory +{ + public DataCollectorAttachmentProcessor[] Create(InvokedDataCollector[] invokedDataCollectors, IMessageLogger logger) + { + throw new NotImplementedException(); + } +} diff --git a/test/vstest.ProgrammerTests/Fakes/FakeDllFile.cs b/test/vstest.ProgrammerTests/Fakes/FakeDllFile.cs new file mode 100644 index 0000000000..4bc1aedd75 --- /dev/null +++ b/test/vstest.ProgrammerTests/Fakes/FakeDllFile.cs @@ -0,0 +1,22 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using System.Runtime.Versioning; + +using Microsoft.VisualStudio.TestPlatform.ObjectModel; + +using vstest.ProgrammerTests.CommandLine.Fakes; + +namespace vstest.ProgrammerTests.CommandLine; + +internal class FakeDllFile : FakeFile +{ + public FrameworkName FrameworkName { get; init; } + public Architecture Architecture { get; init; } + + public FakeDllFile(string path, FrameworkName frameworkName, Architecture architecture) : base(path) + { + FrameworkName = frameworkName; + Architecture = architecture; + } +} diff --git a/test/vstest.ProgrammerTests/Fakes/FakeFile.cs b/test/vstest.ProgrammerTests/Fakes/FakeFile.cs new file mode 100644 index 0000000000..4df355eeb1 --- /dev/null +++ b/test/vstest.ProgrammerTests/Fakes/FakeFile.cs @@ -0,0 +1,14 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +namespace vstest.ProgrammerTests.CommandLine.Fakes; + +internal class FakeFile +{ + public string Path { get; } + + public FakeFile(string path) + { + Path = path; + } +} diff --git a/test/vstest.ProgrammerTests/Fakes/FakeFileHelper.cs b/test/vstest.ProgrammerTests/Fakes/FakeFileHelper.cs new file mode 100644 index 0000000000..cdbd9a24c6 --- /dev/null +++ b/test/vstest.ProgrammerTests/Fakes/FakeFileHelper.cs @@ -0,0 +1,104 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using Microsoft.VisualStudio.TestPlatform.Utilities.Helpers.Interfaces; + +#pragma warning disable IDE1006 // Naming Styles +namespace vstest.ProgrammerTests.CommandLine.Fakes; + +internal class FakeFileHelper : IFileHelper +{ + public List Files { get; } = new(); + + public void CopyFile(string sourcePath, string destinationPath) + { + throw new NotImplementedException(); + } + + public DirectoryInfo CreateDirectory(string path) + { + throw new NotImplementedException(); + } + + public void Delete(string path) + { + throw new NotImplementedException(); + } + + public void DeleteEmptyDirectroy(string directoryPath) + { + throw new NotImplementedException(); + } + + public bool DirectoryExists(string path) + { + // TODO: Check if any file has the directory in name. This will improve. + var directoryExists = Files.Select(f => Path.GetDirectoryName(f.Path)).Any(p => p.StartsWith(path)); + return directoryExists; + } + + public IEnumerable EnumerateFiles(string directory, SearchOption searchOption, params string[] endsWithSearchPatterns) + { + Func predicate = searchOption == SearchOption.TopDirectoryOnly + ? (f, dir) => Path.GetDirectoryName(f.Path) == dir + : (f, dir) => Path.GetDirectoryName(f.Path)!.Contains(dir); + + var files = Files.Where(f => predicate(f, directory)).Select(f => f.Path).ToList(); + return files; + } + + public bool Exists(string path) + { + throw new NotImplementedException(); + } + + public string GetCurrentDirectory() + { + throw new NotImplementedException(); + } + + public FileAttributes GetFileAttributes(string path) + { + throw new NotImplementedException(); + } + + public string[] GetFiles(string path, string searchPattern, SearchOption searchOption) + { + throw new NotImplementedException(); + } + + public Version GetFileVersion(string path) + { + throw new NotImplementedException(); + } + + public string GetFullPath(string path) + { + throw new NotImplementedException(); + } + + public Stream GetStream(string filePath, FileMode mode, FileAccess access = FileAccess.ReadWrite) + { + throw new NotImplementedException(); + } + + public Stream GetStream(string filePath, FileMode mode, FileAccess access, FileShare share) + { + throw new NotImplementedException(); + } + + public void MoveFile(string sourcePath, string destinationPath) + { + throw new NotImplementedException(); + } + + public void WriteAllTextToFile(string filePath, string content) + { + throw new NotImplementedException(); + } + + internal void AddFile(T file) where T : FakeFile + { + Files.Add(file); + } +} diff --git a/test/vstest.ProgrammerTests/Fakes/FakeMetricsPublisher.cs b/test/vstest.ProgrammerTests/Fakes/FakeMetricsPublisher.cs new file mode 100644 index 0000000000..164d235552 --- /dev/null +++ b/test/vstest.ProgrammerTests/Fakes/FakeMetricsPublisher.cs @@ -0,0 +1,20 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using Microsoft.VisualStudio.TestPlatform.CommandLine.Publisher; + +#pragma warning disable IDE1006 // Naming Styles +namespace vstest.ProgrammerTests.CommandLine.Fakes; + +internal class FakeMetricsPublisher : IMetricsPublisher +{ + public void Dispose() + { + throw new NotImplementedException(); + } + + public void PublishMetrics(string eventName, IDictionary metrics) + { + throw new NotImplementedException(); + } +} diff --git a/test/vstest.ProgrammerTests/Fakes/FakeOutput.cs b/test/vstest.ProgrammerTests/Fakes/FakeOutput.cs new file mode 100644 index 0000000000..60cc095311 --- /dev/null +++ b/test/vstest.ProgrammerTests/Fakes/FakeOutput.cs @@ -0,0 +1,32 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using System.Text; + +using Microsoft.VisualStudio.TestPlatform.Utilities; + +#pragma warning disable IDE1006 // Naming Styles +namespace vstest.ProgrammerTests.CommandLine.Fakes; + +internal class FakeOutput : IOutput +{ + public FakeOutput() + { + } + + public List Messages { get; } = new(); + public StringBuilder CurrentLine { get; } = new(); + public List Lines { get; } = new(); + + public void Write(string message, OutputLevel level) + { + Messages.Add(new OutputMessage(message, level, isNewLine: false)); + CurrentLine.Append(message); + } + + public void WriteLine(string message, OutputLevel level) + { + Lines.Add(CurrentLine + message); + CurrentLine.Clear(); + } +} diff --git a/test/vstest.ProgrammerTests/Fakes/FakeProcess.cs b/test/vstest.ProgrammerTests/Fakes/FakeProcess.cs new file mode 100644 index 0000000000..49469009b5 --- /dev/null +++ b/test/vstest.ProgrammerTests/Fakes/FakeProcess.cs @@ -0,0 +1,38 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + + +#pragma warning disable IDE1006 // Naming Styles +using Microsoft.VisualStudio.TestPlatform.PlatformAbstractions; + +namespace vstest.ProgrammerTests.CommandLine.Fakes; + +internal class FakeProcess +{ + public int Id { get; internal set; } + public string Name { get; init; } + public string Path { get; } + public string Arguments { get; set; } + + public PlatformArchitecture Architecture { get; init; } = PlatformArchitecture.X64; + + public FakeProcess(string path, string arguments = null) + { + Path = path; + Name = System.IO.Path.GetFileName(path); + Arguments = arguments; + } + + internal static FakeProcess EnsureFakeProcess(object process) + { + return (FakeProcess)process; + } + + internal void SetId(int id) + { + if (Id != 0) + throw new InvalidOperationException($"Cannot set Id to {id} for fake process {Name}, {Id}, because it was already set."); + + Id = id; + } +} diff --git a/test/vstest.ProgrammerTests/Fakes/FakeProcessHelper.cs b/test/vstest.ProgrammerTests/Fakes/FakeProcessHelper.cs new file mode 100644 index 0000000000..cbefdf6728 --- /dev/null +++ b/test/vstest.ProgrammerTests/Fakes/FakeProcessHelper.cs @@ -0,0 +1,106 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using Microsoft.VisualStudio.TestPlatform.PlatformAbstractions; +using Microsoft.VisualStudio.TestPlatform.PlatformAbstractions.Interfaces; + +#pragma warning disable IDE1006 // Naming Styles +namespace vstest.ProgrammerTests.CommandLine.Fakes; + +internal class FakeProcessHelper : IProcessHelper +{ + // starting from 100 for no particular reason + // I want to avoid processId 0 and 1 as they are + // "reserved" on Windows (0) and Linux (both 0 and 1) + private int _lastProcessId = 100; + + public FakeProcess CurrentProcess { get; } + public List Processes { get; } = new(); + public int LastProcessId => _lastProcessId; + + public FakeProcessHelper(FakeProcess currentProcess) + { + CurrentProcess = currentProcess; + AddProcess(currentProcess); + } + + private void AddProcess(FakeProcess currentProcess) + { + var id = Interlocked.Increment(ref _lastProcessId); + currentProcess.SetId(id); + Processes.Add(currentProcess); + } + + public PlatformArchitecture GetCurrentProcessArchitecture() + { + return CurrentProcess.Architecture; + } + + public string GetCurrentProcessFileName() + { + return CurrentProcess.Path; + } + + public int GetCurrentProcessId() + { + return CurrentProcess.Id; + } + + public string GetCurrentProcessLocation() + { + // TODO: how is this different from Path + throw new NotImplementedException(); + } + + public string GetNativeDllDirectory() + { + throw new NotImplementedException(); + } + + public IntPtr GetProcessHandle(int processId) + { + throw new NotImplementedException(); + } + + public int GetProcessId(object process) + { + var fakeProcess = FakeProcess.EnsureFakeProcess(process); + return fakeProcess.Id; + } + + public string GetProcessName(int processId) + { + var process = Processes.Single(p => p.Id == processId); + return process.Name; + } + + public string GetTestEngineDirectory() + { + throw new NotImplementedException(); + } + + public object LaunchProcess(string processPath, string arguments, string workingDirectory, IDictionary environmentVariables, Action errorCallback, Action exitCallBack, Action outputCallback) + { + throw new NotImplementedException(); + } + + public void SetExitCallback(int processId, Action callbackAction) + { + throw new NotImplementedException(); + } + + public void TerminateProcess(object process) + { + throw new NotImplementedException(); + } + + public bool TryGetExitCode(object process, out int exitCode) + { + throw new NotImplementedException(); + } + + public void WaitForProcessExit(object process) + { + throw new NotImplementedException(); + } +} diff --git a/test/vstest.ProgrammerTests/Fakes/FakeTestExtensionManager.cs b/test/vstest.ProgrammerTests/Fakes/FakeTestExtensionManager.cs new file mode 100644 index 0000000000..b503278332 --- /dev/null +++ b/test/vstest.ProgrammerTests/Fakes/FakeTestExtensionManager.cs @@ -0,0 +1,20 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using Microsoft.VisualStudio.TestPlatform.ObjectModel.Engine; + +#pragma warning disable IDE1006 // Naming Styles +namespace vstest.ProgrammerTests.CommandLine.Fakes; + +internal class FakeTestExtensionManager : ITestExtensionManager +{ + public void ClearExtensions() + { + throw new NotImplementedException(); + } + + public void UseAdditionalExtensions(IEnumerable pathToAdditionalExtensions, bool skipExtensionFilters) + { + throw new NotImplementedException(); + } +} diff --git a/test/vstest.ProgrammerTests/Fakes/FakeTestHostLauncher.cs b/test/vstest.ProgrammerTests/Fakes/FakeTestHostLauncher.cs new file mode 100644 index 0000000000..b7e318a3c0 --- /dev/null +++ b/test/vstest.ProgrammerTests/Fakes/FakeTestHostLauncher.cs @@ -0,0 +1,31 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using Microsoft.VisualStudio.TestPlatform.ObjectModel; +using Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.Interfaces; + +#pragma warning disable IDE1006 // Naming Styles +namespace vstest.ProgrammerTests.CommandLine.Fakes; + +internal class FakeTestHostLauncher : ITestHostLauncher +{ + private readonly FakeProcessHelper _fakeProcessHelper; + + public FakeTestHostLauncher(FakeProcessHelper fakeProcessHelper, bool isDebug = false) + { + IsDebug = isDebug; + _fakeProcessHelper = fakeProcessHelper; + } + + public bool IsDebug { get; } + + public int LaunchTestHost(TestProcessStartInfo defaultTestHostStartInfo) + { + throw new NotImplementedException(); + } + + public int LaunchTestHost(TestProcessStartInfo defaultTestHostStartInfo, CancellationToken cancellationToken) + { + throw new NotImplementedException(); + } +} diff --git a/test/vstest.ProgrammerTests/Fakes/FakeTestHostProcess.cs b/test/vstest.ProgrammerTests/Fakes/FakeTestHostProcess.cs new file mode 100644 index 0000000000..934f369e80 --- /dev/null +++ b/test/vstest.ProgrammerTests/Fakes/FakeTestHostProcess.cs @@ -0,0 +1,15 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + + +#pragma warning disable IDE1006 // Naming Styles +namespace vstest.ProgrammerTests.CommandLine.Fakes; + +internal class FakeTestHostProcess : FakeProcess +{ + public FakeTestHostProcess(string commandLine) : base(commandLine) + { + } + + public CapturedRunSettings? RunSettings { get; internal set; } +} diff --git a/test/vstest.ProgrammerTests/Fakes/FakeTestPlatformEventSource.cs b/test/vstest.ProgrammerTests/Fakes/FakeTestPlatformEventSource.cs new file mode 100644 index 0000000000..8559468963 --- /dev/null +++ b/test/vstest.ProgrammerTests/Fakes/FakeTestPlatformEventSource.cs @@ -0,0 +1,234 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using Microsoft.VisualStudio.TestPlatform.CoreUtilities.Tracing.Interfaces; + +#pragma warning disable IDE1006 // Naming Styles +namespace vstest.ProgrammerTests.CommandLine.Fakes; + +internal class FakeTestPlatformEventSource : ITestPlatformEventSource +{ + public FakeTestPlatformEventSource() + { + } + + public void AdapterDiscoveryStart(string executorUri) + { + // do nothing + } + + public void AdapterDiscoveryStop(long numberOfTests) + { + // do nothing + } + + public void AdapterExecutionStart(string executorUri) + { + // do nothing + } + + public void AdapterExecutionStop(long numberOfTests) + { + // do nothing + } + + public void AdapterSearchStart() + { + // do nothing + } + + public void AdapterSearchStop() + { + // do nothing + } + + public void DataCollectionStart(string dataCollectorUri) + { + // do nothing + } + + public void DataCollectionStop() + { + // do nothing + } + + public void DiscoveryRequestStart() + { + // do nothing + } + + public void DiscoveryRequestStop() + { + // do nothing + } + + public void DiscoveryStart() + { + // do nothing + } + + public void DiscoveryStop(long numberOfTests) + { + // do nothing + } + + public void ExecutionRequestStart() + { + // do nothing + } + + public void ExecutionRequestStop() + { + // do nothing + } + + public void ExecutionStart() + { + // do nothing + } + + public void ExecutionStop(long numberOfTests) + { + // do nothing + } + + public void MetricsDisposeStart() + { + // do nothing + } + + public void MetricsDisposeStop() + { + // do nothing + } + + public void StartTestSessionStart() + { + // do nothing + } + + public void StartTestSessionStop() + { + // do nothing + } + + public void StopTestSessionStart() + { + // do nothing + } + + public void StopTestSessionStop() + { + // do nothing + } + + public void TestHostAppDomainCreationStart() + { + // do nothing + } + + public void TestHostAppDomainCreationStop() + { + // do nothing + } + + public void TestHostStart() + { + // do nothing + } + + public void TestHostStop() + { + // do nothing + } + + public void TestRunAttachmentsProcessingRequestStart() + { + // do nothing + } + + public void TestRunAttachmentsProcessingRequestStop() + { + // do nothing + } + + public void TestRunAttachmentsProcessingStart(long numberOfAttachments) + { + // do nothing + } + + public void TestRunAttachmentsProcessingStop(long numberOfAttachments) + { + // do nothing + } + + public void TranslationLayerDiscoveryStart() + { + // do nothing + } + + public void TranslationLayerDiscoveryStop() + { + // do nothing + } + + public void TranslationLayerExecutionStart(long customTestHost, long sourcesCount, long testCasesCount, string runSettings) + { + // do nothing + } + + public void TranslationLayerExecutionStop() + { + // do nothing + } + + public void TranslationLayerInitializeStart() + { + // do nothing + } + + public void TranslationLayerInitializeStop() + { + // do nothing + } + + public void TranslationLayerStartTestSessionStart() + { + // do nothing + } + + public void TranslationLayerStartTestSessionStop() + { + // do nothing + } + + public void TranslationLayerStopTestSessionStart() + { + // do nothing + } + + public void TranslationLayerStopTestSessionStop() + { + // do nothing + } + + public void TranslationLayerTestRunAttachmentsProcessingStart() + { + // do nothing + } + + public void TranslationLayerTestRunAttachmentsProcessingStop() + { + // do nothing + } + + public void VsTestConsoleStart() + { + // do nothing + } + + public void VsTestConsoleStop() + { + // do nothing + } +} diff --git a/test/vstest.ProgrammerTests/Fakes/FakeTestRunEventsRegistrar.cs b/test/vstest.ProgrammerTests/Fakes/FakeTestRunEventsRegistrar.cs new file mode 100644 index 0000000000..c7d6b2ce2a --- /dev/null +++ b/test/vstest.ProgrammerTests/Fakes/FakeTestRunEventsRegistrar.cs @@ -0,0 +1,79 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using Microsoft.VisualStudio.TestPlatform.Common.Interfaces; +using Microsoft.VisualStudio.TestPlatform.ObjectModel.Client; +using Microsoft.VisualStudio.TestPlatform.ObjectModel.Logging; + +#pragma warning disable IDE1006 // Naming Styles +namespace vstest.ProgrammerTests.CommandLine.Fakes; + +internal class FakeTestRunEventsRegistrar : ITestRunEventsRegistrar +{ + public List AllEvents { get; } = new(); + public List Warnings { get; } = new(); + public List> RunCompletionEvents { get; } = new(); + public List> RunStartEvents { get; } = new(); + public List> RunStatsChange { get; } = new(); + public List> RawMessageEvents { get; } = new(); + public List> TestRunMessageEvents { get; } = new(); + + public void LogWarning(string message) + { + AllEvents.Add(message); + Warnings.Add(message); + } + + public void RegisterTestRunEvents(ITestRunRequest testRunRequest) + { + testRunRequest.TestRunMessage += OnTestRunMessage; + testRunRequest.OnRawMessageReceived += OnRawMessage; + testRunRequest.OnRunStart += OnRunStart; + testRunRequest.OnRunStatsChange += OnRunStatsChange; + testRunRequest.OnRunCompletion += OnRunCompletion; + } + + public void UnregisterTestRunEvents(ITestRunRequest testRunRequest) + { + testRunRequest.TestRunMessage -= OnTestRunMessage; + testRunRequest.OnRawMessageReceived -= OnRawMessage; + testRunRequest.OnRunStart -= OnRunStart; + testRunRequest.OnRunStatsChange -= OnRunStatsChange; + testRunRequest.OnRunCompletion -= OnRunCompletion; + } + + private void OnRunCompletion(object? sender, TestRunCompleteEventArgs e) + { + var eventRecord = new EventRecord(sender, e); + AllEvents.Add(eventRecord); + RunCompletionEvents.Add(eventRecord); + } + + private void OnRunStart(object? sender, TestRunStartEventArgs e) + { + var eventRecord = new EventRecord(sender, e); + AllEvents.Add(eventRecord); + RunStartEvents.Add(eventRecord); + } + + private void OnRunStatsChange(object? sender, TestRunChangedEventArgs e) + { + var eventRecord = new EventRecord(sender, e); + AllEvents.Add(eventRecord); + RunStatsChange.Add(eventRecord); + } + + private void OnRawMessage(object? sender, string e) + { + var eventRecord = new EventRecord(sender, e); + AllEvents.Add(eventRecord); + RawMessageEvents.Add(eventRecord); + } + + private void OnTestRunMessage(object? sender, TestRunMessageEventArgs e) + { + var eventRecord = new EventRecord(sender, e); + AllEvents.Add(eventRecord); + TestRunMessageEvents.Add(eventRecord); + } +} diff --git a/test/vstest.ProgrammerTests/Fakes/FakeTestRuntimeProvider.cs b/test/vstest.ProgrammerTests/Fakes/FakeTestRuntimeProvider.cs new file mode 100644 index 0000000000..b74e1b9253 --- /dev/null +++ b/test/vstest.ProgrammerTests/Fakes/FakeTestRuntimeProvider.cs @@ -0,0 +1,81 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using Microsoft.VisualStudio.TestPlatform.ObjectModel; +using Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.Interfaces; +using Microsoft.VisualStudio.TestPlatform.ObjectModel.Host; +using Microsoft.VisualStudio.TestPlatform.ObjectModel.Logging; + +#pragma warning disable IDE1006 // Naming Styles +namespace vstest.ProgrammerTests.CommandLine.Fakes; + +internal class FakeTestRuntimeProvider : ITestRuntimeProvider +{ + public FakeProcessHelper FakeProcessHelper { get; } + public FakeProcess? TestHostProcess { get; private set; } + + // TODO: make this configurable? + public bool Shared => false; + + public event EventHandler HostLaunched; + public event EventHandler HostExited; + + public FakeTestRuntimeProvider(FakeProcessHelper fakeProcessHelper) + { + FakeProcessHelper = fakeProcessHelper; + } + + public bool CanExecuteCurrentRunConfiguration(string runsettingsXml) + { + return true; + } + + public Task CleanTestHostAsync(CancellationToken cancellationToken) + { + if (TestHostProcess == null) + throw new InvalidOperationException("Cannot clean testhost, no testhost process was started"); + FakeProcessHelper.TerminateProcess(TestHostProcess); + return Task.CompletedTask; + } + + public TestHostConnectionInfo GetTestHostConnectionInfo() + { + // TODO: Makes this configurable? + return new TestHostConnectionInfo + { + Endpoint = "127.0.0.0:8080", + Role = ConnectionRole.Client, + Transport = Transport.Sockets, + }; + } + + public TestProcessStartInfo GetTestHostProcessStartInfo(IEnumerable sources, IDictionary environmentVariables, TestRunnerConnectionInfo connectionInfo) + { + throw new NotImplementedException(); + } + + public IEnumerable GetTestPlatformExtensions(IEnumerable sources, IEnumerable extensions) + { + throw new NotImplementedException(); + } + + public IEnumerable GetTestSources(IEnumerable sources) + { + throw new NotImplementedException(); + } + + public void Initialize(IMessageLogger logger, string runsettingsXml) + { + // do nothing + } + + public Task LaunchTestHostAsync(TestProcessStartInfo testHostStartInfo, CancellationToken cancellationToken) + { + throw new NotImplementedException(); + } + + public void SetCustomLauncher(ITestHostLauncher customLauncher) + { + throw new NotImplementedException(); + } +} diff --git a/test/vstest.ProgrammerTests/Fakes/FakeTestRuntimeProviderManager.cs b/test/vstest.ProgrammerTests/Fakes/FakeTestRuntimeProviderManager.cs new file mode 100644 index 0000000000..e67fc29ad7 --- /dev/null +++ b/test/vstest.ProgrammerTests/Fakes/FakeTestRuntimeProviderManager.cs @@ -0,0 +1,28 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using Microsoft.VisualStudio.TestPlatform.Common.Hosting; +using Microsoft.VisualStudio.TestPlatform.ObjectModel.Host; + +#pragma warning disable IDE1006 // Naming Styles +namespace vstest.ProgrammerTests.CommandLine.Fakes; + +internal class FakeTestRuntimeProviderManager : ITestRuntimeProviderManager +{ + public FakeTestRuntimeProviderManager(FakeProcessHelper fakeProcessHelper) + { + TestRuntimeProvider = new FakeTestRuntimeProvider(fakeProcessHelper); + } + + public FakeTestRuntimeProvider TestRuntimeProvider { get; private set; } + + public ITestRuntimeProvider GetTestHostManagerByRunConfiguration(string runConfiguration) + { + return TestRuntimeProvider; + } + + public ITestRuntimeProvider GetTestHostManagerByUri(string hostUri) + { + throw new NotImplementedException(); + } +} diff --git a/test/vstest.ProgrammerTests/OutputMessage.cs b/test/vstest.ProgrammerTests/OutputMessage.cs new file mode 100644 index 0000000000..4ee4fbd94b --- /dev/null +++ b/test/vstest.ProgrammerTests/OutputMessage.cs @@ -0,0 +1,21 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using Microsoft.VisualStudio.TestPlatform.Utilities; + +#pragma warning disable IDE1006 // Naming Styles +namespace vstest.ProgrammerTests.CommandLine; + +internal class OutputMessage +{ + public OutputMessage(string message, OutputLevel level, bool isNewLine) + { + Message = message; + Level = level; + IsNewLine = isNewLine; + } + + public string Message { get; } + public OutputLevel Level { get; } + public bool IsNewLine { get; } +} diff --git a/test/vstest.ProgrammerTests/Properties/launchSettings.json b/test/vstest.ProgrammerTests/Properties/launchSettings.json index 33504c948a..7425c62a46 100644 --- a/test/vstest.ProgrammerTests/Properties/launchSettings.json +++ b/test/vstest.ProgrammerTests/Properties/launchSettings.json @@ -3,6 +3,9 @@ "WSL": { "commandName": "WSL2", "distributionName": "" + }, + "vstest.ProgrammerTests": { + "commandName": "Project" } } } \ No newline at end of file diff --git a/test/vstest.ProgrammerTests/StringExtensions.cs b/test/vstest.ProgrammerTests/StringExtensions.cs new file mode 100644 index 0000000000..b1406afd71 --- /dev/null +++ b/test/vstest.ProgrammerTests/StringExtensions.cs @@ -0,0 +1,22 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +namespace vstest.ProgrammerTests.CommandLine; + +internal static class StringExtensions +{ + public static string Join(this IEnumerable value, string separator) + { + return string.Join(separator, value); + } + + public static string JoinBySpace(this IEnumerable value) + { + return string.Join(" ", value); + } + + public static List ToList(this string value) + { + return new List { value }; + } +} diff --git a/test/vstest.ProgrammerTests/UnitTest1.cs b/test/vstest.ProgrammerTests/UnitTest1.cs index df55767bb9..791d7fcd88 100644 --- a/test/vstest.ProgrammerTests/UnitTest1.cs +++ b/test/vstest.ProgrammerTests/UnitTest1.cs @@ -1,24 +1,23 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT license. See LICENSE file in the project root for full license information. -using System.Text; + +using System.Runtime.Versioning; using FluentAssertions; using Microsoft.VisualStudio.TestPlatform.Client; using Microsoft.VisualStudio.TestPlatform.CommandLine; -using Microsoft.VisualStudio.TestPlatform.CommandLine.Processors; using Microsoft.VisualStudio.TestPlatform.CommandLine.Publisher; using Microsoft.VisualStudio.TestPlatform.CommandLine.TestPlatformHelpers; using Microsoft.VisualStudio.TestPlatform.CommandLineUtilities; -using Microsoft.VisualStudio.TestPlatform.Common.Hosting; -using Microsoft.VisualStudio.TestPlatform.CoreUtilities.Tracing.Interfaces; +using Microsoft.VisualStudio.TestPlatform.CommunicationUtilities.Interfaces; using Microsoft.VisualStudio.TestPlatform.CrossPlatEngine; +using Microsoft.VisualStudio.TestPlatform.CrossPlatEngine.TestRunAttachmentsProcessing; +using Microsoft.VisualStudio.TestPlatform.ObjectModel; using Microsoft.VisualStudio.TestPlatform.ObjectModel.Client; -using Microsoft.VisualStudio.TestPlatform.ObjectModel.Engine; -using Microsoft.VisualStudio.TestPlatform.PlatformAbstractions.Interfaces; -using Microsoft.VisualStudio.TestPlatform.Utilities; -using Microsoft.VisualStudio.TestPlatform.Utilities.Helpers.Interfaces; + +using vstest.ProgrammerTests.CommandLine.Fakes; #pragma warning disable IDE1006 // Naming Styles namespace vstest.ProgrammerTests.CommandLine; @@ -48,78 +47,67 @@ public class TestDiscoveryTests { public void GivenAnMSTestAssemblyWith5Tests_WhenTestsAreDiscovered_Then5TestsAreFound() { + // -- arrange var commandLineOptions = CommandLineOptions.Instance; - var fakeTestRuntimeProviderManager = new FakeTestRuntimeProviderManager(); - var fakeProcessHelper = new FakeProcessHelper(); + var fakeCurrentProcess = new FakeProcess(@"C:\temp\vstest.console.exe"); + var fakeProcessHelper = new FakeProcessHelper(fakeCurrentProcess); + + var fakeCommunicationEndpoint = new FakeCommunicationEndpoint(); + TestServiceLocator.Register(fakeCommunicationEndpoint); + var fakeTestRuntimeProviderManager = new FakeTestRuntimeProviderManager(fakeProcessHelper); var testEngine = new TestEngine(fakeTestRuntimeProviderManager, fakeProcessHelper); var fakeFileHelper = new FakeFileHelper(); var testPlatform = new TestPlatform(testEngine, fakeFileHelper, fakeTestRuntimeProviderManager); - var testRunResultAggregator = new TestRunResultAggregator(); + var testRunResultAggregator = new TestRunResultAggregator(); var fakeTestPlatformEventSource = new FakeTestPlatformEventSource(); - var fakeAssemblyMetadataProvider = new FakeAssemblyMetadataProvider(); + var fakeAssemblyMetadataProvider = new FakeAssemblyMetadataProvider(fakeFileHelper); var inferHelper = new InferHelper(fakeAssemblyMetadataProvider); - Task fakeMetricsPublisherTask = - TestRequestManager trm = new( + // This is most likely not the correctl place where to cut this off, plugin cache is probably the better place, + // but it is not injected, and I don't want to investigate this now. + var fakeDataCollectorAttachmentsProcessorsFactory = new FakeDataCollectorAttachmentsProcessorsFactory(); + var testRunAttachmentsProcessingManager = new TestRunAttachmentsProcessingManager(fakeTestPlatformEventSource, fakeDataCollectorAttachmentsProcessorsFactory); + + Task fakeMetricsPublisherTask = Task.FromResult(new FakeMetricsPublisher()); + TestRequestManager testRequestManager = new( commandLineOptions, testPlatform, testRunResultAggregator, fakeTestPlatformEventSource, inferHelper, - metricsPublisherTask, - + fakeMetricsPublisherTask, + fakeProcessHelper, + testRunAttachmentsProcessingManager ); + // -- act - } - -} - -internal class FakeAssemblyMetadataProvider : IAssemblyMetadataProvider -{ - public FakeAssemblyMetadataProvider() - { - } -} - -internal class FakeTestPlatformEventSource : ITestPlatformEventSource -{ - public FakeTestPlatformEventSource() - { - } -} - -internal class FakeFileHelper : IFileHelper -{ - public FakeFileHelper() - { - } -} + // TODO: Get framework name from constants + // TODO: have mstest1dll canned + var mstest1Dll = new FakeDllFile(@"C:\temp\mstest1.dll", new FrameworkName(".NETCoreApp,Version=v5.0"), Architecture.X64); + fakeFileHelper.AddFile(mstest1Dll); + var testRunRequestPayload = new TestRunRequestPayload + { + // TODO: passing null sources and null testcases does not fail fast + Sources = mstest1Dll.Path.ToList(), + // TODO: passing null runsettings does not fail fast, instead it fails in Fakes settings code + // TODO: passing empty string fails in the xml parser code + RunSettings = "" + }; -internal class FakeProcessHelper : IProcessHelper -{ - public FakeProcessHelper() - { - } -} + // var fakeTestHostLauncher = new FakeTestHostLauncher(); + var fakeTestRunEventsRegistrar = new FakeTestRunEventsRegistrar(); + var protocolConfig = new ProtocolConfig(); -internal class FakeTestRuntimeProviderManager : ITestRuntimeProviderManager -{ - public FakeTestRuntimeProviderManager() - { - } -} + testRequestManager.RunTests(testRunRequestPayload, testHostLauncher: null, fakeTestRunEventsRegistrar, protocolConfig); -internal class FakeTestHostProcess : FakeProcess -{ - public FakeTestHostProcess(string commandLine) : base(commandLine) - { + // -- assert + // fakeTestRunEventsRegistrar.RunTests.Should().HaveCount(2); } - public CapturedRunSettings? RunSettings { get; internal set; } } internal class Fixture : IDisposable @@ -141,29 +129,6 @@ public void Dispose() } } -internal class FakeTestExtensionManager : ITestExtensionManager -{ - public void ClearExtensions() - { - throw new NotImplementedException(); - } - - public void UseAdditionalExtensions(IEnumerable pathToAdditionalExtensions, bool skipExtensionFilters) - { - throw new NotImplementedException(); - } -} - -internal class FakeProcess -{ - public string CommandLine { get; } - - public FakeProcess(string commandLine) - { - CommandLine = commandLine; - } -} - internal class CapturedRunSettings @@ -171,43 +136,6 @@ internal class CapturedRunSettings public int MaxParallelLevel { get; internal set; } } -internal class FakeOutput : IOutput -{ - public FakeOutput() - { - } - - public List Messages { get; } = new(); - public StringBuilder CurrentLine { get; } = new(); - public List Lines { get; } = new(); - - public void Write(string message, OutputLevel level) - { - Messages.Add(new OutputMessage(message, level, isNewLine: false)); - CurrentLine.Append(message); - } - - public void WriteLine(string message, OutputLevel level) - { - Lines.Add(CurrentLine + message); - CurrentLine.Clear(); - } -} - -internal class OutputMessage -{ - public OutputMessage(string message, OutputLevel level, bool isNewLine) - { - Message = message; - Level = level; - IsNewLine = isNewLine; - } - - public string Message { get; } - public OutputLevel Level { get; } - public bool IsNewLine { get; } -} - internal static class RunConfiguration { public static ConfigurationEntry MaxParallelLevel { get; } = new(nameof(MaxParallelLevel)); diff --git a/test/vstest.ProgrammerTests/VstestConsole.cs b/test/vstest.ProgrammerTests/VstestConsole.cs index ce2cd698e5..e49ffa3315 100644 --- a/test/vstest.ProgrammerTests/VstestConsole.cs +++ b/test/vstest.ProgrammerTests/VstestConsole.cs @@ -3,6 +3,8 @@ using Microsoft.VisualStudio.TestPlatform.CommandLine; +using vstest.ProgrammerTests.CommandLine.Fakes; + namespace vstest.ProgrammerTests.CommandLine; internal class VstestConsole @@ -32,16 +34,3 @@ internal void Execute() var console = new Executor(Output).Execute(commandLine); } } - -internal static class StringExtensions -{ - public static string Join(this IEnumerable value, string separator) - { - return string.Join(separator, value); - } - - public static string JoinBySpace(this IEnumerable value) - { - return string.Join(" ", value); - } -} From 308ccd2c4196eec7f00fadcd2ae29b7fc79d1570 Mon Sep 17 00:00:00 2001 From: nohwnd Date: Mon, 7 Feb 2022 23:55:11 +0100 Subject: [PATCH 03/32] More code --- .../Communication/ConnectedEventArgs.cs | 1 + .../TestRequestSender.cs | 9 +- .../Client/ProxyOperationManager.cs | 3 + .../Fakes/FakeCommunicationChannel.cs | 92 +++++++++++++++++++ .../Fakes/FakeCommunicationEndpoint.cs | 10 +- .../Fakes/FakeProcess.cs | 2 + .../Fakes/FakeTestRuntimeProvider.cs | 16 +++- .../Fakes/FakeTestRuntimeProviderManager.cs | 4 +- test/vstest.ProgrammerTests/UnitTest1.cs | 2 +- .../vstest.ProgrammerTests.csproj | 1 + 10 files changed, 129 insertions(+), 11 deletions(-) create mode 100644 test/vstest.ProgrammerTests/Fakes/FakeCommunicationChannel.cs diff --git a/src/Microsoft.TestPlatform.CommunicationUtilities/Interfaces/Communication/ConnectedEventArgs.cs b/src/Microsoft.TestPlatform.CommunicationUtilities/Interfaces/Communication/ConnectedEventArgs.cs index 24ba6292fc..148cc2c348 100644 --- a/src/Microsoft.TestPlatform.CommunicationUtilities/Interfaces/Communication/ConnectedEventArgs.cs +++ b/src/Microsoft.TestPlatform.CommunicationUtilities/Interfaces/Communication/ConnectedEventArgs.cs @@ -13,6 +13,7 @@ public class ConnectedEventArgs : EventArgs /// /// Initializes a new instance of the class. /// + // TODO: Do we need this constructor? public ConnectedEventArgs() { } diff --git a/src/Microsoft.TestPlatform.CommunicationUtilities/TestRequestSender.cs b/src/Microsoft.TestPlatform.CommunicationUtilities/TestRequestSender.cs index eaf114df66..56fe1f2ff3 100644 --- a/src/Microsoft.TestPlatform.CommunicationUtilities/TestRequestSender.cs +++ b/src/Microsoft.TestPlatform.CommunicationUtilities/TestRequestSender.cs @@ -150,6 +150,12 @@ public int InitializeCommunication() _communicationEndpoint.Connected += (sender, args) => { _channel = args.Channel; + // TODO: I suspect that Channel can be null only because of some unit tests, + // and being connected and actually not setting any channel should be error + // rather than silently waiting for timeout + // TODO: also this event is called back on connected, why are the event args holding + // the Connected boolean and why do we check it here. If we did not connect we should + // have not fired this event. if (args.Connected && _channel != null) { _connected.Set(); @@ -194,7 +200,8 @@ public void CheckVersionWithTestHost() { // Negotiation follows these steps: // Runner sends highest supported version to Test host - // Test host sends the version it can support (must be less than highest) to runner + // Test host compares the version with the highest version it can support. + // Test host sends back the lower number of the two. So the highest protocol version, that both sides support is used. // Error case: test host can send a protocol error if it cannot find a supported version var protocolNegotiated = new ManualResetEvent(false); _onMessageReceived = (sender, args) => diff --git a/src/Microsoft.TestPlatform.CrossPlatEngine/Client/ProxyOperationManager.cs b/src/Microsoft.TestPlatform.CrossPlatEngine/Client/ProxyOperationManager.cs index 3c3f20f510..d71363505c 100644 --- a/src/Microsoft.TestPlatform.CrossPlatEngine/Client/ProxyOperationManager.cs +++ b/src/Microsoft.TestPlatform.CrossPlatEngine/Client/ProxyOperationManager.cs @@ -410,6 +410,9 @@ private void CompatIssueWithVersionCheckAndRunsettings() { var properties = TestHostManager.GetType().GetRuntimeProperties(); + // The field is actually defaulting to true, so this is just a complicated way to set or not set + // this to true (modern testhosts shoul have it set to true). Bad thing about this is that we are checking + // internal "undocumented" property. Good thing is that if you don't implement it you get the modern behavior. var versionCheckProperty = properties.FirstOrDefault(p => string.Equals(p.Name, _versionCheckPropertyName, StringComparison.OrdinalIgnoreCase)); if (versionCheckProperty != null) { diff --git a/test/vstest.ProgrammerTests/Fakes/FakeCommunicationChannel.cs b/test/vstest.ProgrammerTests/Fakes/FakeCommunicationChannel.cs new file mode 100644 index 0000000000..6a3e673624 --- /dev/null +++ b/test/vstest.ProgrammerTests/Fakes/FakeCommunicationChannel.cs @@ -0,0 +1,92 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using System.Collections.Concurrent; + +using Microsoft.VisualStudio.TestPlatform.CommunicationUtilities; +using Microsoft.VisualStudio.TestPlatform.CommunicationUtilities.Interfaces; +using Microsoft.VisualStudio.TestPlatform.CommunicationUtilities.ObjectModel; + +using Newtonsoft.Json; + +namespace vstest.ProgrammerTests.Fakes; + +internal class FakeCommunicationChannel : ICommunicationChannel +{ + // The naming here is a bit confusing when this is implemented in process. + // Normally one side in one process would have one end of the communication channel, + // and would use Send to pass message to another process. The other side would get notified + // about new data by NotifyDataAvailable, read the data there, and send them to other consumers + // using MessageReceived event. These consumers would then call Send, the channel and post the data + // to the other process. The other process would recieve the data and be notified by MessageReceived. + // + // But when we are in the same process, one side sends data using Send. We recieve them here by reading + // them from the queue (NotifyDataAvailable is not used, because we monitor the queue directly here, instead of + // in communication manager). And then we reply back to the sender, by invoking MessageReceived. + + public BlockingCollection InQueue { get; } = new(); + public Task Spin { get; } + + public CancellationTokenSource CancellationTokenSource = new(); + + public event EventHandler? MessageReceived; + + public FakeCommunicationChannel() + { + Spin = Task.Run(async () => + { + var token = CancellationTokenSource.Token; + while (!token.IsCancellationRequested) + { + try + { + // Only consume messages if someone is listening on the other side. + if (MessageReceived != null) + { + var rawMessage = InQueue.Take(token); + var message = JsonDataSerializer.Instance.DeserializeMessage(rawMessage); + if (message.MessageType == MessageType.VersionCheck) + { + // VersionCheck message expects a response with VersionCheck message, + // correctly the message will contain number that is the same or lower than the received number, + // so we use the highest version of protocol that both sides support + // TODO: I am just replying with the same message. + // + // Notifiy the listening side that there is new data. + MessageReceived(this, new MessageReceivedEventArgs { Data = rawMessage }); + } + + + } + else + { + await Task.Delay(100); + } + } + catch + { + + } + } + + }, CancellationTokenSource.Token); + } + + public void Dispose() + { + CancellationTokenSource.Cancel(); + InQueue.CompleteAdding(); + } + + public Task NotifyDataAvailable() + { + // This is used only by communication manager. vstest.console does not use communication manager. + throw new NotImplementedException(); + } + + public Task Send(string data) + { + InQueue.Add(data); + return Task.CompletedTask; + } +} diff --git a/test/vstest.ProgrammerTests/Fakes/FakeCommunicationEndpoint.cs b/test/vstest.ProgrammerTests/Fakes/FakeCommunicationEndpoint.cs index 279f3142e8..4db622fa29 100644 --- a/test/vstest.ProgrammerTests/Fakes/FakeCommunicationEndpoint.cs +++ b/test/vstest.ProgrammerTests/Fakes/FakeCommunicationEndpoint.cs @@ -2,6 +2,9 @@ // Licensed under the MIT license. See LICENSE file in the project root for full license information. using Microsoft.VisualStudio.TestPlatform.CommunicationUtilities.Interfaces; +using Microsoft.VisualStudio.TestPlatform.Utilities; + +using vstest.ProgrammerTests.Fakes; namespace vstest.ProgrammerTests.CommandLine; @@ -11,12 +14,13 @@ public FakeCommunicationEndpoint() { } - public event EventHandler Connected; - public event EventHandler Disconnected; + public event EventHandler? Connected; + public event EventHandler? Disconnected; public string Start(string endPoint) { - Connected?.Invoke(this, new ConnectedEventArgs()); + // TODO: insert this from the outside so some channel manager can give us overview of the open channels? + Connected?.SafeInvoke(this, new ConnectedEventArgs(new FakeCommunicationChannel()), "FakeCommunicationEndpoint.Start"); return endPoint; } diff --git a/test/vstest.ProgrammerTests/Fakes/FakeProcess.cs b/test/vstest.ProgrammerTests/Fakes/FakeProcess.cs index 49469009b5..c6c5839169 100644 --- a/test/vstest.ProgrammerTests/Fakes/FakeProcess.cs +++ b/test/vstest.ProgrammerTests/Fakes/FakeProcess.cs @@ -3,6 +3,7 @@ #pragma warning disable IDE1006 // Naming Styles +using Microsoft.VisualStudio.TestPlatform.ObjectModel.Host; using Microsoft.VisualStudio.TestPlatform.PlatformAbstractions; namespace vstest.ProgrammerTests.CommandLine.Fakes; @@ -15,6 +16,7 @@ internal class FakeProcess public string Arguments { get; set; } public PlatformArchitecture Architecture { get; init; } = PlatformArchitecture.X64; + public event EventHandler ProcessExited = delegate { }; public FakeProcess(string path, string arguments = null) { diff --git a/test/vstest.ProgrammerTests/Fakes/FakeTestRuntimeProvider.cs b/test/vstest.ProgrammerTests/Fakes/FakeTestRuntimeProvider.cs index b74e1b9253..12449d85ca 100644 --- a/test/vstest.ProgrammerTests/Fakes/FakeTestRuntimeProvider.cs +++ b/test/vstest.ProgrammerTests/Fakes/FakeTestRuntimeProvider.cs @@ -12,6 +12,7 @@ namespace vstest.ProgrammerTests.CommandLine.Fakes; internal class FakeTestRuntimeProvider : ITestRuntimeProvider { public FakeProcessHelper FakeProcessHelper { get; } + public FakeCommunicationEndpoint FakeCommunicationEndpoint { get; } public FakeProcess? TestHostProcess { get; private set; } // TODO: make this configurable? @@ -20,9 +21,10 @@ internal class FakeTestRuntimeProvider : ITestRuntimeProvider public event EventHandler HostLaunched; public event EventHandler HostExited; - public FakeTestRuntimeProvider(FakeProcessHelper fakeProcessHelper) + public FakeTestRuntimeProvider(FakeProcessHelper fakeProcessHelper, FakeCommunicationEndpoint fakeCommunicationEndpoint) { FakeProcessHelper = fakeProcessHelper; + FakeCommunicationEndpoint = fakeCommunicationEndpoint; } public bool CanExecuteCurrentRunConfiguration(string runsettingsXml) @@ -43,7 +45,8 @@ public TestHostConnectionInfo GetTestHostConnectionInfo() // TODO: Makes this configurable? return new TestHostConnectionInfo { - Endpoint = "127.0.0.0:8080", + // using 9090 for no particular reason, apart from knowing that port 0 is ignored somewhere in our codebase + Endpoint = "127.0.0.1:9090", Role = ConnectionRole.Client, Transport = Transport.Sockets, }; @@ -51,7 +54,8 @@ public TestHostConnectionInfo GetTestHostConnectionInfo() public TestProcessStartInfo GetTestHostProcessStartInfo(IEnumerable sources, IDictionary environmentVariables, TestRunnerConnectionInfo connectionInfo) { - throw new NotImplementedException(); + // TODO: do we need to do more here? How to link testhost to the fake one we "start"? + return new TestProcessStartInfo(); } public IEnumerable GetTestPlatformExtensions(IEnumerable sources, IEnumerable extensions) @@ -71,7 +75,11 @@ public void Initialize(IMessageLogger logger, string runsettingsXml) public Task LaunchTestHostAsync(TestProcessStartInfo testHostStartInfo, CancellationToken cancellationToken) { - throw new NotImplementedException(); + // todo: Add fake process for testhost + // TestHostProcess = FakeProcessHelper.LaunchProcess(testHostStartInfo); + //HostLaunched(this, new HostProviderEventArgs("Fake testhost launched", 0, TestHostProcess.Id)); + //TestHostProcess.ProcessExited += (s, e) => HostExited(s, new HostProviderEventArgs($"Fake testhost {TestHostProcess.Id} exited", -99999, e)); + return Task.FromResult(true); } public void SetCustomLauncher(ITestHostLauncher customLauncher) diff --git a/test/vstest.ProgrammerTests/Fakes/FakeTestRuntimeProviderManager.cs b/test/vstest.ProgrammerTests/Fakes/FakeTestRuntimeProviderManager.cs index e67fc29ad7..aab1d3554d 100644 --- a/test/vstest.ProgrammerTests/Fakes/FakeTestRuntimeProviderManager.cs +++ b/test/vstest.ProgrammerTests/Fakes/FakeTestRuntimeProviderManager.cs @@ -9,9 +9,9 @@ namespace vstest.ProgrammerTests.CommandLine.Fakes; internal class FakeTestRuntimeProviderManager : ITestRuntimeProviderManager { - public FakeTestRuntimeProviderManager(FakeProcessHelper fakeProcessHelper) + public FakeTestRuntimeProviderManager(FakeProcessHelper fakeProcessHelper, FakeCommunicationEndpoint fakeCommunicationEndpoint) { - TestRuntimeProvider = new FakeTestRuntimeProvider(fakeProcessHelper); + TestRuntimeProvider = new FakeTestRuntimeProvider(fakeProcessHelper, fakeCommunicationEndpoint); } public FakeTestRuntimeProvider TestRuntimeProvider { get; private set; } diff --git a/test/vstest.ProgrammerTests/UnitTest1.cs b/test/vstest.ProgrammerTests/UnitTest1.cs index 791d7fcd88..4c84633e4b 100644 --- a/test/vstest.ProgrammerTests/UnitTest1.cs +++ b/test/vstest.ProgrammerTests/UnitTest1.cs @@ -55,7 +55,7 @@ public void GivenAnMSTestAssemblyWith5Tests_WhenTestsAreDiscovered_Then5TestsAre var fakeCommunicationEndpoint = new FakeCommunicationEndpoint(); TestServiceLocator.Register(fakeCommunicationEndpoint); - var fakeTestRuntimeProviderManager = new FakeTestRuntimeProviderManager(fakeProcessHelper); + var fakeTestRuntimeProviderManager = new FakeTestRuntimeProviderManager(fakeProcessHelper, fakeCommunicationEndpoint); var testEngine = new TestEngine(fakeTestRuntimeProviderManager, fakeProcessHelper); var fakeFileHelper = new FakeFileHelper(); var testPlatform = new TestPlatform(testEngine, fakeFileHelper, fakeTestRuntimeProviderManager); diff --git a/test/vstest.ProgrammerTests/vstest.ProgrammerTests.csproj b/test/vstest.ProgrammerTests/vstest.ProgrammerTests.csproj index 648d985bc9..21a59d0ab8 100644 --- a/test/vstest.ProgrammerTests/vstest.ProgrammerTests.csproj +++ b/test/vstest.ProgrammerTests/vstest.ProgrammerTests.csproj @@ -19,6 +19,7 @@ + true From 69ee04e41e938a469b0b26e08bdc8c4f5c3d80b8 Mon Sep 17 00:00:00 2001 From: nohwnd Date: Tue, 8 Feb 2022 00:39:29 +0100 Subject: [PATCH 04/32] Add response pairs --- .../Fakes/FakeCommunicationChannel.cs | 53 ++++++++++++++++++- .../Fakes/FakeTestRuntimeProvider.cs | 7 ++- 2 files changed, 56 insertions(+), 4 deletions(-) diff --git a/test/vstest.ProgrammerTests/Fakes/FakeCommunicationChannel.cs b/test/vstest.ProgrammerTests/Fakes/FakeCommunicationChannel.cs index 6a3e673624..1aacb766bf 100644 --- a/test/vstest.ProgrammerTests/Fakes/FakeCommunicationChannel.cs +++ b/test/vstest.ProgrammerTests/Fakes/FakeCommunicationChannel.cs @@ -2,6 +2,7 @@ // Licensed under the MIT license. See LICENSE file in the project root for full license information. using System.Collections.Concurrent; +using System.Linq.Expressions; using Microsoft.VisualStudio.TestPlatform.CommunicationUtilities; using Microsoft.VisualStudio.TestPlatform.CommunicationUtilities.Interfaces; @@ -27,12 +28,22 @@ internal class FakeCommunicationChannel : ICommunicationChannel public BlockingCollection InQueue { get; } = new(); public Task Spin { get; } + public List UnknowsMessages { get; } = new(); + + public List> ProcessedMessages { get; } = new(); + + /// + /// Queue of MessageType of the incoming request, and the response that will be sent back. + /// + public Queue> NextResponses { get; } = new(); + public CancellationTokenSource CancellationTokenSource = new(); public event EventHandler? MessageReceived; public FakeCommunicationChannel() { + NextResponses.Enqueue(new RequestResponsePair(MessageType.VersionCheck, new Message(MessageType.VersionCheck, 1))); Spin = Task.Run(async () => { var token = CancellationTokenSource.Token; @@ -53,9 +64,12 @@ public FakeCommunicationChannel() // TODO: I am just replying with the same message. // // Notifiy the listening side that there is new data. - MessageReceived(this, new MessageReceivedEventArgs { Data = rawMessage }); + MessageReceived(this, new MessageReceivedEventArgs { Data = rawMessage }); + } + else + { + throw new InvalidOperationException($"unknown message, {message.MessageType}, {message.Payload}"); } - } else @@ -90,3 +104,38 @@ public Task Send(string data) return Task.CompletedTask; } } + +internal class RequestResponsePair +{ + public RequestResponsePair(T request, U response) + { + Request = request; + Response = response; + } + + public RequestResponsePair(T request, Expression> responseFactory) + { + Request = request; + ResponseFactory = responseFactory; + } + + public T Request { get; set; } + + // TODO: make this Expression ResponseFactory { get; } + public U? Response { get; set; } + + // TODO: Let's sleep on this and see if I understand tomorrow what I was trying to do, because this has too many usages now... + public GetResponse(T request = null) + { + if (ResponseFactory != null) + { + Response = ResponseFactory(request ?? Request); + return Response; + } + else + { + return Response; + } + } +} diff --git a/test/vstest.ProgrammerTests/Fakes/FakeTestRuntimeProvider.cs b/test/vstest.ProgrammerTests/Fakes/FakeTestRuntimeProvider.cs index 12449d85ca..43e60e3720 100644 --- a/test/vstest.ProgrammerTests/Fakes/FakeTestRuntimeProvider.cs +++ b/test/vstest.ProgrammerTests/Fakes/FakeTestRuntimeProvider.cs @@ -60,12 +60,15 @@ public TestProcessStartInfo GetTestHostProcessStartInfo(IEnumerable sour public IEnumerable GetTestPlatformExtensions(IEnumerable sources, IEnumerable extensions) { - throw new NotImplementedException(); + // send extensions so we send InitializeExecutionMessage + return new[] { @"c:\temp\extension.dll" }; } public IEnumerable GetTestSources(IEnumerable sources) { - throw new NotImplementedException(); + // gives testhost opportunity to translate sources to something else, + // e.g. in uwp the main exe is returned, rather than the dlls that dlls that are tested + return sources; } public void Initialize(IMessageLogger logger, string runsettingsXml) From eb37be0da5da7e3afeb34f5056a0f1fa48617bb0 Mon Sep 17 00:00:00 2001 From: nohwnd Date: Tue, 8 Feb 2022 12:51:54 +0100 Subject: [PATCH 05/32] respond me --- .../Messages/Message.cs | 5 + .../Fakes/FakeCommunicationChannel.cs | 141 ++++++++++++++---- 2 files changed, 121 insertions(+), 25 deletions(-) diff --git a/src/Microsoft.TestPlatform.CommunicationUtilities/Messages/Message.cs b/src/Microsoft.TestPlatform.CommunicationUtilities/Messages/Message.cs index 3c9bae3b54..2d27477efd 100644 --- a/src/Microsoft.TestPlatform.CommunicationUtilities/Messages/Message.cs +++ b/src/Microsoft.TestPlatform.CommunicationUtilities/Messages/Message.cs @@ -19,6 +19,9 @@ public class Message /// /// Gets or sets the payload. /// + // TODO: Our public contract says that we should be able to communicate over JSON, but we should not be stopping ourselves from + // negotiating a different protocol. Or using a different serialization library than NewtonsoftJson. Check why this is published as JToken + // and not as a string. public JToken Payload { get; set; } /// @@ -27,6 +30,8 @@ public class Message /// The . public override string ToString() { + // TODO: Review where this is used, we should avoid extensive serialization and deserialization, + // and this might be happening in multiple places that are not the edge of our process. return "(" + MessageType + ") -> " + (Payload == null ? "null" : Payload.ToString(Formatting.Indented)); } } diff --git a/test/vstest.ProgrammerTests/Fakes/FakeCommunicationChannel.cs b/test/vstest.ProgrammerTests/Fakes/FakeCommunicationChannel.cs index 1aacb766bf..1e0f6809dc 100644 --- a/test/vstest.ProgrammerTests/Fakes/FakeCommunicationChannel.cs +++ b/test/vstest.ProgrammerTests/Fakes/FakeCommunicationChannel.cs @@ -2,13 +2,11 @@ // Licensed under the MIT license. See LICENSE file in the project root for full license information. using System.Collections.Concurrent; -using System.Linq.Expressions; using Microsoft.VisualStudio.TestPlatform.CommunicationUtilities; using Microsoft.VisualStudio.TestPlatform.CommunicationUtilities.Interfaces; using Microsoft.VisualStudio.TestPlatform.CommunicationUtilities.ObjectModel; - -using Newtonsoft.Json; +using Microsoft.VisualStudio.TestPlatform.ObjectModel.Logging; namespace vstest.ProgrammerTests.Fakes; @@ -28,14 +26,19 @@ internal class FakeCommunicationChannel : ICommunicationChannel public BlockingCollection InQueue { get; } = new(); public Task Spin { get; } + /// + /// True if we encountered unexpected message (e.g. unknown message, message out of order) or when we sent all our prepared responses, and there were still more requests coming. + /// + public bool Faulted { get; private set; } + public List UnknowsMessages { get; } = new(); - public List> ProcessedMessages { get; } = new(); + public List> ProcessedMessages { get; } = new(); /// /// Queue of MessageType of the incoming request, and the response that will be sent back. /// - public Queue> NextResponses { get; } = new(); + public Queue> NextResponses { get; } = new(); public CancellationTokenSource CancellationTokenSource = new(); @@ -43,7 +46,8 @@ internal class FakeCommunicationChannel : ICommunicationChannel public FakeCommunicationChannel() { - NextResponses.Enqueue(new RequestResponsePair(MessageType.VersionCheck, new Message(MessageType.VersionCheck, 1))); + NextResponses.Enqueue(new RequestResponsePair(MessageType.VersionCheck, new FakeMessage(MessageType.VersionCheck, 5))); + Spin = Task.Run(async () => { var token = CancellationTokenSource.Token; @@ -55,22 +59,53 @@ public FakeCommunicationChannel() if (MessageReceived != null) { var rawMessage = InQueue.Take(token); - var message = JsonDataSerializer.Instance.DeserializeMessage(rawMessage); - if (message.MessageType == MessageType.VersionCheck) + var requestMessage = JsonDataSerializer.Instance.DeserializeMessage(rawMessage); + + if (Faulted) { - // VersionCheck message expects a response with VersionCheck message, - // correctly the message will contain number that is the same or lower than the received number, - // so we use the highest version of protocol that both sides support - // TODO: I am just replying with the same message. + // We already failed, when there are more requests coming, just save them and respond with error. We want to avoid + // a situation where server ignores our error message and responds with another request, for which we accidentally + // have the right answer in queue. + // + // E.g. We have VersionCheck, TestRunStart prepared, and server sends: VersionCheck, TestInitialize, TestRunStart. + // The first request has a valid response. The next TestInitialize does not have a valid response and errors out, + // but the server ignores it, and sends TestRunStart, which would normally have a prepared response, and lead to + // possibly overlooking the error response to TestInitialize. // - // Notifiy the listening side that there is new data. - MessageReceived(this, new MessageReceivedEventArgs { Data = rawMessage }); + // With this check in place we will not respond to TestRunStart with success, but with error. + // TODO: Better way to map MessageType and the payload type. + var errorResponse = new FakeMessage(MessageType.TestMessage, new TestMessagePayload { MessageLevel = TestMessageLevel.Error, Message = "FakeCommunicationChannel: Channel is faulted." }); + ProcessedMessages.Add(new RequestResponsePair(requestMessage, errorResponse)); + Faulted = true; + Respond(errorResponse); + } + + // Just peek at it so we can keep the message on the the queue in case of error. + if (!NextResponses.TryPeek(out var nextResponsePair)) + { + // If there are no more prepared responses then return protocol error. + var errorResponse = new FakeMessage(MessageType.ProtocolError, new TestMessagePayload { MessageLevel = TestMessageLevel.Error, Message = "FakeCommunicationChannel: No more responses are available." }); + ProcessedMessages.Add(new RequestResponsePair(requestMessage, errorResponse)); + Faulted = true; + Respond(errorResponse); + } + else if (nextResponsePair.Request != requestMessage.MessageType) + { + // If the incoming message does not match what we expected return protocol error. The lsat message will remain in the + // NextResponses queue. + var errorResponse = new FakeMessage(MessageType.ProtocolError, new TestMessagePayload { MessageLevel = TestMessageLevel.Error, Message = $"FakeCommunicationChannel: Excpected message {nextResponsePair.Request} but got {requestMessage.MessageType}." }); + ProcessedMessages.Add(new RequestResponsePair(requestMessage, errorResponse)); + Faulted = true; + Respond(errorResponse); } else { - throw new InvalidOperationException($"unknown message, {message.MessageType}, {message.Payload}"); + var responsePair = NextResponses.Dequeue(); + // TODO: remove !, once we fix the type + var response = responsePair.Response!; + ProcessedMessages.Add(new RequestResponsePair(requestMessage, response)); + Respond(response); } - } else { @@ -86,6 +121,12 @@ public FakeCommunicationChannel() }, CancellationTokenSource.Token); } + private void Respond(FakeMessage response) + { + // TODO: we never call this when MessageRecieved is null, but how do I tell that to the compiler? + MessageReceived!(this, new MessageReceivedEventArgs { Data = response.SerializedMessage }); + } + public void Dispose() { CancellationTokenSource.Cancel(); @@ -105,7 +146,7 @@ public Task Send(string data) } } -internal class RequestResponsePair +internal class RequestResponsePair where T : class { public RequestResponsePair(T request, U response) { @@ -113,29 +154,79 @@ public RequestResponsePair(T request, U response) Response = response; } - public RequestResponsePair(T request, Expression> responseFactory) + public RequestResponsePair(T request, Func responseFactory) { Request = request; ResponseFactory = responseFactory; } - public T Request { get; set; } + public T Request { get; } - // TODO: make this Expression ResponseFactory { get; } - public U? Response { get; set; } + // TODO: make this Expression? ResponseFactory { get; } + public U? Response { get; private set; } // TODO: Let's sleep on this and see if I understand tomorrow what I was trying to do, because this has too many usages now... - public GetResponse(T request = null) + // One day later I do remember it, but I am still not convinced. But let's keep that for now. The idea here is to get either a canned + // response or a response generated based on the incoming request (e.g. version comes in that is 3, response is lower version (2)). + // both of these could be done by just executing Func, but that is not readable during debug time. Or maybe some variation on Either<> + // but that seems as a very foreign concept to common C#. + public U GetResponse(T? request = null) { if (ResponseFactory != null) { - Response = ResponseFactory(request ?? Request); + // TODO: what am I doing wrong? (Why do I need that '!' ? I assign Request in both ctors and don't have setter, is the null coalescing not + // supposed to propagate non-nullable type to the target type? + // TODO: I don't like rewriting the response here. This is yet another sign that I am possibly mixing concepts in this class. + Response = ResponseFactory(request ?? Request!); return Response; } else { - return Response; + // TODO: split this class to two that has the same parent, so we are sure we have a response in any case. + return Response!; } } } + +/// +/// A class like Message / VersionedMessage that is easier to create and review during debugging. +/// +internal sealed class FakeMessage : FakeMessage +{ + public FakeMessage(string messageType, T payload, int version = 0) + { + MessageType = messageType; + Payload = payload; + Version = version; + SerializedMessage = JsonDataSerializer.Instance.SerializePayload(MessageType, payload, version); + } + + /// + /// Message identifier, usually coming from the MessageType class. + /// + public string MessageType { get; } + + /// + /// The payload that this message is holding. + /// + public T Payload { get; } + + /// + /// Version of the message to allow the internal serializer to choose the correct serialization strategy. + /// + public int Version { get; } +} + +/// +/// Marker for Fake message so we can put put all FakeMessages into one collection, without making it too wide. +/// +internal abstract class FakeMessage +{ + /// + /// The message serialized using the default JsonDataSerializer. + /// + // TODO: Is there a better way to ensure that is is not null, we will always set it in the inherited types, but it would be nice to have warning if we did not. + // And adding constructor makes it difficult to use the serializer, especially if we wanted to the serializer dynamic and not a static instance. + public string SerializedMessage { get; init; } = string.Empty; +} From 1a8090caf83262c8dc9507bab0816b9bed28909b Mon Sep 17 00:00:00 2001 From: nohwnd Date: Tue, 8 Feb 2022 13:56:29 +0100 Subject: [PATCH 06/32] Add comment --- .../DataCollectionRequestHandler.cs | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/Microsoft.TestPlatform.CommunicationUtilities/DataCollectionRequestHandler.cs b/src/Microsoft.TestPlatform.CommunicationUtilities/DataCollectionRequestHandler.cs index fef255311d..7e899fbe5f 100644 --- a/src/Microsoft.TestPlatform.CommunicationUtilities/DataCollectionRequestHandler.cs +++ b/src/Microsoft.TestPlatform.CommunicationUtilities/DataCollectionRequestHandler.cs @@ -150,6 +150,15 @@ public static DataCollectionRequestHandler Create( { ValidateArg.NotNull(communicationManager, nameof(communicationManager)); ValidateArg.NotNull(messageSink, nameof(messageSink)); + // TODO: The MessageSink and DataCollectionRequestHandler have circular dependency. + // Message sink is injected into this Create method and then into constructor + // and into the constructor of DataCollectionRequestHandler. Data collection manager + // is then assigned to .Instace (which unlike many other .Instance is not populated + // directly in that property, but is created here). And then MessageSink depends on + // the .Instance. This is a very complicated way of solving the circular dependency, + // and should be replaced by adding a property to Message and assigning it. + // .Instance can then be removed. + if (Instance == null) { lock (SyncObject) From 6053c8149d50018ca34e8790062ac575daadac22 Mon Sep 17 00:00:00 2001 From: nohwnd Date: Wed, 9 Feb 2022 11:39:56 +0100 Subject: [PATCH 07/32] stuff --- .../Execution/TestRunRequest.cs | 2 +- .../SocketClient.cs | 2 +- .../FakeErrorAggregator.cs | 14 +++++ .../Fakes/FakeAssemblyMetadataProvider.cs | 13 +++-- .../Fakes/FakeCommunicationChannel.cs | 52 +++++++++++++++---- .../Fakes/FakeCommunicationEndpoint.cs | 17 ++++-- ...taCollectorAttachmentsProcessorsFactory.cs | 7 +++ .../Fakes/FakeFileHelper.cs | 6 +++ .../Fakes/FakeMetricsPublisher.cs | 11 +++- .../Fakes/FakeProcess.cs | 6 ++- .../Fakes/FakeProcessHelper.cs | 5 +- .../Fakes/FakeTestHostProcess.cs | 2 +- .../Fakes/FakeTestPlatformEventSource.cs | 5 +- .../Fakes/FakeTestRunEventsRegistrar.cs | 6 +++ .../Fakes/FakeTestRuntimeProviderManager.cs | 4 +- test/vstest.ProgrammerTests/UnitTest1.cs | 32 +++++++----- 16 files changed, 143 insertions(+), 41 deletions(-) create mode 100644 test/vstest.ProgrammerTests/FakeErrorAggregator.cs diff --git a/src/Microsoft.TestPlatform.Client/Execution/TestRunRequest.cs b/src/Microsoft.TestPlatform.Client/Execution/TestRunRequest.cs index 05b14ccdcd..2b9bc78184 100644 --- a/src/Microsoft.TestPlatform.Client/Execution/TestRunRequest.cs +++ b/src/Microsoft.TestPlatform.Client/Execution/TestRunRequest.cs @@ -193,7 +193,7 @@ internal void OnTestSessionTimeout(object obj) HandleLogMessage(TestMessageLevel.Error, message); HandleRawMessage(rawMessage); - Abort(); + Abort(); } /// diff --git a/src/Microsoft.TestPlatform.CommunicationUtilities/SocketClient.cs b/src/Microsoft.TestPlatform.CommunicationUtilities/SocketClient.cs index f2f7332466..7dcaab3b42 100644 --- a/src/Microsoft.TestPlatform.CommunicationUtilities/SocketClient.cs +++ b/src/Microsoft.TestPlatform.CommunicationUtilities/SocketClient.cs @@ -110,7 +110,7 @@ private void OnServerConnected(Task connectAsyncTask) private void Stop(Exception error) { EqtTrace.Info("SocketClient.PrivateStop: Stop communication from server endpoint: {0}, error:{1}", _endPoint, error); - + // TODO: this is here to prevent stack overflow. if (!_stopped) { // Do not allow stop to be called multiple times. diff --git a/test/vstest.ProgrammerTests/FakeErrorAggregator.cs b/test/vstest.ProgrammerTests/FakeErrorAggregator.cs new file mode 100644 index 0000000000..db5f70b4f3 --- /dev/null +++ b/test/vstest.ProgrammerTests/FakeErrorAggregator.cs @@ -0,0 +1,14 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +namespace vstest.ProgrammerTests.CommandLine; + +internal class FakeErrorAggregator +{ + public List Errors { get; } = new(); + + public void Add(object error) + { + Errors.Add(error); + } +} diff --git a/test/vstest.ProgrammerTests/Fakes/FakeAssemblyMetadataProvider.cs b/test/vstest.ProgrammerTests/Fakes/FakeAssemblyMetadataProvider.cs index b6a723652a..c1443dfe62 100644 --- a/test/vstest.ProgrammerTests/Fakes/FakeAssemblyMetadataProvider.cs +++ b/test/vstest.ProgrammerTests/Fakes/FakeAssemblyMetadataProvider.cs @@ -11,22 +11,25 @@ namespace vstest.ProgrammerTests.CommandLine.Fakes; internal class FakeAssemblyMetadataProvider : IAssemblyMetadataProvider { - private readonly FakeFileHelper _fakeFileHelper; + public FakeFileHelper FakeFileHelper { get; } - public FakeAssemblyMetadataProvider(FakeFileHelper fakeFileHelper) + public FakeErrorAggregator FakeErrorAggregator { get; } + + public FakeAssemblyMetadataProvider(FakeFileHelper fakeFileHelper, FakeErrorAggregator fakeErrorAggregator) { - _fakeFileHelper = fakeFileHelper; + FakeFileHelper = fakeFileHelper; + FakeErrorAggregator = fakeErrorAggregator; } public Architecture GetArchitecture(string filePath) { - var file = (FakeDllFile)_fakeFileHelper.Files.Single(f => f.Path == filePath); + var file = (FakeDllFile)FakeFileHelper.Files.Single(f => f.Path == filePath); return file.Architecture; } public FrameworkName GetFrameWork(string filePath) { - var file = (FakeDllFile)_fakeFileHelper.Files.Single(f => f.Path == filePath); + var file = (FakeDllFile)FakeFileHelper.Files.Single(f => f.Path == filePath); return file.FrameworkName; } } diff --git a/test/vstest.ProgrammerTests/Fakes/FakeCommunicationChannel.cs b/test/vstest.ProgrammerTests/Fakes/FakeCommunicationChannel.cs index 1e0f6809dc..c7052651be 100644 --- a/test/vstest.ProgrammerTests/Fakes/FakeCommunicationChannel.cs +++ b/test/vstest.ProgrammerTests/Fakes/FakeCommunicationChannel.cs @@ -6,8 +6,12 @@ using Microsoft.VisualStudio.TestPlatform.CommunicationUtilities; using Microsoft.VisualStudio.TestPlatform.CommunicationUtilities.Interfaces; using Microsoft.VisualStudio.TestPlatform.CommunicationUtilities.ObjectModel; +using Microsoft.VisualStudio.TestPlatform.ObjectModel; +using Microsoft.VisualStudio.TestPlatform.ObjectModel.Client; using Microsoft.VisualStudio.TestPlatform.ObjectModel.Logging; +using vstest.ProgrammerTests.CommandLine; + namespace vstest.ProgrammerTests.Fakes; internal class FakeCommunicationChannel : ICommunicationChannel @@ -31,22 +35,30 @@ internal class FakeCommunicationChannel : ICommunicationChannel /// public bool Faulted { get; private set; } - public List UnknowsMessages { get; } = new(); - public List> ProcessedMessages { get; } = new(); /// /// Queue of MessageType of the incoming request, and the response that will be sent back. /// public Queue> NextResponses { get; } = new(); + public FakeErrorAggregator FakeErrorAggregator { get; } public CancellationTokenSource CancellationTokenSource = new(); public event EventHandler? MessageReceived; - public FakeCommunicationChannel() + public FakeCommunicationChannel(FakeErrorAggregator fakeErrorAggregator) { + FakeErrorAggregator = fakeErrorAggregator; + NextResponses.Enqueue(new RequestResponsePair(MessageType.VersionCheck, new FakeMessage(MessageType.VersionCheck, 5))); + NextResponses.Enqueue(new RequestResponsePair(MessageType.ExecutionInitialize, FakeMessage.NoResponse)); + var result = new TestRunCompletePayload + { + TestRunCompleteArgs = new TestRunCompleteEventArgs(new TestRunStatistics(), false, false, null, new System.Collections.ObjectModel.Collection(), TimeSpan.Zero), + LastRunTests = new TestRunChangedEventArgs(new TestRunStatistics(), new List(), new List()), + }; + NextResponses.Enqueue(new RequestResponsePair(MessageType.StartTestExecutionWithSources, new FakeMessage(MessageType.ExecutionComplete, result))); Spin = Task.Run(async () => { @@ -74,7 +86,11 @@ public FakeCommunicationChannel() // // With this check in place we will not respond to TestRunStart with success, but with error. // TODO: Better way to map MessageType and the payload type. - var errorResponse = new FakeMessage(MessageType.TestMessage, new TestMessagePayload { MessageLevel = TestMessageLevel.Error, Message = "FakeCommunicationChannel: Channel is faulted." }); + // TODO: simpler way to report error, and add it to the error aggregator + var errorMessage = $"FakeCommunicationChannel: FakeCommunicationChannel: Got message {requestMessage.MessageType}. But a message that was unexptected was received previously and the channel is now faulted. Review {nameof(ProcessedMessages)}, and {nameof(NextResponses)}."; + var exception = new Exception(errorMessage); + FakeErrorAggregator.Add(exception); + var errorResponse = new FakeMessage(MessageType.TestMessage, new TestMessagePayload { MessageLevel = TestMessageLevel.Error, Message = errorMessage }); ProcessedMessages.Add(new RequestResponsePair(requestMessage, errorResponse)); Faulted = true; Respond(errorResponse); @@ -84,7 +100,10 @@ public FakeCommunicationChannel() if (!NextResponses.TryPeek(out var nextResponsePair)) { // If there are no more prepared responses then return protocol error. - var errorResponse = new FakeMessage(MessageType.ProtocolError, new TestMessagePayload { MessageLevel = TestMessageLevel.Error, Message = "FakeCommunicationChannel: No more responses are available." }); + var errorMessage = $"FakeCommunicationChannel: Got message {requestMessage.MessageType}, but no more requests were expected, because there are no more responses in {nameof(NextResponses)}."; + var exception = new Exception(errorMessage); + FakeErrorAggregator.Add(exception); + var errorResponse = new FakeMessage(MessageType.ProtocolError, new TestMessagePayload { MessageLevel = TestMessageLevel.Error, Message = errorMessage }); ProcessedMessages.Add(new RequestResponsePair(requestMessage, errorResponse)); Faulted = true; Respond(errorResponse); @@ -93,7 +112,10 @@ public FakeCommunicationChannel() { // If the incoming message does not match what we expected return protocol error. The lsat message will remain in the // NextResponses queue. - var errorResponse = new FakeMessage(MessageType.ProtocolError, new TestMessagePayload { MessageLevel = TestMessageLevel.Error, Message = $"FakeCommunicationChannel: Excpected message {nextResponsePair.Request} but got {requestMessage.MessageType}." }); + var errorMessage = $"FakeCommunicationChannel: Excpected message {nextResponsePair.Request} but got {requestMessage.MessageType}."; + var exception = new Exception(errorMessage); + FakeErrorAggregator.Add(exception); + var errorResponse = new FakeMessage(MessageType.ProtocolError, new TestMessagePayload { MessageLevel = TestMessageLevel.Error, Message = errorMessage }); ProcessedMessages.Add(new RequestResponsePair(requestMessage, errorResponse)); Faulted = true; Respond(errorResponse); @@ -104,7 +126,12 @@ public FakeCommunicationChannel() // TODO: remove !, once we fix the type var response = responsePair.Response!; ProcessedMessages.Add(new RequestResponsePair(requestMessage, response)); - Respond(response); + + // If we created a pair with NoResponse message, we won't send that back to the server. + if (response != FakeMessage.NoResponse) + { + Respond(response); + } } } else @@ -112,9 +139,9 @@ public FakeCommunicationChannel() await Task.Delay(100); } } - catch + catch (Exception ex) { - + FakeErrorAggregator.Add(ex); } } @@ -123,7 +150,7 @@ public FakeCommunicationChannel() private void Respond(FakeMessage response) { - // TODO: we never call this when MessageRecieved is null, but how do I tell that to the compiler? + // TODO: we never call this when MessageRecieved is null, but how do I tell that to the compiler? And is that even true? MessageReceived!(this, new MessageReceivedEventArgs { Data = response.SerializedMessage }); } @@ -229,4 +256,9 @@ internal abstract class FakeMessage // TODO: Is there a better way to ensure that is is not null, we will always set it in the inherited types, but it would be nice to have warning if we did not. // And adding constructor makes it difficult to use the serializer, especially if we wanted to the serializer dynamic and not a static instance. public string SerializedMessage { get; init; } = string.Empty; + + /// + /// When paired with this message, the channel should only recieve the request but not send this message back to the caller. + /// + public static FakeMessage NoResponse { get; } } diff --git a/test/vstest.ProgrammerTests/Fakes/FakeCommunicationEndpoint.cs b/test/vstest.ProgrammerTests/Fakes/FakeCommunicationEndpoint.cs index 4db622fa29..91382f767e 100644 --- a/test/vstest.ProgrammerTests/Fakes/FakeCommunicationEndpoint.cs +++ b/test/vstest.ProgrammerTests/Fakes/FakeCommunicationEndpoint.cs @@ -10,22 +10,33 @@ namespace vstest.ProgrammerTests.CommandLine; internal class FakeCommunicationEndpoint : ICommunicationEndPoint { - public FakeCommunicationEndpoint() + private bool _stopped; + + public FakeCommunicationEndpoint(FakeErrorAggregator fakeErrorAggregator) { + FakeErrorAggregator = fakeErrorAggregator; } + public FakeErrorAggregator FakeErrorAggregator { get; } + public event EventHandler? Connected; public event EventHandler? Disconnected; public string Start(string endPoint) { // TODO: insert this from the outside so some channel manager can give us overview of the open channels? - Connected?.SafeInvoke(this, new ConnectedEventArgs(new FakeCommunicationChannel()), "FakeCommunicationEndpoint.Start"); + Connected?.SafeInvoke(this, new ConnectedEventArgs(new FakeCommunicationChannel(FakeErrorAggregator)), "FakeCommunicationEndpoint.Start"); return endPoint; } public void Stop() { - Disconnected?.Invoke(this, new DisconnectedEventArgs()); + if (!_stopped) + { + // Do not allow stop to be called multiple times, because it will end up calling us back and stack overflows. + _stopped = true; + + Disconnected?.Invoke(this, new DisconnectedEventArgs()); + } } } diff --git a/test/vstest.ProgrammerTests/Fakes/FakeDataCollectorAttachmentsProcessorsFactory.cs b/test/vstest.ProgrammerTests/Fakes/FakeDataCollectorAttachmentsProcessorsFactory.cs index bd9b2fe3cc..ffcf56fa4a 100644 --- a/test/vstest.ProgrammerTests/Fakes/FakeDataCollectorAttachmentsProcessorsFactory.cs +++ b/test/vstest.ProgrammerTests/Fakes/FakeDataCollectorAttachmentsProcessorsFactory.cs @@ -10,6 +10,13 @@ namespace vstest.ProgrammerTests.CommandLine.Fakes; internal class FakeDataCollectorAttachmentsProcessorsFactory : IDataCollectorAttachmentsProcessorsFactory { + public FakeDataCollectorAttachmentsProcessorsFactory(FakeErrorAggregator fakeErrorAggregator) + { + FakeErrorAggregator = fakeErrorAggregator; + } + + public FakeErrorAggregator FakeErrorAggregator { get; } + public DataCollectorAttachmentProcessor[] Create(InvokedDataCollector[] invokedDataCollectors, IMessageLogger logger) { throw new NotImplementedException(); diff --git a/test/vstest.ProgrammerTests/Fakes/FakeFileHelper.cs b/test/vstest.ProgrammerTests/Fakes/FakeFileHelper.cs index cdbd9a24c6..fa8584e375 100644 --- a/test/vstest.ProgrammerTests/Fakes/FakeFileHelper.cs +++ b/test/vstest.ProgrammerTests/Fakes/FakeFileHelper.cs @@ -8,7 +8,13 @@ namespace vstest.ProgrammerTests.CommandLine.Fakes; internal class FakeFileHelper : IFileHelper { + public FakeFileHelper(FakeErrorAggregator fakeErrorAggregator) + { + FakeErrorAggregator = fakeErrorAggregator; + } + public List Files { get; } = new(); + public FakeErrorAggregator FakeErrorAggregator { get; } public void CopyFile(string sourcePath, string destinationPath) { diff --git a/test/vstest.ProgrammerTests/Fakes/FakeMetricsPublisher.cs b/test/vstest.ProgrammerTests/Fakes/FakeMetricsPublisher.cs index 164d235552..bf86207444 100644 --- a/test/vstest.ProgrammerTests/Fakes/FakeMetricsPublisher.cs +++ b/test/vstest.ProgrammerTests/Fakes/FakeMetricsPublisher.cs @@ -8,13 +8,20 @@ namespace vstest.ProgrammerTests.CommandLine.Fakes; internal class FakeMetricsPublisher : IMetricsPublisher { + public FakeMetricsPublisher(FakeErrorAggregator fakeErrorAggregator) + { + FakeErrorAggregator = fakeErrorAggregator; + } + + public FakeErrorAggregator FakeErrorAggregator { get; } + public void Dispose() { - throw new NotImplementedException(); + // do nothing } public void PublishMetrics(string eventName, IDictionary metrics) { - throw new NotImplementedException(); + // TODO: does nothing but probably should } } diff --git a/test/vstest.ProgrammerTests/Fakes/FakeProcess.cs b/test/vstest.ProgrammerTests/Fakes/FakeProcess.cs index c6c5839169..5e70e94067 100644 --- a/test/vstest.ProgrammerTests/Fakes/FakeProcess.cs +++ b/test/vstest.ProgrammerTests/Fakes/FakeProcess.cs @@ -16,13 +16,17 @@ internal class FakeProcess public string Arguments { get; set; } public PlatformArchitecture Architecture { get; init; } = PlatformArchitecture.X64; + public string V { get; } + public FakeErrorAggregator FakeErrorAggregator { get; } + public event EventHandler ProcessExited = delegate { }; - public FakeProcess(string path, string arguments = null) + public FakeProcess(string path, string arguments, FakeErrorAggregator fakeErrorAggregator) { Path = path; Name = System.IO.Path.GetFileName(path); Arguments = arguments; + FakeErrorAggregator = fakeErrorAggregator; } internal static FakeProcess EnsureFakeProcess(object process) diff --git a/test/vstest.ProgrammerTests/Fakes/FakeProcessHelper.cs b/test/vstest.ProgrammerTests/Fakes/FakeProcessHelper.cs index cbefdf6728..69c00eaa9e 100644 --- a/test/vstest.ProgrammerTests/Fakes/FakeProcessHelper.cs +++ b/test/vstest.ProgrammerTests/Fakes/FakeProcessHelper.cs @@ -18,10 +18,13 @@ internal class FakeProcessHelper : IProcessHelper public List Processes { get; } = new(); public int LastProcessId => _lastProcessId; - public FakeProcessHelper(FakeProcess currentProcess) + public FakeErrorAggregator FakeErrorAggregator { get; } + + public FakeProcessHelper(FakeProcess currentProcess, FakeErrorAggregator fakeErrorAggregator) { CurrentProcess = currentProcess; AddProcess(currentProcess); + FakeErrorAggregator = fakeErrorAggregator; } private void AddProcess(FakeProcess currentProcess) diff --git a/test/vstest.ProgrammerTests/Fakes/FakeTestHostProcess.cs b/test/vstest.ProgrammerTests/Fakes/FakeTestHostProcess.cs index 934f369e80..723f5b4099 100644 --- a/test/vstest.ProgrammerTests/Fakes/FakeTestHostProcess.cs +++ b/test/vstest.ProgrammerTests/Fakes/FakeTestHostProcess.cs @@ -7,7 +7,7 @@ namespace vstest.ProgrammerTests.CommandLine.Fakes; internal class FakeTestHostProcess : FakeProcess { - public FakeTestHostProcess(string commandLine) : base(commandLine) + public FakeTestHostProcess(string commandLine, string arguments, FakeErrorAggregator fakeErrorAggregator) : base(commandLine, arguments, fakeErrorAggregator) { } diff --git a/test/vstest.ProgrammerTests/Fakes/FakeTestPlatformEventSource.cs b/test/vstest.ProgrammerTests/Fakes/FakeTestPlatformEventSource.cs index 8559468963..02ee19f378 100644 --- a/test/vstest.ProgrammerTests/Fakes/FakeTestPlatformEventSource.cs +++ b/test/vstest.ProgrammerTests/Fakes/FakeTestPlatformEventSource.cs @@ -8,10 +8,13 @@ namespace vstest.ProgrammerTests.CommandLine.Fakes; internal class FakeTestPlatformEventSource : ITestPlatformEventSource { - public FakeTestPlatformEventSource() + public FakeTestPlatformEventSource(FakeErrorAggregator fakeErrorAggregator) { + FakeErrorAggregator = fakeErrorAggregator; } + public FakeErrorAggregator FakeErrorAggregator { get; } + public void AdapterDiscoveryStart(string executorUri) { // do nothing diff --git a/test/vstest.ProgrammerTests/Fakes/FakeTestRunEventsRegistrar.cs b/test/vstest.ProgrammerTests/Fakes/FakeTestRunEventsRegistrar.cs index c7d6b2ce2a..9b221feb2b 100644 --- a/test/vstest.ProgrammerTests/Fakes/FakeTestRunEventsRegistrar.cs +++ b/test/vstest.ProgrammerTests/Fakes/FakeTestRunEventsRegistrar.cs @@ -10,6 +10,11 @@ namespace vstest.ProgrammerTests.CommandLine.Fakes; internal class FakeTestRunEventsRegistrar : ITestRunEventsRegistrar { + public FakeTestRunEventsRegistrar(FakeErrorAggregator fakeErrorAggregator) + { + FakeErrorAggregator = fakeErrorAggregator; + } + public List AllEvents { get; } = new(); public List Warnings { get; } = new(); public List> RunCompletionEvents { get; } = new(); @@ -17,6 +22,7 @@ internal class FakeTestRunEventsRegistrar : ITestRunEventsRegistrar public List> RunStatsChange { get; } = new(); public List> RawMessageEvents { get; } = new(); public List> TestRunMessageEvents { get; } = new(); + public FakeErrorAggregator FakeErrorAggregator { get; } public void LogWarning(string message) { diff --git a/test/vstest.ProgrammerTests/Fakes/FakeTestRuntimeProviderManager.cs b/test/vstest.ProgrammerTests/Fakes/FakeTestRuntimeProviderManager.cs index aab1d3554d..77fe1ec717 100644 --- a/test/vstest.ProgrammerTests/Fakes/FakeTestRuntimeProviderManager.cs +++ b/test/vstest.ProgrammerTests/Fakes/FakeTestRuntimeProviderManager.cs @@ -9,12 +9,14 @@ namespace vstest.ProgrammerTests.CommandLine.Fakes; internal class FakeTestRuntimeProviderManager : ITestRuntimeProviderManager { - public FakeTestRuntimeProviderManager(FakeProcessHelper fakeProcessHelper, FakeCommunicationEndpoint fakeCommunicationEndpoint) + public FakeTestRuntimeProviderManager(FakeProcessHelper fakeProcessHelper, FakeCommunicationEndpoint fakeCommunicationEndpoint, FakeErrorAggregator fakeErrorAggregator) { TestRuntimeProvider = new FakeTestRuntimeProvider(fakeProcessHelper, fakeCommunicationEndpoint); + FakeErrorAggregator = fakeErrorAggregator; } public FakeTestRuntimeProvider TestRuntimeProvider { get; private set; } + public FakeErrorAggregator FakeErrorAggregator { get; } public ITestRuntimeProvider GetTestHostManagerByRunConfiguration(string runConfiguration) { diff --git a/test/vstest.ProgrammerTests/UnitTest1.cs b/test/vstest.ProgrammerTests/UnitTest1.cs index 4c84633e4b..1703ad69de 100644 --- a/test/vstest.ProgrammerTests/UnitTest1.cs +++ b/test/vstest.ProgrammerTests/UnitTest1.cs @@ -45,33 +45,34 @@ public void GivenInlineRunsettingsWhenCallingVstestConsoleThenTheyPropagateToTes public class TestDiscoveryTests { - public void GivenAnMSTestAssemblyWith5Tests_WhenTestsAreDiscovered_Then5TestsAreFound() + public void GivenAnMSTestAssemblyWith5Tests_WhenTestsAreRun_Then5TestsAreExecuted() { // -- arrange + var fakeErrorAggregator = new FakeErrorAggregator(); var commandLineOptions = CommandLineOptions.Instance; - var fakeCurrentProcess = new FakeProcess(@"C:\temp\vstest.console.exe"); - var fakeProcessHelper = new FakeProcessHelper(fakeCurrentProcess); + var fakeCurrentProcess = new FakeProcess(@"C:\temp\vstest.console.exe", string.Empty, fakeErrorAggregator); + var fakeProcessHelper = new FakeProcessHelper(fakeCurrentProcess, fakeErrorAggregator); - var fakeCommunicationEndpoint = new FakeCommunicationEndpoint(); + var fakeCommunicationEndpoint = new FakeCommunicationEndpoint(fakeErrorAggregator); TestServiceLocator.Register(fakeCommunicationEndpoint); - var fakeTestRuntimeProviderManager = new FakeTestRuntimeProviderManager(fakeProcessHelper, fakeCommunicationEndpoint); + var fakeTestRuntimeProviderManager = new FakeTestRuntimeProviderManager(fakeProcessHelper, fakeCommunicationEndpoint, fakeErrorAggregator); var testEngine = new TestEngine(fakeTestRuntimeProviderManager, fakeProcessHelper); - var fakeFileHelper = new FakeFileHelper(); + var fakeFileHelper = new FakeFileHelper(fakeErrorAggregator); var testPlatform = new TestPlatform(testEngine, fakeFileHelper, fakeTestRuntimeProviderManager); var testRunResultAggregator = new TestRunResultAggregator(); - var fakeTestPlatformEventSource = new FakeTestPlatformEventSource(); + var fakeTestPlatformEventSource = new FakeTestPlatformEventSource(fakeErrorAggregator); - var fakeAssemblyMetadataProvider = new FakeAssemblyMetadataProvider(fakeFileHelper); + var fakeAssemblyMetadataProvider = new FakeAssemblyMetadataProvider(fakeFileHelper, fakeErrorAggregator); var inferHelper = new InferHelper(fakeAssemblyMetadataProvider); // This is most likely not the correctl place where to cut this off, plugin cache is probably the better place, // but it is not injected, and I don't want to investigate this now. - var fakeDataCollectorAttachmentsProcessorsFactory = new FakeDataCollectorAttachmentsProcessorsFactory(); + var fakeDataCollectorAttachmentsProcessorsFactory = new FakeDataCollectorAttachmentsProcessorsFactory(fakeErrorAggregator); var testRunAttachmentsProcessingManager = new TestRunAttachmentsProcessingManager(fakeTestPlatformEventSource, fakeDataCollectorAttachmentsProcessorsFactory); - Task fakeMetricsPublisherTask = Task.FromResult(new FakeMetricsPublisher()); + Task fakeMetricsPublisherTask = Task.FromResult(new FakeMetricsPublisher(fakeErrorAggregator)); TestRequestManager testRequestManager = new( commandLineOptions, testPlatform, @@ -89,25 +90,28 @@ public void GivenAnMSTestAssemblyWith5Tests_WhenTestsAreDiscovered_Then5TestsAre // TODO: have mstest1dll canned var mstest1Dll = new FakeDllFile(@"C:\temp\mstest1.dll", new FrameworkName(".NETCoreApp,Version=v5.0"), Architecture.X64); fakeFileHelper.AddFile(mstest1Dll); + // TODO: this gives me run configuration that is way too complete, do we a way to generate "bare" runsettings? if not we should add them. Would be also useful to get + // runsettings from parameter set so people can use it + // TODO: TestSessionTimeout gives me way to abort the run without having to cancel it externally, but could probably still lead to hangs if that funtionality is broken + var runConfiguration = new Microsoft.VisualStudio.TestPlatform.ObjectModel.RunConfiguration { TestSessionTimeout = 60_000 }.ToXml().OuterXml; var testRunRequestPayload = new TestRunRequestPayload { // TODO: passing null sources and null testcases does not fail fast Sources = mstest1Dll.Path.ToList(), // TODO: passing null runsettings does not fail fast, instead it fails in Fakes settings code // TODO: passing empty string fails in the xml parser code - RunSettings = "" + RunSettings = $"{runConfiguration}" }; // var fakeTestHostLauncher = new FakeTestHostLauncher(); - var fakeTestRunEventsRegistrar = new FakeTestRunEventsRegistrar(); + var fakeTestRunEventsRegistrar = new FakeTestRunEventsRegistrar(fakeErrorAggregator); var protocolConfig = new ProtocolConfig(); testRequestManager.RunTests(testRunRequestPayload, testHostLauncher: null, fakeTestRunEventsRegistrar, protocolConfig); // -- assert - // fakeTestRunEventsRegistrar.RunTests.Should().HaveCount(2); + fakeErrorAggregator.Errors.Should().BeEmpty(); } - } internal class Fixture : IDisposable From 6766ce645bbc5f385eb271d834425a32ede055e3 Mon Sep 17 00:00:00 2001 From: nohwnd Date: Wed, 9 Feb 2022 16:22:04 +0100 Subject: [PATCH 08/32] Chased down the teardown bug. --- .../Execution/TestRunRequest.cs | 6 +++- .../Client/ProxyExecutionManager.cs | 3 ++ .../Client/ProxyOperationManager.cs | 12 ++++++-- .../TestServiceLocator.cs | 16 ++++++++++ test/Intent/Runner.cs | 9 +++++- .../Fakes/FakeCommunicationChannel.cs | 30 +++++++++++++++---- test/vstest.ProgrammerTests/UnitTest1.cs | 25 ++++++++++++++-- 7 files changed, 90 insertions(+), 11 deletions(-) diff --git a/src/Microsoft.TestPlatform.Client/Execution/TestRunRequest.cs b/src/Microsoft.TestPlatform.Client/Execution/TestRunRequest.cs index 2b9bc78184..c1e1f24ac6 100644 --- a/src/Microsoft.TestPlatform.Client/Execution/TestRunRequest.cs +++ b/src/Microsoft.TestPlatform.Client/Execution/TestRunRequest.cs @@ -193,7 +193,7 @@ internal void OnTestSessionTimeout(object obj) HandleLogMessage(TestMessageLevel.Error, message); HandleRawMessage(rawMessage); - Abort(); + Abort(); } /// @@ -278,6 +278,10 @@ public void Abort() } } + // REVIEW: Added this, review this change. If we call abort, the event is never set, and we end up waiting for it to complete forever. + // I think w + _runCompletionEvent.Set(); + EqtTrace.Info("TestRunRequest.Abort: Aborted."); } diff --git a/src/Microsoft.TestPlatform.CrossPlatEngine/Client/ProxyExecutionManager.cs b/src/Microsoft.TestPlatform.CrossPlatEngine/Client/ProxyExecutionManager.cs index 638dcc603c..744d12659e 100644 --- a/src/Microsoft.TestPlatform.CrossPlatEngine/Client/ProxyExecutionManager.cs +++ b/src/Microsoft.TestPlatform.CrossPlatEngine/Client/ProxyExecutionManager.cs @@ -363,6 +363,9 @@ public void HandleTestRunStatsChange(TestRunChangedEventArgs testRunChangedArgs) /// public void HandleRawMessage(string rawMessage) { + // TODO: perf - why do we have to deserialize the messages here only to read that this is + // execution complete? Why can't we act on it somewhere else where the result of deserialization is not + // thrown away? var message = _dataSerializer.DeserializeMessage(rawMessage); if (string.Equals(message.MessageType, MessageType.ExecutionComplete)) diff --git a/src/Microsoft.TestPlatform.CrossPlatEngine/Client/ProxyOperationManager.cs b/src/Microsoft.TestPlatform.CrossPlatEngine/Client/ProxyOperationManager.cs index d71363505c..fb5d5cbd3f 100644 --- a/src/Microsoft.TestPlatform.CrossPlatEngine/Client/ProxyOperationManager.cs +++ b/src/Microsoft.TestPlatform.CrossPlatEngine/Client/ProxyOperationManager.cs @@ -327,8 +327,16 @@ public virtual void Close() { _initialized = false; - // Please clean up test host. - TestHostManager.CleanTestHostAsync(CancellationToken.None).Wait(); + // This is calling external code, make sure we don't fail when it throws + try + { + // Please clean up test host. + TestHostManager.CleanTestHostAsync(CancellationToken.None).Wait(); + } + catch (Exception ex) + { + EqtTrace.Error($"ProxyOperationManager: Cleaning testhost failed: {ex}"); + } TestHostManager.HostExited -= TestHostManagerHostExited; TestHostManager.HostLaunched -= TestHostManagerHostLaunched; diff --git a/src/Microsoft.TestPlatform.ObjectModel/TestServiceLocator.cs b/src/Microsoft.TestPlatform.ObjectModel/TestServiceLocator.cs index c99af67eaa..c3eefb2b21 100644 --- a/src/Microsoft.TestPlatform.ObjectModel/TestServiceLocator.cs +++ b/src/Microsoft.TestPlatform.ObjectModel/TestServiceLocator.cs @@ -10,6 +10,7 @@ namespace Microsoft.VisualStudio.TestPlatform.ObjectModel; public static class TestServiceLocator { public static Dictionary Instances { get; } = new Dictionary(); + public static List Resolves { get; } = new(); public static void Register(TRegistration instance) { @@ -21,7 +22,22 @@ public static TRegistration Get() if (!Instances.TryGetValue(typeof(TRegistration), out var instance)) throw new InvalidOperationException($"Cannot find instance for type {typeof(TRegistration)}."); +#if !NETSTANDARD1_0 + Resolves.Add(new Resolve(typeof(TRegistration).FullName, Environment.StackTrace)); +#endif return (TRegistration)instance; } } +// TODO: Make this internal, I am just trying to have easier time trying this out. +public class Resolve +{ + public Resolve(string type, string stackTrace) + { + Type = type; + StackTrace = stackTrace; + } + + public string Type { get; } + public string StackTrace { get; } +} diff --git a/test/Intent/Runner.cs b/test/Intent/Runner.cs index 3dbf68ca49..8fb15c7696 100644 --- a/test/Intent/Runner.cs +++ b/test/Intent/Runner.cs @@ -26,7 +26,14 @@ public static void Run(IEnumerable path, IRunLogger logger) try { var i = Activator.CreateInstance(t); - m.Invoke(i, Array.Empty()); + var result = m.Invoke(i, Array.Empty()); + if (result is Task task) + { + // When the result is a task we need to await it. + // TODO: this can be improved with await, imho + task.GetAwaiter().GetResult(); + }; + logger.WriteTestPassed(m); } catch (Exception ex) diff --git a/test/vstest.ProgrammerTests/Fakes/FakeCommunicationChannel.cs b/test/vstest.ProgrammerTests/Fakes/FakeCommunicationChannel.cs index c7052651be..a2d660f1df 100644 --- a/test/vstest.ProgrammerTests/Fakes/FakeCommunicationChannel.cs +++ b/test/vstest.ProgrammerTests/Fakes/FakeCommunicationChannel.cs @@ -2,6 +2,7 @@ // Licensed under the MIT license. See LICENSE file in the project root for full license information. using System.Collections.Concurrent; +using System.Diagnostics; using Microsoft.VisualStudio.TestPlatform.CommunicationUtilities; using Microsoft.VisualStudio.TestPlatform.CommunicationUtilities.Interfaces; @@ -58,7 +59,7 @@ public FakeCommunicationChannel(FakeErrorAggregator fakeErrorAggregator) TestRunCompleteArgs = new TestRunCompleteEventArgs(new TestRunStatistics(), false, false, null, new System.Collections.ObjectModel.Collection(), TimeSpan.Zero), LastRunTests = new TestRunChangedEventArgs(new TestRunStatistics(), new List(), new List()), }; - NextResponses.Enqueue(new RequestResponsePair(MessageType.StartTestExecutionWithSources, new FakeMessage(MessageType.ExecutionComplete, result))); + NextResponses.Enqueue(new RequestResponsePair(MessageType.StartTestExecutionWithSources, new FakeMessage(MessageType.ExecutionComplete, result), true)); Spin = Task.Run(async () => { @@ -123,6 +124,11 @@ public FakeCommunicationChannel(FakeErrorAggregator fakeErrorAggregator) else { var responsePair = NextResponses.Dequeue(); + if (responsePair.Debug && Debugger.IsAttached) + { + // We are about to send an interesting message + Debugger.Break(); + } // TODO: remove !, once we fix the type var response = responsePair.Response!; ProcessedMessages.Add(new RequestResponsePair(requestMessage, response)); @@ -150,8 +156,18 @@ public FakeCommunicationChannel(FakeErrorAggregator fakeErrorAggregator) private void Respond(FakeMessage response) { - // TODO: we never call this when MessageRecieved is null, but how do I tell that to the compiler? And is that even true? - MessageReceived!(this, new MessageReceivedEventArgs { Data = response.SerializedMessage }); + // We started processing the message when there was still someone listenting, but processing the message + // took us some time and the listener might have unsubscribed. check again if anyone is interested in the + // data. This is still a race condondition. In real code we solve this via SafeInvoke that does null check + // and catches the exception. In this code I prefer doing it this way, to see if it is fragile. + // + // The data from the message will be lost if the listener unsubscribes. This is okay, as it is similar to writing data + // to network stream while the other side disconnects. This is purely issue of timing, that can never be avoided. + if (MessageReceived != null) + { + MessageReceived(this, new MessageReceivedEventArgs { Data = response.SerializedMessage }); + } + //TODO: record unprocessed responses? } public void Dispose() @@ -175,16 +191,18 @@ public Task Send(string data) internal class RequestResponsePair where T : class { - public RequestResponsePair(T request, U response) + public RequestResponsePair(T request, U response, bool debug = false) { Request = request; Response = response; + Debug = debug; } - public RequestResponsePair(T request, Func responseFactory) + public RequestResponsePair(T request, Func responseFactory, bool debug = false) { Request = request; ResponseFactory = responseFactory; + Debug = debug; } public T Request { get; } @@ -193,6 +211,8 @@ public RequestResponsePair(T request, Func responseFactory) public Func? ResponseFactory { get; } public U? Response { get; private set; } + public bool Debug { get; init; } + // TODO: Let's sleep on this and see if I understand tomorrow what I was trying to do, because this has too many usages now... // One day later I do remember it, but I am still not convinced. But let's keep that for now. The idea here is to get either a canned // response or a response generated based on the incoming request (e.g. version comes in that is 3, response is lower version (2)). diff --git a/test/vstest.ProgrammerTests/UnitTest1.cs b/test/vstest.ProgrammerTests/UnitTest1.cs index 1703ad69de..46fb46f481 100644 --- a/test/vstest.ProgrammerTests/UnitTest1.cs +++ b/test/vstest.ProgrammerTests/UnitTest1.cs @@ -2,6 +2,7 @@ // Licensed under the MIT license. See LICENSE file in the project root for full license information. +using System.Diagnostics; using System.Runtime.Versioning; using FluentAssertions; @@ -45,7 +46,7 @@ public void GivenInlineRunsettingsWhenCallingVstestConsoleThenTheyPropagateToTes public class TestDiscoveryTests { - public void GivenAnMSTestAssemblyWith5Tests_WhenTestsAreRun_Then5TestsAreExecuted() + public async Task GivenAnMSTestAssemblyWith5Tests_WhenTestsAreRun_Then5TestsAreExecuted() { // -- arrange var fakeErrorAggregator = new FakeErrorAggregator(); @@ -93,7 +94,9 @@ public void GivenAnMSTestAssemblyWith5Tests_WhenTestsAreRun_Then5TestsAreExecute // TODO: this gives me run configuration that is way too complete, do we a way to generate "bare" runsettings? if not we should add them. Would be also useful to get // runsettings from parameter set so people can use it // TODO: TestSessionTimeout gives me way to abort the run without having to cancel it externally, but could probably still lead to hangs if that funtionality is broken - var runConfiguration = new Microsoft.VisualStudio.TestPlatform.ObjectModel.RunConfiguration { TestSessionTimeout = 60_000 }.ToXml().OuterXml; + // TODO: few tries later, that is exactly the case when we abort, it still hangs on waiting to complete request, because test run complete was not sent + // var runConfiguration = new Microsoft.VisualStudio.TestPlatform.ObjectModel.RunConfiguration { TestSessionTimeout = 40_000 }.ToXml().OuterXml; + var runConfiguration = string.Empty; var testRunRequestPayload = new TestRunRequestPayload { // TODO: passing null sources and null testcases does not fail fast @@ -107,7 +110,25 @@ public void GivenAnMSTestAssemblyWith5Tests_WhenTestsAreRun_Then5TestsAreExecute var fakeTestRunEventsRegistrar = new FakeTestRunEventsRegistrar(fakeErrorAggregator); var protocolConfig = new ProtocolConfig(); + var cancelAbort = new CancellationTokenSource(); + var task = Task.Run(async () => + { + // TODO: we make sure the test is running 1 minute at max and then we try to abort + // if we aborted we write the error to aggregator, this needs to be made into a pattern + // so we can avoid hanging if the run does not complete correctly + await Task.Delay(TimeSpan.FromMinutes(10), cancelAbort.Token); + if (Debugger.IsAttached) + { + // we will abort because we are hanging, look at stacks to see what the problem is + Debugger.Break(); + } + fakeErrorAggregator.Add(new Exception("errr we aborted")); + testRequestManager.AbortTestRun(); + + }); testRequestManager.RunTests(testRunRequestPayload, testHostLauncher: null, fakeTestRunEventsRegistrar, protocolConfig); + cancelAbort.Cancel(); + await task; // -- assert fakeErrorAggregator.Errors.Should().BeEmpty(); From 148086ac0b7ef6faef755a903dbd8395cdf42137 Mon Sep 17 00:00:00 2001 From: nohwnd Date: Thu, 10 Feb 2022 11:22:57 +0100 Subject: [PATCH 09/32] Passing test finally, now refactor --- .../Interfaces/ICommunicationEndpoint.cs | 2 +- .../SocketClient.cs | 4 +- .../SocketServer.cs | 9 +- .../TestRequestSender.cs | 17 +- .../Parallel/ParallelRunDataAggregator.cs | 1 + .../Fakes/FakeAssemblyMetadataProvider.cs | 4 +- .../Fakes/FakeCommunicationChannel.cs | 206 +++++++++--------- .../Fakes/FakeCommunicationEndpoint.cs | 10 +- .../{ => Fakes}/FakeErrorAggregator.cs | 0 .../Fakes/FakeProcess.cs | 33 ++- .../Fakes/FakeProcessHelper.cs | 15 +- .../Fakes/FakeTestBatchBuilder.cs | 72 ++++++ .../{FakeDllFile.cs => FakeTestDllFile.cs} | 10 +- .../Fakes/FakeTestHostProcess.cs | 17 +- .../Fakes/FakeTestRunEventsRegistrar.cs | 20 +- .../Fakes/FakeTestRuntimeProvider.cs | 24 +- test/vstest.ProgrammerTests/UnitTest1.cs | 89 ++++++-- 17 files changed, 359 insertions(+), 174 deletions(-) rename test/vstest.ProgrammerTests/{ => Fakes}/FakeErrorAggregator.cs (100%) create mode 100644 test/vstest.ProgrammerTests/Fakes/FakeTestBatchBuilder.cs rename test/vstest.ProgrammerTests/Fakes/{FakeDllFile.cs => FakeTestDllFile.cs} (53%) diff --git a/src/Microsoft.TestPlatform.CommunicationUtilities/Interfaces/ICommunicationEndpoint.cs b/src/Microsoft.TestPlatform.CommunicationUtilities/Interfaces/ICommunicationEndpoint.cs index ef5dbcd07f..eb63913539 100644 --- a/src/Microsoft.TestPlatform.CommunicationUtilities/Interfaces/ICommunicationEndpoint.cs +++ b/src/Microsoft.TestPlatform.CommunicationUtilities/Interfaces/ICommunicationEndpoint.cs @@ -13,7 +13,7 @@ public interface ICommunicationEndPoint event EventHandler Connected; /// - /// Event raised when an endPoint is disconnected. + /// Event raised when an endPoint is disconnected on failure. It should not be notified when we are just closing the connection after success. /// event EventHandler Disconnected; diff --git a/src/Microsoft.TestPlatform.CommunicationUtilities/SocketClient.cs b/src/Microsoft.TestPlatform.CommunicationUtilities/SocketClient.cs index 7dcaab3b42..be78adc910 100644 --- a/src/Microsoft.TestPlatform.CommunicationUtilities/SocketClient.cs +++ b/src/Microsoft.TestPlatform.CommunicationUtilities/SocketClient.cs @@ -100,14 +100,14 @@ private void OnServerConnected(Task connectAsyncTask) // Start the message loop Task.Run(() => _tcpClient.MessageLoopAsync( _channel, - Stop, + StopOnError, _cancellation.Token)) .ConfigureAwait(false); } } } - private void Stop(Exception error) + private void StopOnError(Exception error) { EqtTrace.Info("SocketClient.PrivateStop: Stop communication from server endpoint: {0}, error:{1}", _endPoint, error); // TODO: this is here to prevent stack overflow. diff --git a/src/Microsoft.TestPlatform.CommunicationUtilities/SocketServer.cs b/src/Microsoft.TestPlatform.CommunicationUtilities/SocketServer.cs index b7c505a0e4..b64588f228 100644 --- a/src/Microsoft.TestPlatform.CommunicationUtilities/SocketServer.cs +++ b/src/Microsoft.TestPlatform.CommunicationUtilities/SocketServer.cs @@ -103,11 +103,16 @@ private void OnClientConnected(TcpClient client) } // Start the message loop - Task.Run(() => _tcpClient.MessageLoopAsync(_channel, error => Stop(error), _cancellation.Token)).ConfigureAwait(false); + Task.Run(() => _tcpClient.MessageLoopAsync(_channel, error => StopOnError(error), _cancellation.Token)).ConfigureAwait(false); } } - private void Stop(Exception error) + /// + /// Stop the connection when error was encountered. Dispose all communication, and notify subscribers of Disconnected event + /// that we aborted. + /// + /// + private void StopOnError(Exception error) { EqtTrace.Info("SocketServer.PrivateStop: Stopping server endPoint: {0} error: {1}", _endPoint, error); diff --git a/src/Microsoft.TestPlatform.CommunicationUtilities/TestRequestSender.cs b/src/Microsoft.TestPlatform.CommunicationUtilities/TestRequestSender.cs index 56fe1f2ff3..7c9eade665 100644 --- a/src/Microsoft.TestPlatform.CommunicationUtilities/TestRequestSender.cs +++ b/src/Microsoft.TestPlatform.CommunicationUtilities/TestRequestSender.cs @@ -27,6 +27,8 @@ namespace Microsoft.VisualStudio.TestPlatform.CommunicationUtilities; public class TestRequestSender : ITestRequestSender { // Time to wait for test host exit + // DONOTMERGE: this was 10s, I made it 1 second, because it makes my tests pass faster when I get in error state + // REVIEW: this was 10s, I made it 1 seconds private const int ClientProcessExitWaitTimeout = 10 * 1000; private readonly IDataSerializer _dataSerializer; @@ -576,7 +578,9 @@ private void OnExecutionMessageReceived(object sender, MessageReceivedEventArgs } catch (Exception exception) { - OnTestRunAbort(testRunEventsHandler, exception, false); + // If we failed to process the incoming message, initiate client (testhost) abort, because we can't recover, and don't wait + // for it to exit and write into error stream, because it did not do anything wrong, so no error is coming there + OnTestRunAbort(testRunEventsHandler, exception, getClientError: false); } } @@ -684,20 +688,25 @@ private void OnDiscoveryAbort(ITestDiscoveryEventsHandler2 eventHandler, Excepti private string GetAbortErrorMessage(Exception exception, bool getClientError) { + EqtTrace.Verbose("TestRequestSender: GetAbortErrorMessage: Exception: " + exception); // It is also possible for an operation to abort even if client has not - // disconnected, e.g. if there's an error parsing the response from test host. We - // want the exception to be available in those scenarios. + // disconnected, because we initiate client abort when there is error in processing incoming messages. + // in this case, we will use the exception as the failure result, if it is present. Otherwise we will + // try to wait for the client process to exit, and capture it's error output (we are listening to it's standard and + // error output in the ClientExited callback). var reason = exception?.Message; if (getClientError) { EqtTrace.Verbose("TestRequestSender: GetAbortErrorMessage: Client has disconnected. Wait for standard error."); // Wait for test host to exit for a moment + // TODO: this timeout is 10 seconds, make it also configurable like the other famous if (_clientExited.Wait(_clientExitedWaitTime)) { - // Set a default message of test host process exited and additionally specify the error if present + // Set a default message of test host process exited and additionally specify the error if we were able to get it + // from error output of the process EqtTrace.Info("TestRequestSender: GetAbortErrorMessage: Received test host error message."); reason = CommonResources.TestHostProcessCrashed; if (!string.IsNullOrWhiteSpace(_clientExitErrorMessage)) diff --git a/src/Microsoft.TestPlatform.CrossPlatEngine/Client/Parallel/ParallelRunDataAggregator.cs b/src/Microsoft.TestPlatform.CrossPlatEngine/Client/Parallel/ParallelRunDataAggregator.cs index 44e6eafaa6..d62860e22b 100644 --- a/src/Microsoft.TestPlatform.CrossPlatEngine/Client/Parallel/ParallelRunDataAggregator.cs +++ b/src/Microsoft.TestPlatform.CrossPlatEngine/Client/Parallel/ParallelRunDataAggregator.cs @@ -80,6 +80,7 @@ public ITestRunStatistics GetAggregatedRunStats() { foreach (var runStats in _testRunStatsList) { + // TODO: we get nullref here if the stats are empty. Should that be okay or not? foreach (var outcome in runStats.Stats.Keys) { if (!testOutcomeMap.ContainsKey(outcome)) diff --git a/test/vstest.ProgrammerTests/Fakes/FakeAssemblyMetadataProvider.cs b/test/vstest.ProgrammerTests/Fakes/FakeAssemblyMetadataProvider.cs index c1443dfe62..bb7c4755f0 100644 --- a/test/vstest.ProgrammerTests/Fakes/FakeAssemblyMetadataProvider.cs +++ b/test/vstest.ProgrammerTests/Fakes/FakeAssemblyMetadataProvider.cs @@ -23,13 +23,13 @@ public FakeAssemblyMetadataProvider(FakeFileHelper fakeFileHelper, FakeErrorAggr public Architecture GetArchitecture(string filePath) { - var file = (FakeDllFile)FakeFileHelper.Files.Single(f => f.Path == filePath); + var file = (FakeTestDllFile)FakeFileHelper.Files.Single(f => f.Path == filePath); return file.Architecture; } public FrameworkName GetFrameWork(string filePath) { - var file = (FakeDllFile)FakeFileHelper.Files.Single(f => f.Path == filePath); + var file = (FakeTestDllFile)FakeFileHelper.Files.Single(f => f.Path == filePath); return file.FrameworkName; } } diff --git a/test/vstest.ProgrammerTests/Fakes/FakeCommunicationChannel.cs b/test/vstest.ProgrammerTests/Fakes/FakeCommunicationChannel.cs index a2d660f1df..cd00497896 100644 --- a/test/vstest.ProgrammerTests/Fakes/FakeCommunicationChannel.cs +++ b/test/vstest.ProgrammerTests/Fakes/FakeCommunicationChannel.cs @@ -29,7 +29,9 @@ internal class FakeCommunicationChannel : ICommunicationChannel // in communication manager). And then we reply back to the sender, by invoking MessageReceived. public BlockingCollection InQueue { get; } = new(); - public Task Spin { get; } + public BlockingCollection OutQueue { get; } = new(); + public Task ProcessIncomingMessages { get; } + public Task ProcessOutgoingMessages { get; } /// /// True if we encountered unexpected message (e.g. unknown message, message out of order) or when we sent all our prepared responses, and there were still more requests coming. @@ -43,107 +45,97 @@ internal class FakeCommunicationChannel : ICommunicationChannel /// public Queue> NextResponses { get; } = new(); public FakeErrorAggregator FakeErrorAggregator { get; } + public FakeMessage? OutgoingMessage { get; private set; } public CancellationTokenSource CancellationTokenSource = new(); public event EventHandler? MessageReceived; - public FakeCommunicationChannel(FakeErrorAggregator fakeErrorAggregator) + public FakeCommunicationChannel(List> responses, FakeErrorAggregator fakeErrorAggregator) { FakeErrorAggregator = fakeErrorAggregator; - NextResponses.Enqueue(new RequestResponsePair(MessageType.VersionCheck, new FakeMessage(MessageType.VersionCheck, 5))); - NextResponses.Enqueue(new RequestResponsePair(MessageType.ExecutionInitialize, FakeMessage.NoResponse)); - var result = new TestRunCompletePayload - { - TestRunCompleteArgs = new TestRunCompleteEventArgs(new TestRunStatistics(), false, false, null, new System.Collections.ObjectModel.Collection(), TimeSpan.Zero), - LastRunTests = new TestRunChangedEventArgs(new TestRunStatistics(), new List(), new List()), - }; - NextResponses.Enqueue(new RequestResponsePair(MessageType.StartTestExecutionWithSources, new FakeMessage(MessageType.ExecutionComplete, result), true)); + responses.ForEach(NextResponses.Enqueue); - Spin = Task.Run(async () => + ProcessIncomingMessages = Task.Run(() => { var token = CancellationTokenSource.Token; while (!token.IsCancellationRequested) { try { - // Only consume messages if someone is listening on the other side. - if (MessageReceived != null) + var rawMessage = InQueue.Take(token); + var requestMessage = JsonDataSerializer.Instance.DeserializeMessage(rawMessage); + + if (Faulted) { - var rawMessage = InQueue.Take(token); - var requestMessage = JsonDataSerializer.Instance.DeserializeMessage(rawMessage); + // We already failed, when there are more requests coming, just save them and respond with error. We want to avoid + // a situation where server ignores our error message and responds with another request, for which we accidentally + // have the right answer in queue. + // + // E.g. We have VersionCheck, TestRunStart prepared, and server sends: VersionCheck, TestInitialize, TestRunStart. + // The first request has a valid response. The next TestInitialize does not have a valid response and errors out, + // but the server ignores it, and sends TestRunStart, which would normally have a prepared response, and lead to + // possibly overlooking the error response to TestInitialize. + // + // With this check in place we will not respond to TestRunStart with success, but with error. + // TODO: Better way to map MessageType and the payload type. + // TODO: simpler way to report error, and add it to the error aggregator + var errorMessage = $"FakeCommunicationChannel: FakeCommunicationChannel: Got message {requestMessage.MessageType}. But a message that was unexptected was received previously and the channel is now faulted. Review {nameof(ProcessedMessages)}, and {nameof(NextResponses)}."; + var exception = new Exception(errorMessage); + FakeErrorAggregator.Add(exception); + var errorResponse = new FakeMessage(MessageType.TestMessage, new TestMessagePayload { MessageLevel = TestMessageLevel.Error, Message = errorMessage }); + ProcessedMessages.Add(new RequestResponsePair(requestMessage, errorResponse)); + Faulted = true; + OutQueue.Add(errorResponse); + } - if (Faulted) + // Just peek at it so we can keep the message on the the queue in case of error. + if (!NextResponses.TryPeek(out var nextResponsePair)) + { + // If there are no more prepared responses then return protocol error. + var errorMessage = $"FakeCommunicationChannel: Got message {requestMessage.MessageType}, but no more requests were expected, because there are no more responses in {nameof(NextResponses)}."; + var exception = new Exception(errorMessage); + FakeErrorAggregator.Add(exception); + var errorResponse = new FakeMessage(MessageType.ProtocolError, new TestMessagePayload { MessageLevel = TestMessageLevel.Error, Message = errorMessage }); + ProcessedMessages.Add(new RequestResponsePair(requestMessage, errorResponse)); + Faulted = true; + OutQueue.Add(errorResponse); + } + else if (nextResponsePair.Request != requestMessage.MessageType) + { + // If the incoming message does not match what we expected return protocol error. The lsat message will remain in the + // NextResponses queue. + var errorMessage = $"FakeCommunicationChannel: Excpected message {nextResponsePair.Request} but got {requestMessage.MessageType}."; + var exception = new Exception(errorMessage); + FakeErrorAggregator.Add(exception); + var errorResponse = new FakeMessage(MessageType.ProtocolError, new TestMessagePayload { MessageLevel = TestMessageLevel.Error, Message = errorMessage }); + ProcessedMessages.Add(new RequestResponsePair(requestMessage, errorResponse)); + Faulted = true; + OutQueue.Add(errorResponse); + } + else + { + var responsePair = NextResponses.Dequeue(); + if (responsePair.Debug && Debugger.IsAttached) { - // We already failed, when there are more requests coming, just save them and respond with error. We want to avoid - // a situation where server ignores our error message and responds with another request, for which we accidentally - // have the right answer in queue. - // - // E.g. We have VersionCheck, TestRunStart prepared, and server sends: VersionCheck, TestInitialize, TestRunStart. - // The first request has a valid response. The next TestInitialize does not have a valid response and errors out, - // but the server ignores it, and sends TestRunStart, which would normally have a prepared response, and lead to - // possibly overlooking the error response to TestInitialize. - // - // With this check in place we will not respond to TestRunStart with success, but with error. - // TODO: Better way to map MessageType and the payload type. - // TODO: simpler way to report error, and add it to the error aggregator - var errorMessage = $"FakeCommunicationChannel: FakeCommunicationChannel: Got message {requestMessage.MessageType}. But a message that was unexptected was received previously and the channel is now faulted. Review {nameof(ProcessedMessages)}, and {nameof(NextResponses)}."; - var exception = new Exception(errorMessage); - FakeErrorAggregator.Add(exception); - var errorResponse = new FakeMessage(MessageType.TestMessage, new TestMessagePayload { MessageLevel = TestMessageLevel.Error, Message = errorMessage }); - ProcessedMessages.Add(new RequestResponsePair(requestMessage, errorResponse)); - Faulted = true; - Respond(errorResponse); + // We are about to send an interesting message + Debugger.Break(); } - // Just peek at it so we can keep the message on the the queue in case of error. - if (!NextResponses.TryPeek(out var nextResponsePair)) - { - // If there are no more prepared responses then return protocol error. - var errorMessage = $"FakeCommunicationChannel: Got message {requestMessage.MessageType}, but no more requests were expected, because there are no more responses in {nameof(NextResponses)}."; - var exception = new Exception(errorMessage); - FakeErrorAggregator.Add(exception); - var errorResponse = new FakeMessage(MessageType.ProtocolError, new TestMessagePayload { MessageLevel = TestMessageLevel.Error, Message = errorMessage }); - ProcessedMessages.Add(new RequestResponsePair(requestMessage, errorResponse)); - Faulted = true; - Respond(errorResponse); - } - else if (nextResponsePair.Request != requestMessage.MessageType) - { - // If the incoming message does not match what we expected return protocol error. The lsat message will remain in the - // NextResponses queue. - var errorMessage = $"FakeCommunicationChannel: Excpected message {nextResponsePair.Request} but got {requestMessage.MessageType}."; - var exception = new Exception(errorMessage); - FakeErrorAggregator.Add(exception); - var errorResponse = new FakeMessage(MessageType.ProtocolError, new TestMessagePayload { MessageLevel = TestMessageLevel.Error, Message = errorMessage }); - ProcessedMessages.Add(new RequestResponsePair(requestMessage, errorResponse)); - Faulted = true; - Respond(errorResponse); - } - else - { - var responsePair = NextResponses.Dequeue(); - if (responsePair.Debug && Debugger.IsAttached) - { - // We are about to send an interesting message - Debugger.Break(); - } - // TODO: remove !, once we fix the type - var response = responsePair.Response!; - ProcessedMessages.Add(new RequestResponsePair(requestMessage, response)); + // TODO: passing the raw message in, is strange + var responses = responsePair.GetResponse(rawMessage)!; + ProcessedMessages.Add(new RequestResponsePair(requestMessage, responses)); + foreach (var response in responses) + { // If we created a pair with NoResponse message, we won't send that back to the server. if (response != FakeMessage.NoResponse) { - Respond(response); + OutQueue.Add(response); } } } - else - { - await Task.Delay(100); - } } catch (Exception ex) { @@ -152,22 +144,31 @@ public FakeCommunicationChannel(FakeErrorAggregator fakeErrorAggregator) } }, CancellationTokenSource.Token); - } - private void Respond(FakeMessage response) - { - // We started processing the message when there was still someone listenting, but processing the message - // took us some time and the listener might have unsubscribed. check again if anyone is interested in the - // data. This is still a race condondition. In real code we solve this via SafeInvoke that does null check - // and catches the exception. In this code I prefer doing it this way, to see if it is fragile. - // - // The data from the message will be lost if the listener unsubscribes. This is okay, as it is similar to writing data - // to network stream while the other side disconnects. This is purely issue of timing, that can never be avoided. - if (MessageReceived != null) + ProcessOutgoingMessages = Task.Run(() => { - MessageReceived(this, new MessageReceivedEventArgs { Data = response.SerializedMessage }); - } - //TODO: record unprocessed responses? + var token = CancellationTokenSource.Token; + while (!token.IsCancellationRequested) + { + try + { + // TODO: better name? this is message that we are currently trying to send + OutgoingMessage = OutQueue.Take(); + // This is still a race condition. In real code we solve this via SafeInvoke that does null check + // and catches the exception. In this code I prefer doing it this way, to see if it is fragile. + if (MessageReceived != null) + { + MessageReceived(this, new MessageReceivedEventArgs { Data = OutgoingMessage.SerializedMessage }); + } + OutgoingMessage = null; + } + catch (Exception ex) + { + FakeErrorAggregator.Add(ex); + } + } + + }, CancellationTokenSource.Token); } public void Dispose() @@ -194,11 +195,18 @@ internal class RequestResponsePair where T : class public RequestResponsePair(T request, U response, bool debug = false) { Request = request; - Response = response; + Responses = new List { response }; + Debug = debug; + } + + public RequestResponsePair(T request, IEnumerable responses, bool debug = false) + { + Request = request; + Responses = responses.ToList(); Debug = debug; } - public RequestResponsePair(T request, Func responseFactory, bool debug = false) + public RequestResponsePair(T request, Func> responseFactory, bool debug = false) { Request = request; ResponseFactory = responseFactory; @@ -208,8 +216,8 @@ public RequestResponsePair(T request, Func responseFactory, bool debug = f public T Request { get; } // TODO: make this Expression? ResponseFactory { get; } - public U? Response { get; private set; } + public Func>? ResponseFactory { get; } + public List Responses { get; private set; } public bool Debug { get; init; } @@ -218,20 +226,20 @@ public RequestResponsePair(T request, Func responseFactory, bool debug = f // response or a response generated based on the incoming request (e.g. version comes in that is 3, response is lower version (2)). // both of these could be done by just executing Func, but that is not readable during debug time. Or maybe some variation on Either<> // but that seems as a very foreign concept to common C#. - public U GetResponse(T? request = null) + public List GetResponse(T? request = null) { if (ResponseFactory != null) { // TODO: what am I doing wrong? (Why do I need that '!' ? I assign Request in both ctors and don't have setter, is the null coalescing not // supposed to propagate non-nullable type to the target type? // TODO: I don't like rewriting the response here. This is yet another sign that I am possibly mixing concepts in this class. - Response = ResponseFactory(request ?? Request!); - return Response; + Responses = ResponseFactory(request ?? Request!); + return Responses; } else { // TODO: split this class to two that has the same parent, so we are sure we have a response in any case. - return Response!; + return Responses!; } } } @@ -278,7 +286,7 @@ internal abstract class FakeMessage public string SerializedMessage { get; init; } = string.Empty; /// - /// When paired with this message, the channel should only recieve the request but not send this message back to the caller. + /// /// - public static FakeMessage NoResponse { get; } + public static FakeMessage NoResponse { get; } = new FakeMessage("NoResponse", 0); } diff --git a/test/vstest.ProgrammerTests/Fakes/FakeCommunicationEndpoint.cs b/test/vstest.ProgrammerTests/Fakes/FakeCommunicationEndpoint.cs index 91382f767e..71d227e32f 100644 --- a/test/vstest.ProgrammerTests/Fakes/FakeCommunicationEndpoint.cs +++ b/test/vstest.ProgrammerTests/Fakes/FakeCommunicationEndpoint.cs @@ -12,20 +12,21 @@ internal class FakeCommunicationEndpoint : ICommunicationEndPoint { private bool _stopped; - public FakeCommunicationEndpoint(FakeErrorAggregator fakeErrorAggregator) + public FakeCommunicationEndpoint(FakeCommunicationChannel fakeCommunicationChannel, FakeErrorAggregator fakeErrorAggregator) { + Channel = fakeCommunicationChannel; FakeErrorAggregator = fakeErrorAggregator; } public FakeErrorAggregator FakeErrorAggregator { get; } + public FakeCommunicationChannel Channel { get; private set; } public event EventHandler? Connected; public event EventHandler? Disconnected; public string Start(string endPoint) { - // TODO: insert this from the outside so some channel manager can give us overview of the open channels? - Connected?.SafeInvoke(this, new ConnectedEventArgs(new FakeCommunicationChannel(FakeErrorAggregator)), "FakeCommunicationEndpoint.Start"); + Connected?.SafeInvoke(this, new ConnectedEventArgs(Channel), "FakeCommunicationEndpoint.Start"); return endPoint; } @@ -36,7 +37,8 @@ public void Stop() // Do not allow stop to be called multiple times, because it will end up calling us back and stack overflows. _stopped = true; - Disconnected?.Invoke(this, new DisconnectedEventArgs()); + // TODO: notify this in case of error in the process, so we can initiate abort flow + // Disconnected?.Invoke(this, new DisconnectedEventArgs()); } } } diff --git a/test/vstest.ProgrammerTests/FakeErrorAggregator.cs b/test/vstest.ProgrammerTests/Fakes/FakeErrorAggregator.cs similarity index 100% rename from test/vstest.ProgrammerTests/FakeErrorAggregator.cs rename to test/vstest.ProgrammerTests/Fakes/FakeErrorAggregator.cs diff --git a/test/vstest.ProgrammerTests/Fakes/FakeProcess.cs b/test/vstest.ProgrammerTests/Fakes/FakeProcess.cs index 5e70e94067..1c61a938a9 100644 --- a/test/vstest.ProgrammerTests/Fakes/FakeProcess.cs +++ b/test/vstest.ProgrammerTests/Fakes/FakeProcess.cs @@ -2,8 +2,6 @@ // Licensed under the MIT license. See LICENSE file in the project root for full license information. -#pragma warning disable IDE1006 // Naming Styles -using Microsoft.VisualStudio.TestPlatform.ObjectModel.Host; using Microsoft.VisualStudio.TestPlatform.PlatformAbstractions; namespace vstest.ProgrammerTests.CommandLine.Fakes; @@ -14,18 +12,27 @@ internal class FakeProcess public string Name { get; init; } public string Path { get; } public string Arguments { get; set; } - + public string WorkingDirectory { get; } + public IDictionary EnvironmentVariables { get; } + public Action ErrorCallback { get; } + public Action ExitCallback { get; } + public Action OutputCallback { get; } public PlatformArchitecture Architecture { get; init; } = PlatformArchitecture.X64; - public string V { get; } public FakeErrorAggregator FakeErrorAggregator { get; } + public string? ErrorOutput { get; init; } + public int ExitCode { get; init; } = -1; + public bool Exited { get; private set; } - public event EventHandler ProcessExited = delegate { }; - - public FakeProcess(string path, string arguments, FakeErrorAggregator fakeErrorAggregator) + public FakeProcess(string path, string arguments, string workingDirectory, IDictionary environmentVariables, Action errorCallback, Action exitCallBack, Action outputCallback, FakeErrorAggregator fakeErrorAggregator) { Path = path; Name = System.IO.Path.GetFileName(path); Arguments = arguments; + WorkingDirectory = workingDirectory; + EnvironmentVariables = environmentVariables; + ErrorCallback = errorCallback; + ExitCallback = exitCallBack; + OutputCallback = outputCallback; FakeErrorAggregator = fakeErrorAggregator; } @@ -41,4 +48,16 @@ internal void SetId(int id) Id = id; } + + internal void Exit() + { + // We want to call the exit callback just once. This is behavior inherent to being a real process, + // that also exits only once. + var exited = Exited; + Exited = true; + if (!exited && ExitCallback != null) + { + ExitCallback(this); + } + } } diff --git a/test/vstest.ProgrammerTests/Fakes/FakeProcessHelper.cs b/test/vstest.ProgrammerTests/Fakes/FakeProcessHelper.cs index 69c00eaa9e..bad95982c2 100644 --- a/test/vstest.ProgrammerTests/Fakes/FakeProcessHelper.cs +++ b/test/vstest.ProgrammerTests/Fakes/FakeProcessHelper.cs @@ -84,26 +84,31 @@ public string GetTestEngineDirectory() public object LaunchProcess(string processPath, string arguments, string workingDirectory, IDictionary environmentVariables, Action errorCallback, Action exitCallBack, Action outputCallback) { - throw new NotImplementedException(); + var process = new FakeProcess(processPath, arguments, workingDirectory, environmentVariables, errorCallback, exitCallBack, outputCallback, FakeErrorAggregator); + Processes.Add(process); + + return process; } public void SetExitCallback(int processId, Action callbackAction) { - throw new NotImplementedException(); + // TODO: implement? } public void TerminateProcess(object process) { - throw new NotImplementedException(); + var fakeProcess = (FakeProcess)process; + fakeProcess.Exit(); } public bool TryGetExitCode(object process, out int exitCode) { - throw new NotImplementedException(); + exitCode = ((FakeProcess)process).ExitCode; + return true; } public void WaitForProcessExit(object process) { - throw new NotImplementedException(); + // todo: implement for timeouts? } } diff --git a/test/vstest.ProgrammerTests/Fakes/FakeTestBatchBuilder.cs b/test/vstest.ProgrammerTests/Fakes/FakeTestBatchBuilder.cs new file mode 100644 index 0000000000..0aec28a880 --- /dev/null +++ b/test/vstest.ProgrammerTests/Fakes/FakeTestBatchBuilder.cs @@ -0,0 +1,72 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + + + +#pragma warning disable IDE1006 // Naming Styles +using Microsoft.VisualStudio.TestPlatform.ObjectModel; + +namespace vstest.ProgrammerTests.CommandLine; + +internal class FakeTestBatchBuilder +{ + public int TotalCount { get; private set; } + public TimeSpan Duration { get; private set; } + public int BatchSize { get; private set; } + + public FakeTestBatchBuilder() + { + } + + /// + /// Total test count in all batches. + /// + internal FakeTestBatchBuilder WithTotalCount(int count) + { + TotalCount = count; + return this; + } + + internal FakeTestBatchBuilder WithDuration(TimeSpan duration) + { + + // TODO: add min duration and max duration, and distribution, if timing becomes relevant + // TODO: and replay rate, if we actually want to simulate stuff like really executing the tests + Duration = duration; + return this; + } + + /// + /// Splits the tests to batches of this size when reporting them back. + /// + /// + /// + internal FakeTestBatchBuilder WithBatchSize(int batchSize) + { + BatchSize = batchSize; + return this; + } + + internal List> Build() + { + var numberOfBatches = Math.DivRem(TotalCount, BatchSize, out int remainder); + + // TODO: Add adapter uri, and dll name + // TODO: set duration + var batches = + Enumerable.Range(0, numberOfBatches) + .Select(batchNumber => + Enumerable.Range(0, BatchSize) + .Select((index) => new TestResult(new TestCase($"Test{batchNumber}-{index}", new Uri("some://uri"), "DummySourceFileName"))).ToList()).ToList(); + + if (remainder > 0) + { + var reminderBatch = Enumerable.Range(0, BatchSize) + .Select((index) => new TestResult(new TestCase($"Test{numberOfBatches + 1}-{index}", new Uri("some://uri"), "DummySourceFileName"))).ToList(); + + batches.Add(reminderBatch); + } + + return batches; + } +} diff --git a/test/vstest.ProgrammerTests/Fakes/FakeDllFile.cs b/test/vstest.ProgrammerTests/Fakes/FakeTestDllFile.cs similarity index 53% rename from test/vstest.ProgrammerTests/Fakes/FakeDllFile.cs rename to test/vstest.ProgrammerTests/Fakes/FakeTestDllFile.cs index 4bc1aedd75..3c215861f9 100644 --- a/test/vstest.ProgrammerTests/Fakes/FakeDllFile.cs +++ b/test/vstest.ProgrammerTests/Fakes/FakeTestDllFile.cs @@ -9,14 +9,16 @@ namespace vstest.ProgrammerTests.CommandLine; -internal class FakeDllFile : FakeFile +internal class FakeTestDllFile : FakeFile { - public FrameworkName FrameworkName { get; init; } - public Architecture Architecture { get; init; } + public FrameworkName FrameworkName { get; } + public Architecture Architecture { get; } + public List> TestResultBatches { get; } - public FakeDllFile(string path, FrameworkName frameworkName, Architecture architecture) : base(path) + public FakeTestDllFile(string path, FrameworkName frameworkName, Architecture architecture, List> testResultBatches) : base(path) { FrameworkName = frameworkName; Architecture = architecture; + TestResultBatches = testResultBatches; } } diff --git a/test/vstest.ProgrammerTests/Fakes/FakeTestHostProcess.cs b/test/vstest.ProgrammerTests/Fakes/FakeTestHostProcess.cs index 723f5b4099..f78ef8ae11 100644 --- a/test/vstest.ProgrammerTests/Fakes/FakeTestHostProcess.cs +++ b/test/vstest.ProgrammerTests/Fakes/FakeTestHostProcess.cs @@ -5,11 +5,12 @@ #pragma warning disable IDE1006 // Naming Styles namespace vstest.ProgrammerTests.CommandLine.Fakes; -internal class FakeTestHostProcess : FakeProcess -{ - public FakeTestHostProcess(string commandLine, string arguments, FakeErrorAggregator fakeErrorAggregator) : base(commandLine, arguments, fakeErrorAggregator) - { - } - - public CapturedRunSettings? RunSettings { get; internal set; } -} +// TODO: was used in first test but is not correct design +//internal class FakeTestHostProcess : FakeProcess +//{ +// public FakeTestHostProcess(string commandLine, string arguments, FakeErrorAggregator fakeErrorAggregator) : base(commandLine, arguments, fakeErrorAggregator) +// { +// } + +// public CapturedRunSettings? RunSettings { get; internal set; } +//} diff --git a/test/vstest.ProgrammerTests/Fakes/FakeTestRunEventsRegistrar.cs b/test/vstest.ProgrammerTests/Fakes/FakeTestRunEventsRegistrar.cs index 9b221feb2b..ddd8c5d57a 100644 --- a/test/vstest.ProgrammerTests/Fakes/FakeTestRunEventsRegistrar.cs +++ b/test/vstest.ProgrammerTests/Fakes/FakeTestRunEventsRegistrar.cs @@ -16,18 +16,18 @@ public FakeTestRunEventsRegistrar(FakeErrorAggregator fakeErrorAggregator) } public List AllEvents { get; } = new(); - public List Warnings { get; } = new(); - public List> RunCompletionEvents { get; } = new(); + public List LoggedWarnings { get; } = new(); + public List> RunCompleteEvents { get; } = new(); public List> RunStartEvents { get; } = new(); - public List> RunStatsChange { get; } = new(); + public List> RunChangedEvents { get; } = new(); public List> RawMessageEvents { get; } = new(); - public List> TestRunMessageEvents { get; } = new(); + public List> RunMessageEvents { get; } = new(); public FakeErrorAggregator FakeErrorAggregator { get; } public void LogWarning(string message) { AllEvents.Add(message); - Warnings.Add(message); + LoggedWarnings.Add(message); } public void RegisterTestRunEvents(ITestRunRequest testRunRequest) @@ -52,7 +52,7 @@ private void OnRunCompletion(object? sender, TestRunCompleteEventArgs e) { var eventRecord = new EventRecord(sender, e); AllEvents.Add(eventRecord); - RunCompletionEvents.Add(eventRecord); + RunCompleteEvents.Add(eventRecord); } private void OnRunStart(object? sender, TestRunStartEventArgs e) @@ -66,7 +66,7 @@ private void OnRunStatsChange(object? sender, TestRunChangedEventArgs e) { var eventRecord = new EventRecord(sender, e); AllEvents.Add(eventRecord); - RunStatsChange.Add(eventRecord); + RunChangedEvents.Add(eventRecord); } private void OnRawMessage(object? sender, string e) @@ -79,7 +79,11 @@ private void OnRawMessage(object? sender, string e) private void OnTestRunMessage(object? sender, TestRunMessageEventArgs e) { var eventRecord = new EventRecord(sender, e); + if (e.Level == TestMessageLevel.Error) + { + FakeErrorAggregator.Errors.Add(eventRecord); + } AllEvents.Add(eventRecord); - TestRunMessageEvents.Add(eventRecord); + RunMessageEvents.Add(eventRecord); } } diff --git a/test/vstest.ProgrammerTests/Fakes/FakeTestRuntimeProvider.cs b/test/vstest.ProgrammerTests/Fakes/FakeTestRuntimeProvider.cs index 43e60e3720..d3b89e6c2f 100644 --- a/test/vstest.ProgrammerTests/Fakes/FakeTestRuntimeProvider.cs +++ b/test/vstest.ProgrammerTests/Fakes/FakeTestRuntimeProvider.cs @@ -73,15 +73,31 @@ public IEnumerable GetTestSources(IEnumerable sources) public void Initialize(IMessageLogger logger, string runsettingsXml) { + // TODO: this is called twice, is that okay? + // TODO: and also by HandlePartialRunComplete after the test run has completed and we aborted because the client disconnected + // do nothing } public Task LaunchTestHostAsync(TestProcessStartInfo testHostStartInfo, CancellationToken cancellationToken) { - // todo: Add fake process for testhost - // TestHostProcess = FakeProcessHelper.LaunchProcess(testHostStartInfo); - //HostLaunched(this, new HostProviderEventArgs("Fake testhost launched", 0, TestHostProcess.Id)); - //TestHostProcess.ProcessExited += (s, e) => HostExited(s, new HostProviderEventArgs($"Fake testhost {TestHostProcess.Id} exited", -99999, e)); + TestHostProcess = (FakeProcess)FakeProcessHelper.LaunchProcess( + testHostStartInfo.FileName, + testHostStartInfo.Arguments, + testHostStartInfo.WorkingDirectory, + testHostStartInfo.EnvironmentVariables, + errorCallback: (_, _) => { }, + exitCallBack: p => { + var process = (FakeProcess)p; + if (HostExited != null) + { + // TODO: When we exit, eventually there are no subscribers, maybe we should review if we don't lose the error output sometimes, in unnecessary way + HostExited(this, new HostProviderEventArgs(process.ErrorOutput, process.ExitCode, process.Id)); + } + }, + outputCallback: (_, _) => { } + ); + HostLaunched(this, new HostProviderEventArgs("Fake testhost launched", 0, TestHostProcess.Id)); return Task.FromResult(true); } diff --git a/test/vstest.ProgrammerTests/UnitTest1.cs b/test/vstest.ProgrammerTests/UnitTest1.cs index 46fb46f481..ebd948d8af 100644 --- a/test/vstest.ProgrammerTests/UnitTest1.cs +++ b/test/vstest.ProgrammerTests/UnitTest1.cs @@ -6,6 +6,7 @@ using System.Runtime.Versioning; using FluentAssertions; +using FluentAssertions.Extensions; using Microsoft.VisualStudio.TestPlatform.Client; using Microsoft.VisualStudio.TestPlatform.CommandLine; @@ -13,12 +14,14 @@ using Microsoft.VisualStudio.TestPlatform.CommandLine.TestPlatformHelpers; using Microsoft.VisualStudio.TestPlatform.CommandLineUtilities; using Microsoft.VisualStudio.TestPlatform.CommunicationUtilities.Interfaces; +using Microsoft.VisualStudio.TestPlatform.CommunicationUtilities.ObjectModel; using Microsoft.VisualStudio.TestPlatform.CrossPlatEngine; using Microsoft.VisualStudio.TestPlatform.CrossPlatEngine.TestRunAttachmentsProcessing; using Microsoft.VisualStudio.TestPlatform.ObjectModel; using Microsoft.VisualStudio.TestPlatform.ObjectModel.Client; using vstest.ProgrammerTests.CommandLine.Fakes; +using vstest.ProgrammerTests.Fakes; #pragma warning disable IDE1006 // Naming Styles namespace vstest.ProgrammerTests.CommandLine; @@ -29,37 +32,74 @@ internal class InlineRunSettingsTests { public void GivenInlineRunsettingsWhenCallingVstestConsoleThenTheyPropagateToTestHost() { - using Fixture fixture = new(); - fixture.VstestConsole - .WithSource(TestDlls.MSTest1) - .WithArguments($" -- {RunConfiguration.MaxParallelLevel.InlinePath}=3") - .Execute(); - - fixture.Processes.Should().HaveCount(1); - var process = fixture.Processes.First(); - process.Should().BeAssignableTo(); - var testhost = (FakeTestHostProcess)process; - testhost.RunSettings.Should().NotBeNull(); - testhost.RunSettings!.MaxParallelLevel.Should().Be(3); + //using Fixture fixture = new(); + //fixture.VstestConsole + // .WithSource(TestDlls.MSTest1) + // .WithArguments($" -- {RunConfiguration.MaxParallelLevel.InlinePath}=3") + // .Execute(); + + //fixture.Processes.Should().HaveCount(1); + //var process = fixture.Processes.First(); + //process.Should().BeAssignableTo(); + //var testhost = (FakeTestHostProcess)process; + //testhost.RunSettings.Should().NotBeNull(); + //testhost.RunSettings!.MaxParallelLevel.Should().Be(3); } } public class TestDiscoveryTests { - public async Task GivenAnMSTestAssemblyWith5Tests_WhenTestsAreRun_Then5TestsAreExecuted() + public async Task GivenAnMSTestAssemblyWith108Tests_WhenTestsAreRun_Then108TestsAreExecuted() { // -- arrange var fakeErrorAggregator = new FakeErrorAggregator(); var commandLineOptions = CommandLineOptions.Instance; - var fakeCurrentProcess = new FakeProcess(@"C:\temp\vstest.console.exe", string.Empty, fakeErrorAggregator); + var fakeCurrentProcess = new FakeProcess(@"C:\temp\vstest.console.exe", string.Empty, null, null, null, null, null, fakeErrorAggregator); var fakeProcessHelper = new FakeProcessHelper(fakeCurrentProcess, fakeErrorAggregator); - var fakeCommunicationEndpoint = new FakeCommunicationEndpoint(fakeErrorAggregator); + var fakeFileHelper = new FakeFileHelper(fakeErrorAggregator); + // TODO: Get framework name from constants + // TODO: have mstest1dll canned + var tests = new FakeTestBatchBuilder() + .WithTotalCount(108) + .WithDuration(100.Milliseconds()) + .WithBatchSize(10) + .Build(); + var mstest1Dll = new FakeTestDllFile(@"C:\temp\mstest1.dll", new FrameworkName(".NETCoreApp,Version=v5.0"), Architecture.X64, tests); + fakeFileHelper.AddFile(mstest1Dll); + + List changeMessages = tests.Take(tests.Count - 1).Select(batch => // TODO: make the stats agree with the tests below + new FakeMessage(MessageType.TestRunStatsChange, + new TestRunChangedEventArgs(new TestRunStatistics(new Dictionary { [TestOutcome.Passed] = batch.Count }), batch, new List()) + )).ToList(); + FakeMessage completedMessage = new FakeMessage(MessageType.ExecutionComplete, new TestRunCompletePayload + { + // TODO: make the stats agree with the tests below + TestRunCompleteArgs = new TestRunCompleteEventArgs(new TestRunStatistics(new Dictionary { [TestOutcome.Passed] = 1 }), false, false, null, new System.Collections.ObjectModel.Collection(), TimeSpan.Zero), + LastRunTests = new TestRunChangedEventArgs(new TestRunStatistics(new Dictionary { [TestOutcome.Passed] = 1 }), tests.Last(), new List()), + }); + List messages = changeMessages.Concat(new[] { completedMessage }).ToList(); + var responses = new List> { + new RequestResponsePair(MessageType.VersionCheck, new FakeMessage(MessageType.VersionCheck, 5)), + new RequestResponsePair(MessageType.ExecutionInitialize, FakeMessage.NoResponse), + new RequestResponsePair(MessageType.StartTestExecutionWithSources, messages), + new RequestResponsePair(MessageType.SessionEnd, message => + { + // TODO: how do we associate this to the correct process? + var fp = fakeProcessHelper.Processes.Last(); + fakeProcessHelper.TerminateProcess(fp); + + return new List { FakeMessage.NoResponse }; + }), + }; + + + var fakeCommunicationEndpoint = new FakeCommunicationEndpoint(new FakeCommunicationChannel(responses, fakeErrorAggregator), fakeErrorAggregator); TestServiceLocator.Register(fakeCommunicationEndpoint); var fakeTestRuntimeProviderManager = new FakeTestRuntimeProviderManager(fakeProcessHelper, fakeCommunicationEndpoint, fakeErrorAggregator); var testEngine = new TestEngine(fakeTestRuntimeProviderManager, fakeProcessHelper); - var fakeFileHelper = new FakeFileHelper(fakeErrorAggregator); + var testPlatform = new TestPlatform(testEngine, fakeFileHelper, fakeTestRuntimeProviderManager); var testRunResultAggregator = new TestRunResultAggregator(); @@ -87,10 +127,6 @@ public async Task GivenAnMSTestAssemblyWith5Tests_WhenTestsAreRun_Then5TestsAreE // -- act - // TODO: Get framework name from constants - // TODO: have mstest1dll canned - var mstest1Dll = new FakeDllFile(@"C:\temp\mstest1.dll", new FrameworkName(".NETCoreApp,Version=v5.0"), Architecture.X64); - fakeFileHelper.AddFile(mstest1Dll); // TODO: this gives me run configuration that is way too complete, do we a way to generate "bare" runsettings? if not we should add them. Would be also useful to get // runsettings from parameter set so people can use it // TODO: TestSessionTimeout gives me way to abort the run without having to cancel it externally, but could probably still lead to hangs if that funtionality is broken @@ -110,12 +146,12 @@ public async Task GivenAnMSTestAssemblyWith5Tests_WhenTestsAreRun_Then5TestsAreE var fakeTestRunEventsRegistrar = new FakeTestRunEventsRegistrar(fakeErrorAggregator); var protocolConfig = new ProtocolConfig(); + // TODO: we make sure the test is running 10 minutes at max and then we try to abort + // if we aborted we write the error to aggregator, this needs to be made into a pattern + // so we can avoid hanging if the run does not complete correctly var cancelAbort = new CancellationTokenSource(); var task = Task.Run(async () => { - // TODO: we make sure the test is running 1 minute at max and then we try to abort - // if we aborted we write the error to aggregator, this needs to be made into a pattern - // so we can avoid hanging if the run does not complete correctly await Task.Delay(TimeSpan.FromMinutes(10), cancelAbort.Token); if (Debugger.IsAttached) { @@ -128,10 +164,15 @@ public async Task GivenAnMSTestAssemblyWith5Tests_WhenTestsAreRun_Then5TestsAreE }); testRequestManager.RunTests(testRunRequestPayload, testHostLauncher: null, fakeTestRunEventsRegistrar, protocolConfig); cancelAbort.Cancel(); - await task; + if (!task.IsCanceled) + { + await task; + } + // pattern end // -- assert fakeErrorAggregator.Errors.Should().BeEmpty(); + fakeTestRunEventsRegistrar.RunChangedEvents.SelectMany(er => er.Data.NewTestResults).Should().HaveCount(110); } } From 9ce6bc09a9ac647b8d89c66b6f8e76988205dc94 Mon Sep 17 00:00:00 2001 From: nohwnd Date: Wed, 16 Feb 2022 13:04:12 +0100 Subject: [PATCH 10/32] Format the test names so I can read them better --- .../TestEngine.cs | 2 +- .../TestServiceLocator.cs | 4 +- .../ArtifactProcessingCollectModeProcessor.cs | 4 +- ...ifactProcessingPostProcessModeProcessor.cs | 6 +- .../TestSessionCorrelationIdProcessor.cs | 4 +- test/Intent/ConsoleLogger.cs | 24 +++- .../Fakes/FakeFileHelper.cs | 22 ++- test/vstest.ProgrammerTests/UnitTest1.cs | 133 +++++++++++++++++- test/vstest.ProgrammerTests/VstestConsole.cs | 11 +- 9 files changed, 180 insertions(+), 30 deletions(-) diff --git a/src/Microsoft.TestPlatform.CrossPlatEngine/TestEngine.cs b/src/Microsoft.TestPlatform.CrossPlatEngine/TestEngine.cs index 83bd6966e7..e3810008fe 100644 --- a/src/Microsoft.TestPlatform.CrossPlatEngine/TestEngine.cs +++ b/src/Microsoft.TestPlatform.CrossPlatEngine/TestEngine.cs @@ -34,7 +34,7 @@ namespace Microsoft.VisualStudio.TestPlatform.CrossPlatEngine; /// public class TestEngine : ITestEngine { - private readonly TestRuntimeProviderManager _testHostProviderManager; + private readonly ITestRuntimeProviderManager _testHostProviderManager; private ITestExtensionManager _testExtensionManager; private readonly IProcessHelper _processHelper; diff --git a/src/Microsoft.TestPlatform.ObjectModel/TestServiceLocator.cs b/src/Microsoft.TestPlatform.ObjectModel/TestServiceLocator.cs index c3eefb2b21..425241c0fa 100644 --- a/src/Microsoft.TestPlatform.ObjectModel/TestServiceLocator.cs +++ b/src/Microsoft.TestPlatform.ObjectModel/TestServiceLocator.cs @@ -1,11 +1,11 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT license. See LICENSE file in the project root for full license information. +namespace Microsoft.VisualStudio.TestPlatform.ObjectModel; + using System; using System.Collections.Generic; -namespace Microsoft.VisualStudio.TestPlatform.ObjectModel; - // TODO: Make this internal, I am just trying to have easier time trying this out. public static class TestServiceLocator { diff --git a/src/vstest.console/Processors/ArtifactProcessingCollectModeProcessor.cs b/src/vstest.console/Processors/ArtifactProcessingCollectModeProcessor.cs index b085ba3637..f651207cb5 100644 --- a/src/vstest.console/Processors/ArtifactProcessingCollectModeProcessor.cs +++ b/src/vstest.console/Processors/ArtifactProcessingCollectModeProcessor.cs @@ -91,9 +91,9 @@ internal class ArtifactProcessingCollectModeProcessorExecutor : IArgumentExecuto { private readonly CommandLineOptions _commandLineOptions; - public ArtifactProcessingCollectModeProcessorExecutor(CommandLineOptions options) + public ArtifactProcessingCollectModeProcessorExecutor(CommandLineOptions options!!) { - _commandLineOptions = options ?? throw new ArgumentNullException(nameof(options)); + _commandLineOptions = options; } public void Initialize(string argument) diff --git a/src/vstest.console/Processors/ArtifactProcessingPostProcessModeProcessor.cs b/src/vstest.console/Processors/ArtifactProcessingPostProcessModeProcessor.cs index 1693e0d058..65eee5d2b4 100644 --- a/src/vstest.console/Processors/ArtifactProcessingPostProcessModeProcessor.cs +++ b/src/vstest.console/Processors/ArtifactProcessingPostProcessModeProcessor.cs @@ -95,10 +95,10 @@ internal class ArtifactProcessingPostProcessModeProcessorExecutor : IArgumentExe private readonly CommandLineOptions _commandLineOptions; private readonly IArtifactProcessingManager _artifactProcessingManage; - public ArtifactProcessingPostProcessModeProcessorExecutor(CommandLineOptions options, IArtifactProcessingManager artifactProcessingManager) + public ArtifactProcessingPostProcessModeProcessorExecutor(CommandLineOptions options!!, IArtifactProcessingManager artifactProcessingManager!!) { - _commandLineOptions = options ?? throw new ArgumentNullException(nameof(options)); - _artifactProcessingManage = artifactProcessingManager ?? throw new ArgumentNullException(nameof(artifactProcessingManager)); ; + _commandLineOptions = options; + _artifactProcessingManage = artifactProcessingManager; ; } public void Initialize(string argument) diff --git a/src/vstest.console/Processors/TestSessionCorrelationIdProcessor.cs b/src/vstest.console/Processors/TestSessionCorrelationIdProcessor.cs index 053ce3f9d5..52b00e8273 100644 --- a/src/vstest.console/Processors/TestSessionCorrelationIdProcessor.cs +++ b/src/vstest.console/Processors/TestSessionCorrelationIdProcessor.cs @@ -86,9 +86,9 @@ internal class TestSessionCorrelationIdProcessorModeProcessorExecutor : IArgumen { private readonly CommandLineOptions _commandLineOptions; - public TestSessionCorrelationIdProcessorModeProcessorExecutor(CommandLineOptions options) + public TestSessionCorrelationIdProcessorModeProcessorExecutor(CommandLineOptions options!!) { - _commandLineOptions = options ?? throw new ArgumentNullException(nameof(options)); + _commandLineOptions = options; } public void Initialize(string argument) diff --git a/test/Intent/ConsoleLogger.cs b/test/Intent/ConsoleLogger.cs index af1edb5543..45d9dd5805 100644 --- a/test/Intent/ConsoleLogger.cs +++ b/test/Intent/ConsoleLogger.cs @@ -1,20 +1,20 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT license. See LICENSE file in the project root for full license information. +namespace Intent.Console; using System.Reflection; +using System.Text.RegularExpressions; using static System.Console; using static System.ConsoleColor; -namespace Intent.Console; - internal class ConsoleLogger : IRunLogger { public void WriteTestInconclusive(MethodInfo m) { var currentColor = ForegroundColor; ForegroundColor = Yellow; - WriteLine($"[?] {m.Name} inconclusive"); + WriteLine($"[?] {FormatMethodName(m.Name)}"); ForegroundColor = currentColor; } @@ -22,7 +22,7 @@ public void WriteTestPassed(MethodInfo m) { var currentColor = ForegroundColor; ForegroundColor = Green; - WriteLine($"[+] {m.Name} passed"); + WriteLine($"[+] {FormatMethodName(m.Name)}"); ForegroundColor = currentColor; } @@ -30,7 +30,7 @@ public void WriteTestFailure(MethodInfo m, Exception ex) { var currentColor = ForegroundColor; ForegroundColor = Red; - WriteLine($"[-] {m.Name} failed{Environment.NewLine}{ex}"); + WriteLine($"[-] {FormatMethodName(m.Name)}{Environment.NewLine}{ex}"); ForegroundColor = currentColor; } @@ -41,4 +41,18 @@ public void WriteFrameworkError(Exception ex) WriteLine($"[-] framework failed{Environment.NewLine}{ex}{Environment.NewLine}{Environment.NewLine}"); ForegroundColor = currentColor; } + + private static string FormatMethodName(string methodName) + { + var noUnderscores = methodName.Replace('_', ' '); + // insert space before every capital letter or number that is after a non-capital letter + var spaced = Regex.Replace(noUnderscores, "(?<=[a-z])([A-Z0-9])", " $1"); + // insert space before every capital leter that is after a number + var spaced2 = Regex.Replace(spaced, "(?<=[0-9]|^)([A-Z])", " $1"); + var newLines = spaced2.Replace("When", $"{Environment.NewLine} When") + .Replace("Then", $"{Environment.NewLine} Then"); + + return newLines.ToLowerInvariant(); + } + } diff --git a/test/vstest.ProgrammerTests/Fakes/FakeFileHelper.cs b/test/vstest.ProgrammerTests/Fakes/FakeFileHelper.cs index fa8584e375..f9d3a02329 100644 --- a/test/vstest.ProgrammerTests/Fakes/FakeFileHelper.cs +++ b/test/vstest.ProgrammerTests/Fakes/FakeFileHelper.cs @@ -1,11 +1,10 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT license. See LICENSE file in the project root for full license information. -using Microsoft.VisualStudio.TestPlatform.Utilities.Helpers.Interfaces; - -#pragma warning disable IDE1006 // Naming Styles namespace vstest.ProgrammerTests.CommandLine.Fakes; +using Microsoft.VisualStudio.TestPlatform.Utilities.Helpers.Interfaces; + internal class FakeFileHelper : IFileHelper { public FakeFileHelper(FakeErrorAggregator fakeErrorAggregator) @@ -31,6 +30,11 @@ public void Delete(string path) throw new NotImplementedException(); } + public void DeleteDirectory(string directoryPath, bool recursive) + { + throw new NotImplementedException(); + } + public void DeleteEmptyDirectroy(string directoryPath) { throw new NotImplementedException(); @@ -39,7 +43,7 @@ public void DeleteEmptyDirectroy(string directoryPath) public bool DirectoryExists(string path) { // TODO: Check if any file has the directory in name. This will improve. - var directoryExists = Files.Select(f => Path.GetDirectoryName(f.Path)).Any(p => p.StartsWith(path)); + var directoryExists = Files.Select(f => Path.GetDirectoryName(f.Path)).Any(p => p != null && p.StartsWith(path)); return directoryExists; } @@ -68,6 +72,11 @@ public FileAttributes GetFileAttributes(string path) throw new NotImplementedException(); } + public long GetFileLength(string path) + { + throw new NotImplementedException(); + } + public string[] GetFiles(string path, string searchPattern, SearchOption searchOption) { throw new NotImplementedException(); @@ -93,6 +102,11 @@ public Stream GetStream(string filePath, FileMode mode, FileAccess access, FileS throw new NotImplementedException(); } + public string GetTempPath() + { + throw new NotImplementedException(); + } + public void MoveFile(string sourcePath, string destinationPath) { throw new NotImplementedException(); diff --git a/test/vstest.ProgrammerTests/UnitTest1.cs b/test/vstest.ProgrammerTests/UnitTest1.cs index ebd948d8af..5552e9e7e7 100644 --- a/test/vstest.ProgrammerTests/UnitTest1.cs +++ b/test/vstest.ProgrammerTests/UnitTest1.cs @@ -1,6 +1,7 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT license. See LICENSE file in the project root for full license information. +namespace vstest.ProgrammerTests.CommandLine; using System.Diagnostics; using System.Runtime.Versioning; @@ -23,10 +24,6 @@ using vstest.ProgrammerTests.CommandLine.Fakes; using vstest.ProgrammerTests.Fakes; -#pragma warning disable IDE1006 // Naming Styles -namespace vstest.ProgrammerTests.CommandLine; -#pragma warning restore IDE1006 // Naming Styles - // exluded from run internal class InlineRunSettingsTests { @@ -95,6 +92,132 @@ public async Task GivenAnMSTestAssemblyWith108Tests_WhenTestsAreRun_Then108Tests }; + var fakeCommunicationEndpoint = new FakeCommunicationEndpoint(new FakeCommunicationChannel(responses, fakeErrorAggregator), fakeErrorAggregator); + TestServiceLocator.Register(fakeCommunicationEndpoint); + var fakeTestRuntimeProviderManager = new FakeTestRuntimeProviderManager(fakeProcessHelper, fakeCommunicationEndpoint, fakeErrorAggregator); + var testEngine = new TestEngine(fakeTestRuntimeProviderManager, fakeProcessHelper); + + var testPlatform = new TestPlatform(testEngine, fakeFileHelper, fakeTestRuntimeProviderManager); + + var testRunResultAggregator = new TestRunResultAggregator(); + var fakeTestPlatformEventSource = new FakeTestPlatformEventSource(fakeErrorAggregator); + + var fakeAssemblyMetadataProvider = new FakeAssemblyMetadataProvider(fakeFileHelper, fakeErrorAggregator); + var inferHelper = new InferHelper(fakeAssemblyMetadataProvider); + + // This is most likely not the correctl place where to cut this off, plugin cache is probably the better place, + // but it is not injected, and I don't want to investigate this now. + var fakeDataCollectorAttachmentsProcessorsFactory = new FakeDataCollectorAttachmentsProcessorsFactory(fakeErrorAggregator); + var testRunAttachmentsProcessingManager = new TestRunAttachmentsProcessingManager(fakeTestPlatformEventSource, fakeDataCollectorAttachmentsProcessorsFactory); + + Task fakeMetricsPublisherTask = Task.FromResult(new FakeMetricsPublisher(fakeErrorAggregator)); + TestRequestManager testRequestManager = new( + commandLineOptions, + testPlatform, + testRunResultAggregator, + fakeTestPlatformEventSource, + inferHelper, + fakeMetricsPublisherTask, + fakeProcessHelper, + testRunAttachmentsProcessingManager + ); + + // -- act + + // TODO: this gives me run configuration that is way too complete, do we a way to generate "bare" runsettings? if not we should add them. Would be also useful to get + // runsettings from parameter set so people can use it + // TODO: TestSessionTimeout gives me way to abort the run without having to cancel it externally, but could probably still lead to hangs if that funtionality is broken + // TODO: few tries later, that is exactly the case when we abort, it still hangs on waiting to complete request, because test run complete was not sent + // var runConfiguration = new Microsoft.VisualStudio.TestPlatform.ObjectModel.RunConfiguration { TestSessionTimeout = 40_000 }.ToXml().OuterXml; + var runConfiguration = string.Empty; + var testRunRequestPayload = new TestRunRequestPayload + { + // TODO: passing null sources and null testcases does not fail fast + Sources = mstest1Dll.Path.ToList(), + // TODO: passing null runsettings does not fail fast, instead it fails in Fakes settings code + // TODO: passing empty string fails in the xml parser code + RunSettings = $"{runConfiguration}" + }; + + // var fakeTestHostLauncher = new FakeTestHostLauncher(); + var fakeTestRunEventsRegistrar = new FakeTestRunEventsRegistrar(fakeErrorAggregator); + var protocolConfig = new ProtocolConfig(); + + // TODO: we make sure the test is running 10 minutes at max and then we try to abort + // if we aborted we write the error to aggregator, this needs to be made into a pattern + // so we can avoid hanging if the run does not complete correctly + var cancelAbort = new CancellationTokenSource(); + var task = Task.Run(async () => + { + await Task.Delay(TimeSpan.FromMinutes(10), cancelAbort.Token); + if (Debugger.IsAttached) + { + // we will abort because we are hanging, look at stacks to see what the problem is + Debugger.Break(); + } + fakeErrorAggregator.Add(new Exception("errr we aborted")); + testRequestManager.AbortTestRun(); + + }); + testRequestManager.RunTests(testRunRequestPayload, testHostLauncher: null, fakeTestRunEventsRegistrar, protocolConfig); + cancelAbort.Cancel(); + if (!task.IsCanceled) + { + await task; + } + // pattern end + + // -- assert + fakeErrorAggregator.Errors.Should().BeEmpty(); + fakeTestRunEventsRegistrar.RunChangedEvents.SelectMany(er => er.Data.NewTestResults).Should().HaveCount(110); + } + + public async Task GivenMultipleMsTestAssembliesThatUseTheSameTargetFramework_WhenTestsAreRun_ThenAllTestsFromAllTargetFrameworksAreRun() + { + // -- arrange + var fakeErrorAggregator = new FakeErrorAggregator(); + var commandLineOptions = CommandLineOptions.Instance; + + var fakeCurrentProcess = new FakeProcess(@"C:\temp\vstest.console.exe", string.Empty, null, null, null, null, null, fakeErrorAggregator); + var fakeProcessHelper = new FakeProcessHelper(fakeCurrentProcess, fakeErrorAggregator); + + var fakeFileHelper = new FakeFileHelper(fakeErrorAggregator); + // TODO: Get framework name from constants + // TODO: have mstest1dll canned + var tests = new FakeTestBatchBuilder() + .WithTotalCount(108) + .WithDuration(100.Milliseconds()) + .WithBatchSize(10) + .Build(); + var mstest1Dll = new FakeTestDllFile(@"C:\temp\mstest1.dll", new FrameworkName(".NETCoreApp,Version=v5.0"), Architecture.X64, tests); + fakeFileHelper.AddFile(mstest1Dll); + + List changeMessages = tests.Take(tests.Count - 1).Select(batch => // TODO: make the stats agree with the tests below + new FakeMessage(MessageType.TestRunStatsChange, + new TestRunChangedEventArgs(new TestRunStatistics(new Dictionary { [TestOutcome.Passed] = batch.Count }), batch, new List()) + )).ToList(); + FakeMessage completedMessage = new FakeMessage(MessageType.ExecutionComplete, new TestRunCompletePayload + { + // TODO: make the stats agree with the tests below + TestRunCompleteArgs = new TestRunCompleteEventArgs(new TestRunStatistics(new Dictionary { [TestOutcome.Passed] = 1 }), false, false, null, new System.Collections.ObjectModel.Collection(), TimeSpan.Zero), + LastRunTests = new TestRunChangedEventArgs(new TestRunStatistics(new Dictionary { [TestOutcome.Passed] = 1 }), tests.Last(), new List()), + }); + List messages = changeMessages.Concat(new[] { completedMessage }).ToList(); + var responses = new List> { + new RequestResponsePair(MessageType.VersionCheck, new FakeMessage(MessageType.VersionCheck, 5)), + new RequestResponsePair(MessageType.ExecutionInitialize, FakeMessage.NoResponse), + new RequestResponsePair(MessageType.StartTestExecutionWithSources, messages), + new RequestResponsePair(MessageType.SessionEnd, message => + { + // TODO: how do we associate this to the correct process? + var fp = fakeProcessHelper.Processes.Last(); + fakeProcessHelper.TerminateProcess(fp); + + return new List { FakeMessage.NoResponse }; + }), + }; + + var fakeCommunicationEndpoint = new FakeCommunicationEndpoint(new FakeCommunicationChannel(responses, fakeErrorAggregator), fakeErrorAggregator); TestServiceLocator.Register(fakeCommunicationEndpoint); var fakeTestRuntimeProviderManager = new FakeTestRuntimeProviderManager(fakeProcessHelper, fakeCommunicationEndpoint, fakeErrorAggregator); @@ -185,7 +308,7 @@ public Fixture() public List Processes { get; } = new(); - public VstestConsole VstestConsole { get; } = new(); + public VstestConsoleBuilder VstestConsole { get; } = new(); public FakeTestExtensionManager TestExtensionManager { get; } = new(); diff --git a/test/vstest.ProgrammerTests/VstestConsole.cs b/test/vstest.ProgrammerTests/VstestConsole.cs index e49ffa3315..a079e99919 100644 --- a/test/vstest.ProgrammerTests/VstestConsole.cs +++ b/test/vstest.ProgrammerTests/VstestConsole.cs @@ -1,13 +1,12 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT license. See LICENSE file in the project root for full license information. -using Microsoft.VisualStudio.TestPlatform.CommandLine; +namespace vstest.ProgrammerTests.CommandLine; +using Microsoft.VisualStudio.TestPlatform.CommandLine; using vstest.ProgrammerTests.CommandLine.Fakes; -namespace vstest.ProgrammerTests.CommandLine; - -internal class VstestConsole +internal class VstestConsoleBuilder { public List Sources { get; } = new(); @@ -15,13 +14,13 @@ internal class VstestConsole internal FakeOutput Output { get; } = new(); - internal VstestConsole WithSource(params string[] sources) + internal VstestConsoleBuilder WithSource(params string[] sources) { Sources.AddRange(sources); return this; } - internal VstestConsole WithArguments(params string[] arguments) + internal VstestConsoleBuilder WithArguments(params string[] arguments) { Arguments.AddRange(arguments); return this; From d6be5d5845508994305649160e3f722be5a5d9c0 Mon Sep 17 00:00:00 2001 From: nohwnd Date: Wed, 16 Feb 2022 13:44:57 +0100 Subject: [PATCH 11/32] About to split up to multiple assemblies. --- .../TestServiceLocator.cs | 6 ++++ .../Fakes/FakeTestBatchBuilder.cs | 7 ++--- .../StringExtensions.cs | 2 +- test/vstest.ProgrammerTests/UnitTest1.cs | 31 +++++++++++++------ 4 files changed, 31 insertions(+), 15 deletions(-) diff --git a/src/Microsoft.TestPlatform.ObjectModel/TestServiceLocator.cs b/src/Microsoft.TestPlatform.ObjectModel/TestServiceLocator.cs index 425241c0fa..693c5b2cee 100644 --- a/src/Microsoft.TestPlatform.ObjectModel/TestServiceLocator.cs +++ b/src/Microsoft.TestPlatform.ObjectModel/TestServiceLocator.cs @@ -27,6 +27,12 @@ public static TRegistration Get() #endif return (TRegistration)instance; } + + public static void Clear() + { + Instances.Clear(); + Resolves.Clear(); + } } // TODO: Make this internal, I am just trying to have easier time trying this out. diff --git a/test/vstest.ProgrammerTests/Fakes/FakeTestBatchBuilder.cs b/test/vstest.ProgrammerTests/Fakes/FakeTestBatchBuilder.cs index 0aec28a880..b7edfccb1f 100644 --- a/test/vstest.ProgrammerTests/Fakes/FakeTestBatchBuilder.cs +++ b/test/vstest.ProgrammerTests/Fakes/FakeTestBatchBuilder.cs @@ -1,13 +1,12 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT license. See LICENSE file in the project root for full license information. +namespace vstest.ProgrammerTests.CommandLine; +using System.Runtime.CompilerServices; -#pragma warning disable IDE1006 // Naming Styles using Microsoft.VisualStudio.TestPlatform.ObjectModel; -namespace vstest.ProgrammerTests.CommandLine; - internal class FakeTestBatchBuilder { public int TotalCount { get; private set; } @@ -61,7 +60,7 @@ internal List> Build() if (remainder > 0) { - var reminderBatch = Enumerable.Range(0, BatchSize) + var reminderBatch = Enumerable.Range(0, remainder) .Select((index) => new TestResult(new TestCase($"Test{numberOfBatches + 1}-{index}", new Uri("some://uri"), "DummySourceFileName"))).ToList(); batches.Add(reminderBatch); diff --git a/test/vstest.ProgrammerTests/StringExtensions.cs b/test/vstest.ProgrammerTests/StringExtensions.cs index b1406afd71..c0666063db 100644 --- a/test/vstest.ProgrammerTests/StringExtensions.cs +++ b/test/vstest.ProgrammerTests/StringExtensions.cs @@ -15,7 +15,7 @@ public static string JoinBySpace(this IEnumerable value) return string.Join(" ", value); } - public static List ToList(this string value) + public static List AsList(this string value) { return new List { value }; } diff --git a/test/vstest.ProgrammerTests/UnitTest1.cs b/test/vstest.ProgrammerTests/UnitTest1.cs index 5552e9e7e7..64b76bf3f2 100644 --- a/test/vstest.ProgrammerTests/UnitTest1.cs +++ b/test/vstest.ProgrammerTests/UnitTest1.cs @@ -52,7 +52,7 @@ public async Task GivenAnMSTestAssemblyWith108Tests_WhenTestsAreRun_Then108Tests var fakeErrorAggregator = new FakeErrorAggregator(); var commandLineOptions = CommandLineOptions.Instance; - var fakeCurrentProcess = new FakeProcess(@"C:\temp\vstest.console.exe", string.Empty, null, null, null, null, null, fakeErrorAggregator); + var fakeCurrentProcess = new FakeProcess(@"X:\fake\vstest.console.exe", string.Empty, null, null, null, null, null, fakeErrorAggregator); var fakeProcessHelper = new FakeProcessHelper(fakeCurrentProcess, fakeErrorAggregator); var fakeFileHelper = new FakeFileHelper(fakeErrorAggregator); @@ -63,7 +63,7 @@ public async Task GivenAnMSTestAssemblyWith108Tests_WhenTestsAreRun_Then108Tests .WithDuration(100.Milliseconds()) .WithBatchSize(10) .Build(); - var mstest1Dll = new FakeTestDllFile(@"C:\temp\mstest1.dll", new FrameworkName(".NETCoreApp,Version=v5.0"), Architecture.X64, tests); + var mstest1Dll = new FakeTestDllFile(@"X:\fake\mstest1.dll", new FrameworkName(".NETCoreApp,Version=v5.0"), Architecture.X64, tests); fakeFileHelper.AddFile(mstest1Dll); List changeMessages = tests.Take(tests.Count - 1).Select(batch => // TODO: make the stats agree with the tests below @@ -93,6 +93,7 @@ public async Task GivenAnMSTestAssemblyWith108Tests_WhenTestsAreRun_Then108Tests var fakeCommunicationEndpoint = new FakeCommunicationEndpoint(new FakeCommunicationChannel(responses, fakeErrorAggregator), fakeErrorAggregator); + TestServiceLocator.Clear(); TestServiceLocator.Register(fakeCommunicationEndpoint); var fakeTestRuntimeProviderManager = new FakeTestRuntimeProviderManager(fakeProcessHelper, fakeCommunicationEndpoint, fakeErrorAggregator); var testEngine = new TestEngine(fakeTestRuntimeProviderManager, fakeProcessHelper); @@ -133,7 +134,7 @@ public async Task GivenAnMSTestAssemblyWith108Tests_WhenTestsAreRun_Then108Tests var testRunRequestPayload = new TestRunRequestPayload { // TODO: passing null sources and null testcases does not fail fast - Sources = mstest1Dll.Path.ToList(), + Sources = mstest1Dll.Path.AsList(), // TODO: passing null runsettings does not fail fast, instead it fails in Fakes settings code // TODO: passing empty string fails in the xml parser code RunSettings = $"{runConfiguration}" @@ -149,7 +150,7 @@ public async Task GivenAnMSTestAssemblyWith108Tests_WhenTestsAreRun_Then108Tests var cancelAbort = new CancellationTokenSource(); var task = Task.Run(async () => { - await Task.Delay(TimeSpan.FromMinutes(10), cancelAbort.Token); + await Task.Delay(TimeSpan.FromSeconds(10), cancelAbort.Token); if (Debugger.IsAttached) { // we will abort because we are hanging, look at stacks to see what the problem is @@ -169,16 +170,16 @@ public async Task GivenAnMSTestAssemblyWith108Tests_WhenTestsAreRun_Then108Tests // -- assert fakeErrorAggregator.Errors.Should().BeEmpty(); - fakeTestRunEventsRegistrar.RunChangedEvents.SelectMany(er => er.Data.NewTestResults).Should().HaveCount(110); + fakeTestRunEventsRegistrar.RunChangedEvents.SelectMany(er => er.Data.NewTestResults).Should().HaveCount(108); } - public async Task GivenMultipleMsTestAssembliesThatUseTheSameTargetFramework_WhenTestsAreRun_ThenAllTestsFromAllTargetFrameworksAreRun() + public async Task GivenMultipleMsTestAssembliesThatUseTheSameTargetFrameworkAndArchitecture_WhenTestsAreRun_ThenAllTestsFromAllAssembliesAreRun() { // -- arrange var fakeErrorAggregator = new FakeErrorAggregator(); var commandLineOptions = CommandLineOptions.Instance; - var fakeCurrentProcess = new FakeProcess(@"C:\temp\vstest.console.exe", string.Empty, null, null, null, null, null, fakeErrorAggregator); + var fakeCurrentProcess = new FakeProcess(@"X:\fake\vstest.console.exe", string.Empty, null, null, null, null, null, fakeErrorAggregator); var fakeProcessHelper = new FakeProcessHelper(fakeCurrentProcess, fakeErrorAggregator); var fakeFileHelper = new FakeFileHelper(fakeErrorAggregator); @@ -189,9 +190,17 @@ public async Task GivenMultipleMsTestAssembliesThatUseTheSameTargetFramework_Whe .WithDuration(100.Milliseconds()) .WithBatchSize(10) .Build(); - var mstest1Dll = new FakeTestDllFile(@"C:\temp\mstest1.dll", new FrameworkName(".NETCoreApp,Version=v5.0"), Architecture.X64, tests); + var mstest1Dll = new FakeTestDllFile(@"X:\fake\mstest1.dll", new FrameworkName(".NETCoreApp,Version=v5.0"), Architecture.X64, tests); fakeFileHelper.AddFile(mstest1Dll); + var tests2 = new FakeTestBatchBuilder() + .WithTotalCount(108) + .WithDuration(100.Milliseconds()) + .WithBatchSize(10) + .Build(); + var mstest2Dll = new FakeTestDllFile(@"X:\fake\mstest2.dll", new FrameworkName(".NETCoreApp,Version=v5.0"), Architecture.X64, tests2); + fakeFileHelper.AddFile(mstest2Dll); + List changeMessages = tests.Take(tests.Count - 1).Select(batch => // TODO: make the stats agree with the tests below new FakeMessage(MessageType.TestRunStatsChange, new TestRunChangedEventArgs(new TestRunStatistics(new Dictionary { [TestOutcome.Passed] = batch.Count }), batch, new List()) @@ -219,6 +228,7 @@ public async Task GivenMultipleMsTestAssembliesThatUseTheSameTargetFramework_Whe var fakeCommunicationEndpoint = new FakeCommunicationEndpoint(new FakeCommunicationChannel(responses, fakeErrorAggregator), fakeErrorAggregator); + TestServiceLocator.Clear(); TestServiceLocator.Register(fakeCommunicationEndpoint); var fakeTestRuntimeProviderManager = new FakeTestRuntimeProviderManager(fakeProcessHelper, fakeCommunicationEndpoint, fakeErrorAggregator); var testEngine = new TestEngine(fakeTestRuntimeProviderManager, fakeProcessHelper); @@ -259,7 +269,7 @@ public async Task GivenMultipleMsTestAssembliesThatUseTheSameTargetFramework_Whe var testRunRequestPayload = new TestRunRequestPayload { // TODO: passing null sources and null testcases does not fail fast - Sources = mstest1Dll.Path.ToList(), + Sources = new List { mstest1Dll.Path, mstest2Dll.Path }, // TODO: passing null runsettings does not fail fast, instead it fails in Fakes settings code // TODO: passing empty string fails in the xml parser code RunSettings = $"{runConfiguration}" @@ -275,9 +285,10 @@ public async Task GivenMultipleMsTestAssembliesThatUseTheSameTargetFramework_Whe var cancelAbort = new CancellationTokenSource(); var task = Task.Run(async () => { - await Task.Delay(TimeSpan.FromMinutes(10), cancelAbort.Token); + await Task.Delay(TimeSpan.FromSeconds(5), cancelAbort.Token); if (Debugger.IsAttached) { + var errors = fakeErrorAggregator.Errors; // we will abort because we are hanging, look at stacks to see what the problem is Debugger.Break(); } From 8c91ed70d03871bd2093e68d2f4865d65db457b8 Mon Sep 17 00:00:00 2001 From: nohwnd Date: Wed, 16 Feb 2022 17:47:11 +0100 Subject: [PATCH 12/32] make code succint --- .../TestPlatform.cs | 5 + .../TestRequestSender.cs | 3 +- .../TestServiceLocator.cs | 18 +-- .../FakeTestDllBuilder.cs | 73 ++++++++++ .../FakeTestHostBuilder.cs | 22 +++ .../Fakes/FakeAssemblyMetadataProvider.cs | 5 +- .../Fakes/FakeCommunicationChannel.cs | 11 +- .../Fakes/FakeCommunicationEndpoint.cs | 31 ++++- .../Fakes/FakeProcess.cs | 4 +- .../Fakes/FakeProcessHelper.cs | 6 +- .../Fakes/FakeTestRuntimeProvider.cs | 16 +-- .../Fakes/FakeTestRuntimeProviderManager.cs | 24 +++- test/vstest.ProgrammerTests/Fixture.cs | 26 ++++ test/vstest.ProgrammerTests/KnownFramework.cs | 12 ++ test/vstest.ProgrammerTests/UnitTest1.cs | 130 ++++++++++-------- 15 files changed, 287 insertions(+), 99 deletions(-) create mode 100644 test/vstest.ProgrammerTests/FakeTestDllBuilder.cs create mode 100644 test/vstest.ProgrammerTests/FakeTestHostBuilder.cs create mode 100644 test/vstest.ProgrammerTests/Fixture.cs create mode 100644 test/vstest.ProgrammerTests/KnownFramework.cs diff --git a/src/Microsoft.TestPlatform.Client/TestPlatform.cs b/src/Microsoft.TestPlatform.Client/TestPlatform.cs index 61b749a80a..a125a70930 100644 --- a/src/Microsoft.TestPlatform.Client/TestPlatform.cs +++ b/src/Microsoft.TestPlatform.Client/TestPlatform.cs @@ -133,6 +133,11 @@ public ITestRunRequest CreateTestRunRequest( var loggerManager = TestEngine.GetLoggerManager(requestData); loggerManager.Initialize(testRunCriteria.TestRunSettings); + // TODO: PERF: this will create a testhost manager, and then it will pass that to GetExecutionManager, where it will + // be used only when we will run in-process. If we don't run in process, we will throw away the manager we just + // created and let the proxy parallel callbacks to create a new one. This seems to be very easy to move to the GetExecutionManager, + // and safe as well, so we create the manager only once. + // TODO: Of course TestEngine.GetExecutionManager is public api :D var testHostManager = _testHostProviderManager.GetTestHostManagerByRunConfiguration(testRunCriteria.TestRunSettings); ThrowExceptionIfTestHostManagerIsNull(testHostManager, testRunCriteria.TestRunSettings); diff --git a/src/Microsoft.TestPlatform.CommunicationUtilities/TestRequestSender.cs b/src/Microsoft.TestPlatform.CommunicationUtilities/TestRequestSender.cs index fd1510716f..e1a39fcd28 100644 --- a/src/Microsoft.TestPlatform.CommunicationUtilities/TestRequestSender.cs +++ b/src/Microsoft.TestPlatform.CommunicationUtilities/TestRequestSender.cs @@ -107,7 +107,7 @@ internal TestRequestSender( // resort of getting the dependency into the execution flow. // TODO: I am not sure if we need multiple instances of ICommunicationEndpoint, in that case we should register // and resolve Func and invoke that. - _communicationEndpoint = communicationEndPoint ?? TestServiceLocator.Get() ?? SetCommunicationEndPoint(); + _communicationEndpoint = communicationEndPoint ?? TestServiceLocator.Get(connectionInfo.Endpoint) ?? SetCommunicationEndPoint(); _connectionInfo.Endpoint = connectionInfo.Endpoint; _connectionInfo.Role = connectionInfo.Role == ConnectionRole.Host ? ConnectionRole.Client @@ -170,6 +170,7 @@ public int InitializeCommunication() // Server start returns the listener port // return int.Parse(this.communicationServer.Start()); var endpoint = _communicationEndpoint.Start(_connectionInfo.Endpoint); + // TODO: This is forcing us to use ip and port for communication return endpoint.GetIpEndPoint().Port; } diff --git a/src/Microsoft.TestPlatform.ObjectModel/TestServiceLocator.cs b/src/Microsoft.TestPlatform.ObjectModel/TestServiceLocator.cs index 693c5b2cee..9680246f89 100644 --- a/src/Microsoft.TestPlatform.ObjectModel/TestServiceLocator.cs +++ b/src/Microsoft.TestPlatform.ObjectModel/TestServiceLocator.cs @@ -9,21 +9,21 @@ namespace Microsoft.VisualStudio.TestPlatform.ObjectModel; // TODO: Make this internal, I am just trying to have easier time trying this out. public static class TestServiceLocator { - public static Dictionary Instances { get; } = new Dictionary(); + public static Dictionary Instances { get; } = new Dictionary(); public static List Resolves { get; } = new(); - public static void Register(TRegistration instance) + public static void Register(string name, TRegistration instance) { - Instances.Add(typeof(TRegistration), instance); + Instances.Add(name, instance); } - public static TRegistration Get() + public static TRegistration Get(string name) { - if (!Instances.TryGetValue(typeof(TRegistration), out var instance)) - throw new InvalidOperationException($"Cannot find instance for type {typeof(TRegistration)}."); + if (!Instances.TryGetValue(name, out var instance)) + throw new InvalidOperationException($"Cannot find an instance for name {name}."); #if !NETSTANDARD1_0 - Resolves.Add(new Resolve(typeof(TRegistration).FullName, Environment.StackTrace)); + Resolves.Add(new Resolve(name, typeof(TRegistration).FullName, Environment.StackTrace)); #endif return (TRegistration)instance; } @@ -38,12 +38,14 @@ public static void Clear() // TODO: Make this internal, I am just trying to have easier time trying this out. public class Resolve { - public Resolve(string type, string stackTrace) + public Resolve(string name, string type, string stackTrace) { + Name = name; Type = type; StackTrace = stackTrace; } + public string Name { get; } public string Type { get; } public string StackTrace { get; } } diff --git a/test/vstest.ProgrammerTests/FakeTestDllBuilder.cs b/test/vstest.ProgrammerTests/FakeTestDllBuilder.cs new file mode 100644 index 0000000000..dc035f9950 --- /dev/null +++ b/test/vstest.ProgrammerTests/FakeTestDllBuilder.cs @@ -0,0 +1,73 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +namespace vstest.ProgrammerTests.CommandLine; + +using System; +using System.Runtime.Versioning; + +using Microsoft.VisualStudio.TestPlatform.ObjectModel; + +internal class FakeTestDllBuilder +{ + private string _path = @$"X:\fake\mstest_{Guid.NewGuid()}.dll"; + private FrameworkName _framework = KnownFramework.Net50; + private Architecture _architecture = Architecture.X64; + private List>? _testBatches; + + internal FakeTestDllBuilder WithFramework(FrameworkName framework) + { + _framework = framework; + return this; + } + + internal FakeTestDllBuilder WithPath(string path) + { + _path = path; + return this; + } + + internal FakeTestDllBuilder WithArchitecture(Architecture architecture) + { + _architecture = architecture; + return this; + } + + /// + /// Use this together with TestBatchBuilder, or use WithTestCount to get basic test batch. + /// + /// + /// + internal FakeTestDllBuilder WithTestBatches(List> testBatches) + { + _testBatches = testBatches; + return this; + } + + /// + /// Use this to get basic test batch, or use WithTestBatches together with TestBatchBuilder, to get a custom batch. + /// + /// + /// + internal FakeTestDllBuilder WithTestCount(int totalCount, int? batchSize = null) + { + _testBatches = new FakeTestBatchBuilder() + .WithTotalCount(totalCount) + .WithBatchSize(batchSize ?? totalCount) + .Build(); + + return this; + } + + internal FakeTestDllFile Build() + { + if (_testBatches == null) + { + _testBatches = new FakeTestBatchBuilder() + .WithTotalCount(10) + .WithBatchSize(5) + .Build(); + } + return new FakeTestDllFile(_path, _framework, _architecture, _testBatches); + } +} diff --git a/test/vstest.ProgrammerTests/FakeTestHostBuilder.cs b/test/vstest.ProgrammerTests/FakeTestHostBuilder.cs new file mode 100644 index 0000000000..ba4851e34e --- /dev/null +++ b/test/vstest.ProgrammerTests/FakeTestHostBuilder.cs @@ -0,0 +1,22 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +namespace vstest.ProgrammerTests.CommandLine; + +using System; + +internal class FakeTestHostBuilder +{ + // TODO: this would correctly be any test holding container, but let's not get ahead of myself. + private List _dlls = new(); + + public FakeTestHostBuilder() + { + } + + internal FakeTestHostBuilder WithTestDll(FakeTestDllFile dll) + { + _dlls.Add(dll); + return this; + } +} diff --git a/test/vstest.ProgrammerTests/Fakes/FakeAssemblyMetadataProvider.cs b/test/vstest.ProgrammerTests/Fakes/FakeAssemblyMetadataProvider.cs index bb7c4755f0..02f6f57bff 100644 --- a/test/vstest.ProgrammerTests/Fakes/FakeAssemblyMetadataProvider.cs +++ b/test/vstest.ProgrammerTests/Fakes/FakeAssemblyMetadataProvider.cs @@ -1,14 +1,13 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT license. See LICENSE file in the project root for full license information. +namespace vstest.ProgrammerTests.CommandLine.Fakes; + using System.Runtime.Versioning; using Microsoft.VisualStudio.TestPlatform.CommandLine.Processors; using Microsoft.VisualStudio.TestPlatform.ObjectModel; -#pragma warning disable IDE1006 // Naming Styles -namespace vstest.ProgrammerTests.CommandLine.Fakes; - internal class FakeAssemblyMetadataProvider : IAssemblyMetadataProvider { public FakeFileHelper FakeFileHelper { get; } diff --git a/test/vstest.ProgrammerTests/Fakes/FakeCommunicationChannel.cs b/test/vstest.ProgrammerTests/Fakes/FakeCommunicationChannel.cs index cd00497896..0c389c25bf 100644 --- a/test/vstest.ProgrammerTests/Fakes/FakeCommunicationChannel.cs +++ b/test/vstest.ProgrammerTests/Fakes/FakeCommunicationChannel.cs @@ -1,20 +1,18 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT license. See LICENSE file in the project root for full license information. +namespace vstest.ProgrammerTests.Fakes; + using System.Collections.Concurrent; using System.Diagnostics; using Microsoft.VisualStudio.TestPlatform.CommunicationUtilities; using Microsoft.VisualStudio.TestPlatform.CommunicationUtilities.Interfaces; using Microsoft.VisualStudio.TestPlatform.CommunicationUtilities.ObjectModel; -using Microsoft.VisualStudio.TestPlatform.ObjectModel; -using Microsoft.VisualStudio.TestPlatform.ObjectModel.Client; using Microsoft.VisualStudio.TestPlatform.ObjectModel.Logging; using vstest.ProgrammerTests.CommandLine; -namespace vstest.ProgrammerTests.Fakes; - internal class FakeCommunicationChannel : ICommunicationChannel { // The naming here is a bit confusing when this is implemented in process. @@ -46,15 +44,16 @@ internal class FakeCommunicationChannel : ICommunicationChannel public Queue> NextResponses { get; } = new(); public FakeErrorAggregator FakeErrorAggregator { get; } public FakeMessage? OutgoingMessage { get; private set; } + public int Id { get; } public CancellationTokenSource CancellationTokenSource = new(); public event EventHandler? MessageReceived; - public FakeCommunicationChannel(List> responses, FakeErrorAggregator fakeErrorAggregator) + public FakeCommunicationChannel(List> responses, FakeErrorAggregator fakeErrorAggregator, int id) { FakeErrorAggregator = fakeErrorAggregator; - + Id = id; responses.ForEach(NextResponses.Enqueue); ProcessIncomingMessages = Task.Run(() => diff --git a/test/vstest.ProgrammerTests/Fakes/FakeCommunicationEndpoint.cs b/test/vstest.ProgrammerTests/Fakes/FakeCommunicationEndpoint.cs index 71d227e32f..7e472f7d17 100644 --- a/test/vstest.ProgrammerTests/Fakes/FakeCommunicationEndpoint.cs +++ b/test/vstest.ProgrammerTests/Fakes/FakeCommunicationEndpoint.cs @@ -1,13 +1,14 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT license. See LICENSE file in the project root for full license information. +namespace vstest.ProgrammerTests.CommandLine; + using Microsoft.VisualStudio.TestPlatform.CommunicationUtilities.Interfaces; +using Microsoft.VisualStudio.TestPlatform.ObjectModel; using Microsoft.VisualStudio.TestPlatform.Utilities; using vstest.ProgrammerTests.Fakes; -namespace vstest.ProgrammerTests.CommandLine; - internal class FakeCommunicationEndpoint : ICommunicationEndPoint { private bool _stopped; @@ -16,16 +17,40 @@ public FakeCommunicationEndpoint(FakeCommunicationChannel fakeCommunicationChann { Channel = fakeCommunicationChannel; FakeErrorAggregator = fakeErrorAggregator; + TestHostConnectionInfo = new TestHostConnectionInfo + { + Endpoint = $"127.0.0.1:{fakeCommunicationChannel.Id}", + Role = ConnectionRole.Client, + Transport = Transport.Sockets, + }; } public FakeErrorAggregator FakeErrorAggregator { get; } - public FakeCommunicationChannel Channel { get; private set; } + public FakeCommunicationChannel Channel { get; } + public TestHostConnectionInfo TestHostConnectionInfo { get; } public event EventHandler? Connected; public event EventHandler? Disconnected; public string Start(string endPoint) { + // In normal run this endpoint can be a client or a server. When we are a client we will get an address and a port and + // we will try to connect to it. + // If we are a server, we will get an address and port 0, which means we should figure out a port that is free + // and return the address and port back to the caller. + // + // In our fake scenario we know the "port" from the get go, we set it to an id that was given to the testhost + // because that is currently the only way for us to check if we are connecting to the expected fake testhost + // that has a list of canned responses, which must correlate with the requests. So e.g. if we get request for mstest1.dll + // we should return the responses we have prepared for mstest1.dll, and not for mstest2.dll. + // + // We use the port number because the rest of the IP address is validated. We force sockets and IP usage in multiple places, + // so we cannot just past the dll path (or something similar) as the endpoint name, because the other side will check if that is + // a valid IP address and port. + if (endPoint != TestHostConnectionInfo.Endpoint) + { + throw new InvalidOperationException($"Expected to connect to {endPoint} but instead got channel with {TestHostConnectionInfo.Endpoint}."); + } Connected?.SafeInvoke(this, new ConnectedEventArgs(Channel), "FakeCommunicationEndpoint.Start"); return endPoint; } diff --git a/test/vstest.ProgrammerTests/Fakes/FakeProcess.cs b/test/vstest.ProgrammerTests/Fakes/FakeProcess.cs index 1c61a938a9..3d25ead5dc 100644 --- a/test/vstest.ProgrammerTests/Fakes/FakeProcess.cs +++ b/test/vstest.ProgrammerTests/Fakes/FakeProcess.cs @@ -23,8 +23,9 @@ internal class FakeProcess public int ExitCode { get; init; } = -1; public bool Exited { get; private set; } - public FakeProcess(string path, string arguments, string workingDirectory, IDictionary environmentVariables, Action errorCallback, Action exitCallBack, Action outputCallback, FakeErrorAggregator fakeErrorAggregator) + public FakeProcess(FakeErrorAggregator fakeErrorAggregator, string path, string arguments, string workingDirectory, IDictionary environmentVariables, Action errorCallback, Action exitCallBack, Action outputCallback) { + FakeErrorAggregator = fakeErrorAggregator; Path = path; Name = System.IO.Path.GetFileName(path); Arguments = arguments; @@ -33,7 +34,6 @@ public FakeProcess(string path, string arguments, string workingDirectory, IDict ErrorCallback = errorCallback; ExitCallback = exitCallBack; OutputCallback = outputCallback; - FakeErrorAggregator = fakeErrorAggregator; } internal static FakeProcess EnsureFakeProcess(object process) diff --git a/test/vstest.ProgrammerTests/Fakes/FakeProcessHelper.cs b/test/vstest.ProgrammerTests/Fakes/FakeProcessHelper.cs index bad95982c2..11b049a484 100644 --- a/test/vstest.ProgrammerTests/Fakes/FakeProcessHelper.cs +++ b/test/vstest.ProgrammerTests/Fakes/FakeProcessHelper.cs @@ -20,11 +20,11 @@ internal class FakeProcessHelper : IProcessHelper public FakeErrorAggregator FakeErrorAggregator { get; } - public FakeProcessHelper(FakeProcess currentProcess, FakeErrorAggregator fakeErrorAggregator) + public FakeProcessHelper(FakeErrorAggregator fakeErrorAggregator, FakeProcess currentProcess) { + FakeErrorAggregator = fakeErrorAggregator; CurrentProcess = currentProcess; AddProcess(currentProcess); - FakeErrorAggregator = fakeErrorAggregator; } private void AddProcess(FakeProcess currentProcess) @@ -84,7 +84,7 @@ public string GetTestEngineDirectory() public object LaunchProcess(string processPath, string arguments, string workingDirectory, IDictionary environmentVariables, Action errorCallback, Action exitCallBack, Action outputCallback) { - var process = new FakeProcess(processPath, arguments, workingDirectory, environmentVariables, errorCallback, exitCallBack, outputCallback, FakeErrorAggregator); + var process = new FakeProcess(FakeErrorAggregator, processPath, arguments, workingDirectory, environmentVariables, errorCallback, exitCallBack, outputCallback); Processes.Add(process); return process; diff --git a/test/vstest.ProgrammerTests/Fakes/FakeTestRuntimeProvider.cs b/test/vstest.ProgrammerTests/Fakes/FakeTestRuntimeProvider.cs index d3b89e6c2f..d470b15fef 100644 --- a/test/vstest.ProgrammerTests/Fakes/FakeTestRuntimeProvider.cs +++ b/test/vstest.ProgrammerTests/Fakes/FakeTestRuntimeProvider.cs @@ -1,14 +1,13 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT license. See LICENSE file in the project root for full license information. +namespace vstest.ProgrammerTests.CommandLine.Fakes; + using Microsoft.VisualStudio.TestPlatform.ObjectModel; using Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.Interfaces; using Microsoft.VisualStudio.TestPlatform.ObjectModel.Host; using Microsoft.VisualStudio.TestPlatform.ObjectModel.Logging; -#pragma warning disable IDE1006 // Naming Styles -namespace vstest.ProgrammerTests.CommandLine.Fakes; - internal class FakeTestRuntimeProvider : ITestRuntimeProvider { public FakeProcessHelper FakeProcessHelper { get; } @@ -21,7 +20,7 @@ internal class FakeTestRuntimeProvider : ITestRuntimeProvider public event EventHandler HostLaunched; public event EventHandler HostExited; - public FakeTestRuntimeProvider(FakeProcessHelper fakeProcessHelper, FakeCommunicationEndpoint fakeCommunicationEndpoint) + public FakeTestRuntimeProvider(FakeProcessHelper fakeProcessHelper, FakeCommunicationEndpoint fakeCommunicationEndpoint, FakeErrorAggregator fakeErrorAggregator) { FakeProcessHelper = fakeProcessHelper; FakeCommunicationEndpoint = fakeCommunicationEndpoint; @@ -42,14 +41,7 @@ public Task CleanTestHostAsync(CancellationToken cancellationToken) public TestHostConnectionInfo GetTestHostConnectionInfo() { - // TODO: Makes this configurable? - return new TestHostConnectionInfo - { - // using 9090 for no particular reason, apart from knowing that port 0 is ignored somewhere in our codebase - Endpoint = "127.0.0.1:9090", - Role = ConnectionRole.Client, - Transport = Transport.Sockets, - }; + return FakeCommunicationEndpoint.TestHostConnectionInfo; } public TestProcessStartInfo GetTestHostProcessStartInfo(IEnumerable sources, IDictionary environmentVariables, TestRunnerConnectionInfo connectionInfo) diff --git a/test/vstest.ProgrammerTests/Fakes/FakeTestRuntimeProviderManager.cs b/test/vstest.ProgrammerTests/Fakes/FakeTestRuntimeProviderManager.cs index 77fe1ec717..ba7873fd10 100644 --- a/test/vstest.ProgrammerTests/Fakes/FakeTestRuntimeProviderManager.cs +++ b/test/vstest.ProgrammerTests/Fakes/FakeTestRuntimeProviderManager.cs @@ -1,26 +1,36 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT license. See LICENSE file in the project root for full license information. -using Microsoft.VisualStudio.TestPlatform.Common.Hosting; -using Microsoft.VisualStudio.TestPlatform.ObjectModel.Host; - #pragma warning disable IDE1006 // Naming Styles namespace vstest.ProgrammerTests.CommandLine.Fakes; +using System.Collections.Concurrent; + +using Microsoft.VisualStudio.TestPlatform.Common.Hosting; +using Microsoft.VisualStudio.TestPlatform.ObjectModel.Host; + internal class FakeTestRuntimeProviderManager : ITestRuntimeProviderManager { - public FakeTestRuntimeProviderManager(FakeProcessHelper fakeProcessHelper, FakeCommunicationEndpoint fakeCommunicationEndpoint, FakeErrorAggregator fakeErrorAggregator) + public FakeTestRuntimeProviderManager(List testRuntimeProviders, FakeErrorAggregator fakeErrorAggregator) { - TestRuntimeProvider = new FakeTestRuntimeProvider(fakeProcessHelper, fakeCommunicationEndpoint); + testRuntimeProviders.ForEach(TestRuntimeProviders.Enqueue); FakeErrorAggregator = fakeErrorAggregator; } - public FakeTestRuntimeProvider TestRuntimeProvider { get; private set; } + public ConcurrentQueue TestRuntimeProviders { get; } = new(); + public List ProvidedTestRuntimeProviders { get; } = new(); + public FakeErrorAggregator FakeErrorAggregator { get; } public ITestRuntimeProvider GetTestHostManagerByRunConfiguration(string runConfiguration) { - return TestRuntimeProvider; + if (!TestRuntimeProviders.TryDequeue(out var next)) + { + throw new InvalidOperationException("There are no more TestRuntimeProviders to be provided"); + } + + ProvidedTestRuntimeProviders.Add(next); + return next; } public ITestRuntimeProvider GetTestHostManagerByUri(string hostUri) diff --git a/test/vstest.ProgrammerTests/Fixture.cs b/test/vstest.ProgrammerTests/Fixture.cs new file mode 100644 index 0000000000..169aa8358e --- /dev/null +++ b/test/vstest.ProgrammerTests/Fixture.cs @@ -0,0 +1,26 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +namespace vstest.ProgrammerTests.CommandLine; + +using vstest.ProgrammerTests.CommandLine.Fakes; + +internal class Fixture : IDisposable +{ + public FakeErrorAggregator ErrorAggregator { get; } = new(); + public FakeProcessHelper ProcessHelper { get; } + public FakeProcess CurrentProcess { get; } + public FakeFileHelper FileHelper { get; } + + public Fixture() + { + CurrentProcess = new FakeProcess(ErrorAggregator, @"X:\fake\vstest.console.exe", string.Empty, null, null, null, null, null); + ProcessHelper = new FakeProcessHelper(ErrorAggregator, CurrentProcess); + FileHelper = new FakeFileHelper(ErrorAggregator); + + } + public void Dispose() + { + + } +} diff --git a/test/vstest.ProgrammerTests/KnownFramework.cs b/test/vstest.ProgrammerTests/KnownFramework.cs new file mode 100644 index 0000000000..75d044e7bf --- /dev/null +++ b/test/vstest.ProgrammerTests/KnownFramework.cs @@ -0,0 +1,12 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +namespace vstest.ProgrammerTests.CommandLine; +using System.Runtime.Versioning; + +internal static class KnownFramework +{ + public static FrameworkName NetCore(int major, int minor = 0) => new($".NETCoreApp,Version={major}.{minor}"); + + public static FrameworkName Net50 = NetCore(5); +} diff --git a/test/vstest.ProgrammerTests/UnitTest1.cs b/test/vstest.ProgrammerTests/UnitTest1.cs index 64b76bf3f2..019ba336d7 100644 --- a/test/vstest.ProgrammerTests/UnitTest1.cs +++ b/test/vstest.ProgrammerTests/UnitTest1.cs @@ -52,8 +52,8 @@ public async Task GivenAnMSTestAssemblyWith108Tests_WhenTestsAreRun_Then108Tests var fakeErrorAggregator = new FakeErrorAggregator(); var commandLineOptions = CommandLineOptions.Instance; - var fakeCurrentProcess = new FakeProcess(@"X:\fake\vstest.console.exe", string.Empty, null, null, null, null, null, fakeErrorAggregator); - var fakeProcessHelper = new FakeProcessHelper(fakeCurrentProcess, fakeErrorAggregator); + var fakeCurrentProcess = new FakeProcess(fakeErrorAggregator, @"X:\fake\vstest.console.exe", string.Empty, null, null, null, null, null); + var fakeProcessHelper = new FakeProcessHelper(fakeErrorAggregator, fakeCurrentProcess); var fakeFileHelper = new FakeFileHelper(fakeErrorAggregator); // TODO: Get framework name from constants @@ -92,10 +92,11 @@ public async Task GivenAnMSTestAssemblyWith108Tests_WhenTestsAreRun_Then108Tests }; - var fakeCommunicationEndpoint = new FakeCommunicationEndpoint(new FakeCommunicationChannel(responses, fakeErrorAggregator), fakeErrorAggregator); + var fakeCommunicationEndpoint = new FakeCommunicationEndpoint(new FakeCommunicationChannel(responses, fakeErrorAggregator, 1), fakeErrorAggregator); TestServiceLocator.Clear(); - TestServiceLocator.Register(fakeCommunicationEndpoint); - var fakeTestRuntimeProviderManager = new FakeTestRuntimeProviderManager(fakeProcessHelper, fakeCommunicationEndpoint, fakeErrorAggregator); + TestServiceLocator.Register(fakeCommunicationEndpoint.TestHostConnectionInfo.Endpoint, fakeCommunicationEndpoint); + var fakeTestRuntimeProvider = new FakeTestRuntimeProvider(fakeProcessHelper, fakeCommunicationEndpoint, fakeErrorAggregator); + var fakeTestRuntimeProviderManager = new FakeTestRuntimeProviderManager(new[] { fakeTestRuntimeProvider, fakeTestRuntimeProvider }.ToList(), fakeErrorAggregator); var testEngine = new TestEngine(fakeTestRuntimeProviderManager, fakeProcessHelper); var testPlatform = new TestPlatform(testEngine, fakeFileHelper, fakeTestRuntimeProviderManager); @@ -150,7 +151,7 @@ public async Task GivenAnMSTestAssemblyWith108Tests_WhenTestsAreRun_Then108Tests var cancelAbort = new CancellationTokenSource(); var task = Task.Run(async () => { - await Task.Delay(TimeSpan.FromSeconds(10), cancelAbort.Token); + await Task.Delay(TimeSpan.FromSeconds(Debugger.IsAttached ? 100 : 10), cancelAbort.Token); if (Debugger.IsAttached) { // we will abort because we are hanging, look at stacks to see what the problem is @@ -175,23 +176,24 @@ public async Task GivenAnMSTestAssemblyWith108Tests_WhenTestsAreRun_Then108Tests public async Task GivenMultipleMsTestAssembliesThatUseTheSameTargetFrameworkAndArchitecture_WhenTestsAreRun_ThenAllTestsFromAllAssembliesAreRun() { + using var fixture = new FixtureBuilder() + .Build(); + // -- arrange - var fakeErrorAggregator = new FakeErrorAggregator(); + var testhost1 = new FakeTestHostBuilder() + .WithTestDll() + .Build(); + var commandLineOptions = CommandLineOptions.Instance; - - var fakeCurrentProcess = new FakeProcess(@"X:\fake\vstest.console.exe", string.Empty, null, null, null, null, null, fakeErrorAggregator); - var fakeProcessHelper = new FakeProcessHelper(fakeCurrentProcess, fakeErrorAggregator); - - var fakeFileHelper = new FakeFileHelper(fakeErrorAggregator); - // TODO: Get framework name from constants // TODO: have mstest1dll canned - var tests = new FakeTestBatchBuilder() - .WithTotalCount(108) - .WithDuration(100.Milliseconds()) - .WithBatchSize(10) + var mstest1Dll = new FakeTestDllBuilder() + .WithPath(@"X:\fake\mstest1.dll") + .WithFramework(KnownFramework.Net50) + .WithArchitecture(Architecture.X64) + .WithTestCount(108, 10) .Build(); - var mstest1Dll = new FakeTestDllFile(@"X:\fake\mstest1.dll", new FrameworkName(".NETCoreApp,Version=v5.0"), Architecture.X64, tests); - fakeFileHelper.AddFile(mstest1Dll); + var tests = mstest1Dll.TestResultBatches; + fixture.FileHelper.AddFile(mstest1Dll); var tests2 = new FakeTestBatchBuilder() .WithTotalCount(108) @@ -199,8 +201,9 @@ public async Task GivenMultipleMsTestAssembliesThatUseTheSameTargetFrameworkAndA .WithBatchSize(10) .Build(); var mstest2Dll = new FakeTestDllFile(@"X:\fake\mstest2.dll", new FrameworkName(".NETCoreApp,Version=v5.0"), Architecture.X64, tests2); - fakeFileHelper.AddFile(mstest2Dll); + fixture.FileHelper.AddFile(mstest2Dll); + // --- List changeMessages = tests.Take(tests.Count - 1).Select(batch => // TODO: make the stats agree with the tests below new FakeMessage(MessageType.TestRunStatsChange, new TestRunChangedEventArgs(new TestRunStatistics(new Dictionary { [TestOutcome.Passed] = batch.Count }), batch, new List()) @@ -219,34 +222,66 @@ public async Task GivenMultipleMsTestAssembliesThatUseTheSameTargetFrameworkAndA new RequestResponsePair(MessageType.SessionEnd, message => { // TODO: how do we associate this to the correct process? - var fp = fakeProcessHelper.Processes.Last(); - fakeProcessHelper.TerminateProcess(fp); + var fp = fixture.ProcessHelper.Processes.Last(); + fixture.ProcessHelper.TerminateProcess(fp); + + return new List { FakeMessage.NoResponse }; + }), + }; + + var fakeCommunicationEndpoint = new FakeCommunicationEndpoint(new FakeCommunicationChannel(responses, fixture.ErrorAggregator, 1), fixture.ErrorAggregator); + var fakeTestRuntimeProvider = new FakeTestRuntimeProvider(fixture.ProcessHelper, fakeCommunicationEndpoint, fixture.ErrorAggregator); + + // --- + List changeMessages2 = tests2.Take(tests2.Count - 1).Select(batch => // TODO: make the stats agree with the tests below + new FakeMessage(MessageType.TestRunStatsChange, + new TestRunChangedEventArgs(new TestRunStatistics(new Dictionary { [TestOutcome.Passed] = batch.Count }), batch, new List()) + )).ToList(); + FakeMessage completedMessage2 = new FakeMessage(MessageType.ExecutionComplete, new TestRunCompletePayload + { + // TODO: make the stats agree with the tests below + TestRunCompleteArgs = new TestRunCompleteEventArgs(new TestRunStatistics(new Dictionary { [TestOutcome.Passed] = 1 }), false, false, null, new System.Collections.ObjectModel.Collection(), TimeSpan.Zero), + LastRunTests = new TestRunChangedEventArgs(new TestRunStatistics(new Dictionary { [TestOutcome.Passed] = 1 }), tests.Last(), new List()), + }); + List messages2 = changeMessages2.Concat(new[] { completedMessage2 }).ToList(); + var responses2 = new List> { + new RequestResponsePair(MessageType.VersionCheck, new FakeMessage(MessageType.VersionCheck, 5)), + new RequestResponsePair(MessageType.ExecutionInitialize, FakeMessage.NoResponse), + new RequestResponsePair(MessageType.StartTestExecutionWithSources, messages), + new RequestResponsePair(MessageType.SessionEnd, message => + { + // TODO: how do we associate this to the correct process? + var fp = fixture.ProcessHelper.Processes.Last(); + fixture.ProcessHelper.TerminateProcess(fp); return new List { FakeMessage.NoResponse }; }), }; - var fakeCommunicationEndpoint = new FakeCommunicationEndpoint(new FakeCommunicationChannel(responses, fakeErrorAggregator), fakeErrorAggregator); + var fakeCommunicationEndpoint2 = new FakeCommunicationEndpoint(new FakeCommunicationChannel(responses2, fixture.ErrorAggregator, 2), fixture.ErrorAggregator); + var fakeTestRuntimeProvider2 = new FakeTestRuntimeProvider(fixture.ProcessHelper, fakeCommunicationEndpoint2, fixture.ErrorAggregator); + TestServiceLocator.Clear(); - TestServiceLocator.Register(fakeCommunicationEndpoint); - var fakeTestRuntimeProviderManager = new FakeTestRuntimeProviderManager(fakeProcessHelper, fakeCommunicationEndpoint, fakeErrorAggregator); - var testEngine = new TestEngine(fakeTestRuntimeProviderManager, fakeProcessHelper); + TestServiceLocator.Register(fakeCommunicationEndpoint.TestHostConnectionInfo.Endpoint, fakeCommunicationEndpoint); + TestServiceLocator.Register(fakeCommunicationEndpoint2.TestHostConnectionInfo.Endpoint, fakeCommunicationEndpoint2); + var fakeTestRuntimeProviderManager = new FakeTestRuntimeProviderManager(new[] { fakeTestRuntimeProvider, fakeTestRuntimeProvider, fakeTestRuntimeProvider2, fakeTestRuntimeProvider2 }.ToList(), fixture.ErrorAggregator); + var testEngine = new TestEngine(fakeTestRuntimeProviderManager, fixture.ProcessHelper); - var testPlatform = new TestPlatform(testEngine, fakeFileHelper, fakeTestRuntimeProviderManager); + var testPlatform = new TestPlatform(testEngine, fixture.FileHelper, fakeTestRuntimeProviderManager); var testRunResultAggregator = new TestRunResultAggregator(); - var fakeTestPlatformEventSource = new FakeTestPlatformEventSource(fakeErrorAggregator); + var fakeTestPlatformEventSource = new FakeTestPlatformEventSource(fixture.ErrorAggregator); - var fakeAssemblyMetadataProvider = new FakeAssemblyMetadataProvider(fakeFileHelper, fakeErrorAggregator); + var fakeAssemblyMetadataProvider = new FakeAssemblyMetadataProvider(fixture.FileHelper, fixture.ErrorAggregator); var inferHelper = new InferHelper(fakeAssemblyMetadataProvider); // This is most likely not the correctl place where to cut this off, plugin cache is probably the better place, // but it is not injected, and I don't want to investigate this now. - var fakeDataCollectorAttachmentsProcessorsFactory = new FakeDataCollectorAttachmentsProcessorsFactory(fakeErrorAggregator); + var fakeDataCollectorAttachmentsProcessorsFactory = new FakeDataCollectorAttachmentsProcessorsFactory(fixture.ErrorAggregator); var testRunAttachmentsProcessingManager = new TestRunAttachmentsProcessingManager(fakeTestPlatformEventSource, fakeDataCollectorAttachmentsProcessorsFactory); - Task fakeMetricsPublisherTask = Task.FromResult(new FakeMetricsPublisher(fakeErrorAggregator)); + Task fakeMetricsPublisherTask = Task.FromResult(new FakeMetricsPublisher(fixture.ErrorAggregator)); TestRequestManager testRequestManager = new( commandLineOptions, testPlatform, @@ -254,7 +289,7 @@ public async Task GivenMultipleMsTestAssembliesThatUseTheSameTargetFrameworkAndA fakeTestPlatformEventSource, inferHelper, fakeMetricsPublisherTask, - fakeProcessHelper, + fixture.ProcessHelper, testRunAttachmentsProcessingManager ); @@ -276,7 +311,7 @@ public async Task GivenMultipleMsTestAssembliesThatUseTheSameTargetFrameworkAndA }; // var fakeTestHostLauncher = new FakeTestHostLauncher(); - var fakeTestRunEventsRegistrar = new FakeTestRunEventsRegistrar(fakeErrorAggregator); + var fakeTestRunEventsRegistrar = new FakeTestRunEventsRegistrar(fixture.ErrorAggregator); var protocolConfig = new ProtocolConfig(); // TODO: we make sure the test is running 10 minutes at max and then we try to abort @@ -285,14 +320,14 @@ public async Task GivenMultipleMsTestAssembliesThatUseTheSameTargetFrameworkAndA var cancelAbort = new CancellationTokenSource(); var task = Task.Run(async () => { - await Task.Delay(TimeSpan.FromSeconds(5), cancelAbort.Token); + await Task.Delay(TimeSpan.FromSeconds(Debugger.IsAttached ? 50 : 5), cancelAbort.Token); if (Debugger.IsAttached) { - var errors = fakeErrorAggregator.Errors; + var errors = fixture.ErrorAggregator.Errors; // we will abort because we are hanging, look at stacks to see what the problem is Debugger.Break(); } - fakeErrorAggregator.Add(new Exception("errr we aborted")); + fixture.ErrorAggregator.Add(new Exception("errr we aborted")); testRequestManager.AbortTestRun(); }); @@ -305,32 +340,19 @@ public async Task GivenMultipleMsTestAssembliesThatUseTheSameTargetFrameworkAndA // pattern end // -- assert - fakeErrorAggregator.Errors.Should().BeEmpty(); - fakeTestRunEventsRegistrar.RunChangedEvents.SelectMany(er => er.Data.NewTestResults).Should().HaveCount(110); + fixture.ErrorAggregator.Errors.Should().BeEmpty(); + fakeTestRunEventsRegistrar.RunChangedEvents.SelectMany(er => er.Data.NewTestResults).Should().HaveCount(216); } } -internal class Fixture : IDisposable +internal class FixtureBuilder { - public Fixture() + internal Fixture Build() { - - } - - public List Processes { get; } = new(); - - public VstestConsoleBuilder VstestConsole { get; } = new(); - - public FakeTestExtensionManager TestExtensionManager { get; } = new(); - - public void Dispose() - { - + return new Fixture(); } } - - internal class CapturedRunSettings { public int MaxParallelLevel { get; internal set; } From aae9dcd70ff3af5bc82374db77313b8077f4e70e Mon Sep 17 00:00:00 2001 From: nohwnd Date: Wed, 16 Feb 2022 20:01:38 +0100 Subject: [PATCH 13/32] more cleanup --- .../FakeMessagesBuilder.cs | 128 ++++++++++++++++++ .../FakeMessagesBuilder/FromBatches.cs | 18 +++ .../FakeProcessBuilder.cs | 11 ++ test/vstest.ProgrammerTests/FakeTestHost.cs | 39 ++++++ .../FakeTestHostBuilder.cs | 55 +++++++- .../Fakes/FakeProcessHelper.cs | 8 +- .../Fakes/FakeTestDllFile.cs | 4 +- test/vstest.ProgrammerTests/Id.cs | 23 ++++ test/vstest.ProgrammerTests/UnitTest1.cs | 106 +++++---------- 9 files changed, 315 insertions(+), 77 deletions(-) create mode 100644 test/vstest.ProgrammerTests/FakeMessagesBuilder.cs create mode 100644 test/vstest.ProgrammerTests/FakeMessagesBuilder/FromBatches.cs create mode 100644 test/vstest.ProgrammerTests/FakeProcessBuilder.cs create mode 100644 test/vstest.ProgrammerTests/FakeTestHost.cs create mode 100644 test/vstest.ProgrammerTests/Id.cs diff --git a/test/vstest.ProgrammerTests/FakeMessagesBuilder.cs b/test/vstest.ProgrammerTests/FakeMessagesBuilder.cs new file mode 100644 index 0000000000..51692dd152 --- /dev/null +++ b/test/vstest.ProgrammerTests/FakeMessagesBuilder.cs @@ -0,0 +1,128 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +namespace vstest.ProgrammerTests.CommandLine; + +using Microsoft.VisualStudio.TestPlatform.CommunicationUtilities.ObjectModel; +using Microsoft.VisualStudio.TestPlatform.ObjectModel; +using Microsoft.VisualStudio.TestPlatform.ObjectModel.Client; + +using vstest.ProgrammerTests.Fakes; + +/// +/// Builds a list of RequestResponse pairs, with the provided values. Each method is a name of incoming message type. +/// The order in which the builder methods are called determines the order or responses. +/// +internal class FakeMessagesBuilder +{ + private readonly List> _responses = new(); + + /// + /// For VersionCheck message it responds with VersionCheck response that has the given version. + /// + /// + /// + internal FakeMessagesBuilder VersionCheck(int version) + { + AddPair(MessageType.VersionCheck, version); + return this; + } + + /// + /// For VersionCheck message it responds with the given FakeMessage. + /// + /// Message to respond with, or FakeMessage.NoResponse to not respond. + /// + internal FakeMessagesBuilder VersionCheck(FakeMessage message) + { + AddPair(MessageType.VersionCheck, message); + return this; + } + + /// + /// For VersionCheck message it does the given before action and responds with the given FakeMessage and then does the given after action. + /// Use FakeMessage.NoResponse to not respond. + /// + /// + /// + internal FakeMessagesBuilder VersionCheck(Action beforeAction, FakeMessage message, Action afterAction) + { + AddPair(MessageType.VersionCheck, message, beforeAction, afterAction); + return this; + } + + internal FakeMessagesBuilder ExecutionInitialize(FakeMessage message) + { + AddPair(MessageType.ExecutionInitialize, message); + return this; + } + + internal FakeMessagesBuilder StartTestExecutionWithSources(List> testResultBatches) + { + var tests = testResultBatches; + // this will create as many test stats changes messages, as there are batches -1 + // the last batch will be sent as test run complete event + + // TODO: make the stats agree with the tests below + List changeMessages = tests.Take(tests.Count - 1).Select(batch => + new FakeMessage(MessageType.TestRunStatsChange, + new TestRunChangedEventArgs(new TestRunStatistics(new Dictionary { [TestOutcome.Passed] = batch.Count }), batch, new List()) + )).ToList(); + FakeMessage completedMessage = new FakeMessage(MessageType.ExecutionComplete, new TestRunCompletePayload + { + // TODO: make the stats agree with the tests below + TestRunCompleteArgs = new TestRunCompleteEventArgs(new TestRunStatistics(new Dictionary { [TestOutcome.Passed] = 1 }), false, false, null, new System.Collections.ObjectModel.Collection(), TimeSpan.Zero), + LastRunTests = new TestRunChangedEventArgs(new TestRunStatistics(new Dictionary { [TestOutcome.Passed] = 1 }), tests.Last(), new List()), + }); + List messages = changeMessages.Concat(new[] { completedMessage }).ToList(); + + AddPairWithMultipleMessages(MessageType.StartTestExecutionWithSources, messages); + return this; + } + + + internal FakeMessagesBuilder SessionEnd(FakeMessage fakeMessage) + { + AddPair(MessageType.SessionEnd, fakeMessage); + return this; + } + + internal FakeMessagesBuilder SessionEnd(FakeMessage message, Action? beforeAction = null, Action? afterAction = null) + { + AddPair(MessageType.SessionEnd, message, beforeAction, afterAction); + return this; + } + + private void AddPair(string messageType, T value, Action? beforeAction = null, Action? afterAction = null) + { + // TODO: add actions + AddPair(messageType, new FakeMessage(messageType, value), beforeAction, afterAction); + } + + private void AddPair(string messageType, FakeMessage message, Action? beforeAction = null, Action? afterAction = null) + { + // TODO: add actions + AddPairWithMultipleMessages(messageType, new[] { message }, beforeAction, afterAction); + } + + // TODO: this uses different name, because it would never be chosen when we provide IEnumerable, the overload with T value is used instead. This is error prone, better design? + private void AddPairWithMultipleMessages(string messageType, IEnumerable messages, Action? beforeAction = null, Action? afterAction = null) + { + // TODO: add after actions + Func> callActionAndReturnMessages = m => + { + if (beforeAction != null) + { + beforeAction(m); + } + return messages.ToList(); + }; + + _responses.Add(new RequestResponsePair(messageType, callActionAndReturnMessages)); + } + + internal List> Build() + { + return _responses; + } +} diff --git a/test/vstest.ProgrammerTests/FakeMessagesBuilder/FromBatches.cs b/test/vstest.ProgrammerTests/FakeMessagesBuilder/FromBatches.cs new file mode 100644 index 0000000000..f7780d2efb --- /dev/null +++ b/test/vstest.ProgrammerTests/FakeMessagesBuilder/FromBatches.cs @@ -0,0 +1,18 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +namespace FakeMessagesBuilder; + +using System.Collections.Generic; + +using Microsoft.VisualStudio.TestPlatform.ObjectModel; + +internal class FromBatches +{ + private List> _testResultBatches; + + public FromBatches(List> testResultBatches) + { + _testResultBatches = testResultBatches; + } +} \ No newline at end of file diff --git a/test/vstest.ProgrammerTests/FakeProcessBuilder.cs b/test/vstest.ProgrammerTests/FakeProcessBuilder.cs new file mode 100644 index 0000000000..6bfd4efd3f --- /dev/null +++ b/test/vstest.ProgrammerTests/FakeProcessBuilder.cs @@ -0,0 +1,11 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +namespace vstest.ProgrammerTests.CommandLine; + +internal class FakeProcessBuilder +{ + public FakeProcessBuilder() + { + } +} \ No newline at end of file diff --git a/test/vstest.ProgrammerTests/FakeTestHost.cs b/test/vstest.ProgrammerTests/FakeTestHost.cs new file mode 100644 index 0000000000..b1962e9521 --- /dev/null +++ b/test/vstest.ProgrammerTests/FakeTestHost.cs @@ -0,0 +1,39 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +namespace vstest.ProgrammerTests.CommandLine; + +using vstest.ProgrammerTests.CommandLine.Fakes; +using vstest.ProgrammerTests.Fakes; + +internal class FakeTestHost +{ + private readonly Fixture _fixture; + + public int Id { get; } + public List Dlls { get; } + public FakeTestRuntimeProvider FakeTestRuntimeProvider { get; } + public FakeCommunicationEndpoint FakeCommunicationEndpoint { get; } + public FakeCommunicationChannel FakeCommunicationChannel { get; } + public List> Responses { get; } + public FakeProcess Process { get; internal set; } + + public FakeTestHost( + Fixture fixture, + int id, List dlls, + FakeTestRuntimeProvider fakeTestRuntimeProvider, + FakeCommunicationEndpoint fakeCommunicationEndpoint, + FakeCommunicationChannel fakeCommunicationChannel, + FakeProcess process, + List> responses) + { + _fixture = fixture; + Id = id; + Dlls = dlls; + FakeTestRuntimeProvider = fakeTestRuntimeProvider; + FakeCommunicationEndpoint = fakeCommunicationEndpoint; + FakeCommunicationChannel = fakeCommunicationChannel; + Process = process; + Responses = responses; + } +} diff --git a/test/vstest.ProgrammerTests/FakeTestHostBuilder.cs b/test/vstest.ProgrammerTests/FakeTestHostBuilder.cs index ba4851e34e..7877a3351f 100644 --- a/test/vstest.ProgrammerTests/FakeTestHostBuilder.cs +++ b/test/vstest.ProgrammerTests/FakeTestHostBuilder.cs @@ -5,13 +5,28 @@ namespace vstest.ProgrammerTests.CommandLine; using System; +using Microsoft.VisualStudio.TestPlatform.CommunicationUtilities.Interfaces; +using Microsoft.VisualStudio.TestPlatform.ObjectModel; + +using vstest.ProgrammerTests.CommandLine.Fakes; +using vstest.ProgrammerTests.Fakes; + internal class FakeTestHostBuilder { + // This will be also used as a port number, don't start from 0 + // it skips some paths in the real code, because port 0 has special meaning. + private static readonly Id Id = new(1000); + + private readonly Fixture _fixture; + // TODO: this would correctly be any test holding container, but let's not get ahead of myself. - private List _dlls = new(); + private readonly List _dlls = new(); + private FakeProcess? _process; + private List>? _responses; - public FakeTestHostBuilder() + public FakeTestHostBuilder(Fixture fixture) { + _fixture = fixture; } internal FakeTestHostBuilder WithTestDll(FakeTestDllFile dll) @@ -19,4 +34,40 @@ internal FakeTestHostBuilder WithTestDll(FakeTestDllFile dll) _dlls.Add(dll); return this; } + + internal FakeTestHost Build() + { + + if (_responses == null) + throw new InvalidOperationException("Add some reponses to the testhost by using WithResponses."); + + if (_process == null) + throw new InvalidOperationException("Add some process to the testhost by using WithProcess."); + + var id = Id.Next(); + var fakeCommunicationChannel = new FakeCommunicationChannel(_responses, _fixture.ErrorAggregator, id); + var fakeCommunicationEndpoint = new FakeCommunicationEndpoint(fakeCommunicationChannel, _fixture.ErrorAggregator); + var fakeTestRuntimeProvider = new FakeTestRuntimeProvider(_fixture.ProcessHelper, fakeCommunicationEndpoint, _fixture.ErrorAggregator); + + // This registers the endpoint so we can look it up later using the address, the Id from here is propagated to + // testhost connection info, and is used as port in 127.0.0.1:, address so we can lookup the correct channel. + TestServiceLocator.Register(fakeCommunicationEndpoint.TestHostConnectionInfo.Endpoint, fakeCommunicationEndpoint); + + _dlls.ForEach(_fixture.FileHelper.AddFile); + _fixture.ProcessHelper.AddFakeProcess(_process); + + return new FakeTestHost(_fixture, id, _dlls, fakeTestRuntimeProvider, fakeCommunicationEndpoint, fakeCommunicationChannel, _process, _responses); + } + + internal FakeTestHostBuilder WithProcess(FakeProcess process) + { + _process = process; + return this; + } + + internal FakeTestHostBuilder WithResponses(List> responses) + { + _responses = responses; + return this; + } } diff --git a/test/vstest.ProgrammerTests/Fakes/FakeProcessHelper.cs b/test/vstest.ProgrammerTests/Fakes/FakeProcessHelper.cs index 11b049a484..de6209c5ed 100644 --- a/test/vstest.ProgrammerTests/Fakes/FakeProcessHelper.cs +++ b/test/vstest.ProgrammerTests/Fakes/FakeProcessHelper.cs @@ -24,14 +24,14 @@ public FakeProcessHelper(FakeErrorAggregator fakeErrorAggregator, FakeProcess cu { FakeErrorAggregator = fakeErrorAggregator; CurrentProcess = currentProcess; - AddProcess(currentProcess); + AddFakeProcess(currentProcess); } - private void AddProcess(FakeProcess currentProcess) + public void AddFakeProcess(FakeProcess process) { var id = Interlocked.Increment(ref _lastProcessId); - currentProcess.SetId(id); - Processes.Add(currentProcess); + process.SetId(id); + Processes.Add(process); } public PlatformArchitecture GetCurrentProcessArchitecture() diff --git a/test/vstest.ProgrammerTests/Fakes/FakeTestDllFile.cs b/test/vstest.ProgrammerTests/Fakes/FakeTestDllFile.cs index 3c215861f9..b9ac4c7198 100644 --- a/test/vstest.ProgrammerTests/Fakes/FakeTestDllFile.cs +++ b/test/vstest.ProgrammerTests/Fakes/FakeTestDllFile.cs @@ -1,14 +1,14 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT license. See LICENSE file in the project root for full license information. +namespace vstest.ProgrammerTests.CommandLine; + using System.Runtime.Versioning; using Microsoft.VisualStudio.TestPlatform.ObjectModel; using vstest.ProgrammerTests.CommandLine.Fakes; -namespace vstest.ProgrammerTests.CommandLine; - internal class FakeTestDllFile : FakeFile { public FrameworkName FrameworkName { get; } diff --git a/test/vstest.ProgrammerTests/Id.cs b/test/vstest.ProgrammerTests/Id.cs new file mode 100644 index 0000000000..1fe4819422 --- /dev/null +++ b/test/vstest.ProgrammerTests/Id.cs @@ -0,0 +1,23 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +namespace vstest.ProgrammerTests.CommandLine; + +public class Id +{ + private int _id; + + public Id () : this(0) + { + } + + public Id (int firstId) + { + _id = firstId; + } + + public int Next() + { + return Interlocked.Increment(ref _id); + } +} diff --git a/test/vstest.ProgrammerTests/UnitTest1.cs b/test/vstest.ProgrammerTests/UnitTest1.cs index 019ba336d7..e29ba2fb50 100644 --- a/test/vstest.ProgrammerTests/UnitTest1.cs +++ b/test/vstest.ProgrammerTests/UnitTest1.cs @@ -176,14 +176,13 @@ public async Task GivenAnMSTestAssemblyWith108Tests_WhenTestsAreRun_Then108Tests public async Task GivenMultipleMsTestAssembliesThatUseTheSameTargetFrameworkAndArchitecture_WhenTestsAreRun_ThenAllTestsFromAllAssembliesAreRun() { + // We need to use static class to find the communication endpoint, this clears all the registrations of previous tests. + TestServiceLocator.Clear(); using var fixture = new FixtureBuilder() .Build(); // -- arrange - var testhost1 = new FakeTestHostBuilder() - .WithTestDll() - .Build(); - + var commandLineOptions = CommandLineOptions.Instance; // TODO: have mstest1dll canned var mstest1Dll = new FakeTestDllBuilder() @@ -192,79 +191,47 @@ public async Task GivenMultipleMsTestAssembliesThatUseTheSameTargetFrameworkAndA .WithArchitecture(Architecture.X64) .WithTestCount(108, 10) .Build(); - var tests = mstest1Dll.TestResultBatches; - fixture.FileHelper.AddFile(mstest1Dll); - var tests2 = new FakeTestBatchBuilder() - .WithTotalCount(108) - .WithDuration(100.Milliseconds()) - .WithBatchSize(10) - .Build(); - var mstest2Dll = new FakeTestDllFile(@"X:\fake\mstest2.dll", new FrameworkName(".NETCoreApp,Version=v5.0"), Architecture.X64, tests2); - fixture.FileHelper.AddFile(mstest2Dll); + var testhost1Process = new FakeProcess(fixture.ErrorAggregator, @"X:\fake\testhost1.exe", string.Empty, null, null, null, null, null); - // --- - List changeMessages = tests.Take(tests.Count - 1).Select(batch => // TODO: make the stats agree with the tests below - new FakeMessage(MessageType.TestRunStatsChange, - new TestRunChangedEventArgs(new TestRunStatistics(new Dictionary { [TestOutcome.Passed] = batch.Count }), batch, new List()) - )).ToList(); - FakeMessage completedMessage = new FakeMessage(MessageType.ExecutionComplete, new TestRunCompletePayload - { - // TODO: make the stats agree with the tests below - TestRunCompleteArgs = new TestRunCompleteEventArgs(new TestRunStatistics(new Dictionary { [TestOutcome.Passed] = 1 }), false, false, null, new System.Collections.ObjectModel.Collection(), TimeSpan.Zero), - LastRunTests = new TestRunChangedEventArgs(new TestRunStatistics(new Dictionary { [TestOutcome.Passed] = 1 }), tests.Last(), new List()), - }); - List messages = changeMessages.Concat(new[] { completedMessage }).ToList(); - var responses = new List> { - new RequestResponsePair(MessageType.VersionCheck, new FakeMessage(MessageType.VersionCheck, 5)), - new RequestResponsePair(MessageType.ExecutionInitialize, FakeMessage.NoResponse), - new RequestResponsePair(MessageType.StartTestExecutionWithSources, messages), - new RequestResponsePair(MessageType.SessionEnd, message => - { - // TODO: how do we associate this to the correct process? - var fp = fixture.ProcessHelper.Processes.Last(); - fixture.ProcessHelper.TerminateProcess(fp); + var runTests1 = new FakeMessagesBuilder() + .VersionCheck(5) + .ExecutionInitialize(FakeMessage.NoResponse) + .StartTestExecutionWithSources(mstest1Dll.TestResultBatches) + .SessionEnd(FakeMessage.NoResponse, _ => testhost1Process.Exit()) + .Build(); - return new List { FakeMessage.NoResponse }; - }), - }; + var testhost1 = new FakeTestHostBuilder(fixture) + .WithTestDll(mstest1Dll) + .WithProcess(testhost1Process) + .WithResponses(runTests1) + .Build(); - var fakeCommunicationEndpoint = new FakeCommunicationEndpoint(new FakeCommunicationChannel(responses, fixture.ErrorAggregator, 1), fixture.ErrorAggregator); - var fakeTestRuntimeProvider = new FakeTestRuntimeProvider(fixture.ProcessHelper, fakeCommunicationEndpoint, fixture.ErrorAggregator); + var mstest2Dll = new FakeTestDllBuilder() + .WithPath(@"X:\fake\mstest1.dll") + .WithFramework(KnownFramework.Net50) + .WithArchitecture(Architecture.X64) + .WithTestCount(50, 8) + .Build(); - // --- - List changeMessages2 = tests2.Take(tests2.Count - 1).Select(batch => // TODO: make the stats agree with the tests below - new FakeMessage(MessageType.TestRunStatsChange, - new TestRunChangedEventArgs(new TestRunStatistics(new Dictionary { [TestOutcome.Passed] = batch.Count }), batch, new List()) - )).ToList(); - FakeMessage completedMessage2 = new FakeMessage(MessageType.ExecutionComplete, new TestRunCompletePayload - { - // TODO: make the stats agree with the tests below - TestRunCompleteArgs = new TestRunCompleteEventArgs(new TestRunStatistics(new Dictionary { [TestOutcome.Passed] = 1 }), false, false, null, new System.Collections.ObjectModel.Collection(), TimeSpan.Zero), - LastRunTests = new TestRunChangedEventArgs(new TestRunStatistics(new Dictionary { [TestOutcome.Passed] = 1 }), tests.Last(), new List()), - }); - List messages2 = changeMessages2.Concat(new[] { completedMessage2 }).ToList(); - var responses2 = new List> { - new RequestResponsePair(MessageType.VersionCheck, new FakeMessage(MessageType.VersionCheck, 5)), - new RequestResponsePair(MessageType.ExecutionInitialize, FakeMessage.NoResponse), - new RequestResponsePair(MessageType.StartTestExecutionWithSources, messages), - new RequestResponsePair(MessageType.SessionEnd, message => - { - // TODO: how do we associate this to the correct process? - var fp = fixture.ProcessHelper.Processes.Last(); - fixture.ProcessHelper.TerminateProcess(fp); + var testhost2Process = new FakeProcess(fixture.ErrorAggregator, @"X:\fake\testhost1.exe", string.Empty, null, null, null, null, null); - return new List { FakeMessage.NoResponse }; - }), - }; + var runTests2 = new FakeMessagesBuilder() + .VersionCheck(5) + .ExecutionInitialize(FakeMessage.NoResponse) + .StartTestExecutionWithSources(mstest2Dll.TestResultBatches) + .SessionEnd(FakeMessage.NoResponse, _ => testhost1Process.Exit()) + .Build(); + var testhost2 = new FakeTestHostBuilder(fixture) + .WithTestDll(mstest2Dll) + .WithProcess(testhost2Process) + .WithResponses(runTests2) + .Build(); - var fakeCommunicationEndpoint2 = new FakeCommunicationEndpoint(new FakeCommunicationChannel(responses2, fixture.ErrorAggregator, 2), fixture.ErrorAggregator); - var fakeTestRuntimeProvider2 = new FakeTestRuntimeProvider(fixture.ProcessHelper, fakeCommunicationEndpoint2, fixture.ErrorAggregator); + // --- - TestServiceLocator.Clear(); - TestServiceLocator.Register(fakeCommunicationEndpoint.TestHostConnectionInfo.Endpoint, fakeCommunicationEndpoint); - TestServiceLocator.Register(fakeCommunicationEndpoint2.TestHostConnectionInfo.Endpoint, fakeCommunicationEndpoint2); + var fakeTestRuntimeProviderManager = new FakeTestRuntimeProviderManager(new[] { fakeTestRuntimeProvider, fakeTestRuntimeProvider, fakeTestRuntimeProvider2, fakeTestRuntimeProvider2 }.ToList(), fixture.ErrorAggregator); var testEngine = new TestEngine(fakeTestRuntimeProviderManager, fixture.ProcessHelper); @@ -341,10 +308,11 @@ public async Task GivenMultipleMsTestAssembliesThatUseTheSameTargetFrameworkAndA // -- assert fixture.ErrorAggregator.Errors.Should().BeEmpty(); - fakeTestRunEventsRegistrar.RunChangedEvents.SelectMany(er => er.Data.NewTestResults).Should().HaveCount(216); + fakeTestRunEventsRegistrar.RunChangedEvents.SelectMany(er => er.Data.NewTestResults).Should().HaveCount(158); } } + internal class FixtureBuilder { internal Fixture Build() From 1199580e1a1872cc88d0a50becc99c703c7d8b4b Mon Sep 17 00:00:00 2001 From: nohwnd Date: Thu, 17 Feb 2022 15:11:24 +0100 Subject: [PATCH 14/32] More cleanup --- ...Platform.TestHostProvider.UnitTests.csproj | 2 +- .../CapturedRunSettings.cs | 9 ++ .../ConfigurationEntry.cs | 23 +++ .../FakeMessagesBuilder.cs | 27 ++-- test/vstest.ProgrammerTests/FakeTestHost.cs | 4 +- .../FakeTestHostBuilder.cs | 17 +-- .../Fakes/FakeProcess.cs | 32 ++-- .../Fakes/FakeProcessHelper.cs | 11 +- .../Fakes/FakeTestDllFile.cs | 5 + .../Fakes/FakeTestRuntimeProvider.cs | 52 ++++--- .../Fakes/FakeTestRuntimeProviderManager.cs | 18 ++- test/vstest.ProgrammerTests/Fixture.cs | 102 ++++++++++++- .../RunConfiguration.cs | 9 ++ test/vstest.ProgrammerTests/TestDlls.cs | 9 ++ .../TestRequestManagerHelper.cs | 58 +++++++ test/vstest.ProgrammerTests/UnitTest1.cs | 142 +++--------------- 16 files changed, 332 insertions(+), 188 deletions(-) create mode 100644 test/vstest.ProgrammerTests/CapturedRunSettings.cs create mode 100644 test/vstest.ProgrammerTests/ConfigurationEntry.cs create mode 100644 test/vstest.ProgrammerTests/RunConfiguration.cs create mode 100644 test/vstest.ProgrammerTests/TestDlls.cs create mode 100644 test/vstest.ProgrammerTests/TestRequestManagerHelper.cs diff --git a/test/Microsoft.TestPlatform.TestHostProvider.UnitTests/Microsoft.TestPlatform.TestHostProvider.UnitTests.csproj b/test/Microsoft.TestPlatform.TestHostProvider.UnitTests/Microsoft.TestPlatform.TestHostProvider.UnitTests.csproj index e098a6461c..c799e07028 100644 --- a/test/Microsoft.TestPlatform.TestHostProvider.UnitTests/Microsoft.TestPlatform.TestHostProvider.UnitTests.csproj +++ b/test/Microsoft.TestPlatform.TestHostProvider.UnitTests/Microsoft.TestPlatform.TestHostProvider.UnitTests.csproj @@ -8,7 +8,7 @@ Microsoft.TestPlatform.TestHostProvider.UnitTests - netcoreapp2.1;net472 + netcoreapp3.1;net472 netcoreapp3.1 Exe diff --git a/test/vstest.ProgrammerTests/CapturedRunSettings.cs b/test/vstest.ProgrammerTests/CapturedRunSettings.cs new file mode 100644 index 0000000000..d69ce2370a --- /dev/null +++ b/test/vstest.ProgrammerTests/CapturedRunSettings.cs @@ -0,0 +1,9 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +namespace vstest.ProgrammerTests.CommandLine; + +internal class CapturedRunSettings +{ + public int MaxParallelLevel { get; internal set; } +} diff --git a/test/vstest.ProgrammerTests/ConfigurationEntry.cs b/test/vstest.ProgrammerTests/ConfigurationEntry.cs new file mode 100644 index 0000000000..9bcaab5e07 --- /dev/null +++ b/test/vstest.ProgrammerTests/ConfigurationEntry.cs @@ -0,0 +1,23 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +namespace vstest.ProgrammerTests.CommandLine; + +internal class ConfigurationEntry +{ + public ConfigurationEntry(string name) + { + Name = name; + } + + public string Name { get; } + + public string InlinePath => $"RunConfiguration.{Name}"; + + public string FullPath => $"RunSettings.{InlinePath}"; + + public override string ToString() + { + return Name; + } +} diff --git a/test/vstest.ProgrammerTests/FakeMessagesBuilder.cs b/test/vstest.ProgrammerTests/FakeMessagesBuilder.cs index 51692dd152..875433c327 100644 --- a/test/vstest.ProgrammerTests/FakeMessagesBuilder.cs +++ b/test/vstest.ProgrammerTests/FakeMessagesBuilder.cs @@ -24,7 +24,7 @@ internal class FakeMessagesBuilder /// internal FakeMessagesBuilder VersionCheck(int version) { - AddPair(MessageType.VersionCheck, version); + AddPairWithValue(MessageType.VersionCheck, version); return this; } @@ -35,7 +35,7 @@ internal FakeMessagesBuilder VersionCheck(int version) /// internal FakeMessagesBuilder VersionCheck(FakeMessage message) { - AddPair(MessageType.VersionCheck, message); + AddPairWithFakeMessage(MessageType.VersionCheck, message); return this; } @@ -47,13 +47,13 @@ internal FakeMessagesBuilder VersionCheck(FakeMessage message) /// internal FakeMessagesBuilder VersionCheck(Action beforeAction, FakeMessage message, Action afterAction) { - AddPair(MessageType.VersionCheck, message, beforeAction, afterAction); + AddPairWithFakeMessage(MessageType.VersionCheck, message, beforeAction, afterAction); return this; } internal FakeMessagesBuilder ExecutionInitialize(FakeMessage message) { - AddPair(MessageType.ExecutionInitialize, message); + AddPairWithFakeMessage(MessageType.ExecutionInitialize, message); return this; } @@ -76,37 +76,34 @@ internal FakeMessagesBuilder StartTestExecutionWithSources(List }); List messages = changeMessages.Concat(new[] { completedMessage }).ToList(); - AddPairWithMultipleMessages(MessageType.StartTestExecutionWithSources, messages); + AddPairWithMultipleFakeMessages(MessageType.StartTestExecutionWithSources, messages); return this; } internal FakeMessagesBuilder SessionEnd(FakeMessage fakeMessage) { - AddPair(MessageType.SessionEnd, fakeMessage); + AddPairWithFakeMessage(MessageType.SessionEnd, fakeMessage); return this; } internal FakeMessagesBuilder SessionEnd(FakeMessage message, Action? beforeAction = null, Action? afterAction = null) { - AddPair(MessageType.SessionEnd, message, beforeAction, afterAction); + AddPairWithFakeMessage(MessageType.SessionEnd, message, beforeAction, afterAction); return this; } - private void AddPair(string messageType, T value, Action? beforeAction = null, Action? afterAction = null) + private void AddPairWithValue(string messageType, T value, Action? beforeAction = null, Action? afterAction = null) { - // TODO: add actions - AddPair(messageType, new FakeMessage(messageType, value), beforeAction, afterAction); + AddPairWithFakeMessage(messageType, new FakeMessage(messageType, value), beforeAction, afterAction); } - private void AddPair(string messageType, FakeMessage message, Action? beforeAction = null, Action? afterAction = null) + private void AddPairWithFakeMessage(string messageType, FakeMessage message, Action? beforeAction = null, Action? afterAction = null) { - // TODO: add actions - AddPairWithMultipleMessages(messageType, new[] { message }, beforeAction, afterAction); + AddPairWithMultipleFakeMessages(messageType, new[] { message }, beforeAction, afterAction); } - // TODO: this uses different name, because it would never be chosen when we provide IEnumerable, the overload with T value is used instead. This is error prone, better design? - private void AddPairWithMultipleMessages(string messageType, IEnumerable messages, Action? beforeAction = null, Action? afterAction = null) + private void AddPairWithMultipleFakeMessages(string messageType, IEnumerable messages, Action? beforeAction = null, Action? afterAction = null) { // TODO: add after actions Func> callActionAndReturnMessages = m => diff --git a/test/vstest.ProgrammerTests/FakeTestHost.cs b/test/vstest.ProgrammerTests/FakeTestHost.cs index b1962e9521..e054d50b5e 100644 --- a/test/vstest.ProgrammerTests/FakeTestHost.cs +++ b/test/vstest.ProgrammerTests/FakeTestHost.cs @@ -6,7 +6,7 @@ namespace vstest.ProgrammerTests.CommandLine; using vstest.ProgrammerTests.CommandLine.Fakes; using vstest.ProgrammerTests.Fakes; -internal class FakeTestHost +internal class FakeTestFixtureHost { private readonly Fixture _fixture; @@ -18,7 +18,7 @@ internal class FakeTestHost public List> Responses { get; } public FakeProcess Process { get; internal set; } - public FakeTestHost( + public FakeTestFixtureHost( Fixture fixture, int id, List dlls, FakeTestRuntimeProvider fakeTestRuntimeProvider, diff --git a/test/vstest.ProgrammerTests/FakeTestHostBuilder.cs b/test/vstest.ProgrammerTests/FakeTestHostBuilder.cs index 7877a3351f..b290bf9b83 100644 --- a/test/vstest.ProgrammerTests/FakeTestHostBuilder.cs +++ b/test/vstest.ProgrammerTests/FakeTestHostBuilder.cs @@ -11,7 +11,7 @@ namespace vstest.ProgrammerTests.CommandLine; using vstest.ProgrammerTests.CommandLine.Fakes; using vstest.ProgrammerTests.Fakes; -internal class FakeTestHostBuilder +internal class FakeTestHostFixtureBuilder { // This will be also used as a port number, don't start from 0 // it skips some paths in the real code, because port 0 has special meaning. @@ -24,18 +24,18 @@ internal class FakeTestHostBuilder private FakeProcess? _process; private List>? _responses; - public FakeTestHostBuilder(Fixture fixture) + public FakeTestHostFixtureBuilder(Fixture fixture) { _fixture = fixture; } - internal FakeTestHostBuilder WithTestDll(FakeTestDllFile dll) + internal FakeTestHostFixtureBuilder WithTestDll(FakeTestDllFile dll) { _dlls.Add(dll); return this; } - internal FakeTestHost Build() + internal FakeTestFixtureHost Build() { if (_responses == null) @@ -47,25 +47,24 @@ internal FakeTestHost Build() var id = Id.Next(); var fakeCommunicationChannel = new FakeCommunicationChannel(_responses, _fixture.ErrorAggregator, id); var fakeCommunicationEndpoint = new FakeCommunicationEndpoint(fakeCommunicationChannel, _fixture.ErrorAggregator); - var fakeTestRuntimeProvider = new FakeTestRuntimeProvider(_fixture.ProcessHelper, fakeCommunicationEndpoint, _fixture.ErrorAggregator); + var fakeTestRuntimeProvider = new FakeTestRuntimeProvider(_fixture.ProcessHelper, _process, fakeCommunicationEndpoint, _fixture.ErrorAggregator); // This registers the endpoint so we can look it up later using the address, the Id from here is propagated to // testhost connection info, and is used as port in 127.0.0.1:, address so we can lookup the correct channel. TestServiceLocator.Register(fakeCommunicationEndpoint.TestHostConnectionInfo.Endpoint, fakeCommunicationEndpoint); _dlls.ForEach(_fixture.FileHelper.AddFile); - _fixture.ProcessHelper.AddFakeProcess(_process); - return new FakeTestHost(_fixture, id, _dlls, fakeTestRuntimeProvider, fakeCommunicationEndpoint, fakeCommunicationChannel, _process, _responses); + return new FakeTestFixtureHost(_fixture, id, _dlls, fakeTestRuntimeProvider, fakeCommunicationEndpoint, fakeCommunicationChannel, _process, _responses); } - internal FakeTestHostBuilder WithProcess(FakeProcess process) + internal FakeTestHostFixtureBuilder WithProcess(FakeProcess process) { _process = process; return this; } - internal FakeTestHostBuilder WithResponses(List> responses) + internal FakeTestHostFixtureBuilder WithResponses(List> responses) { _responses = responses; return this; diff --git a/test/vstest.ProgrammerTests/Fakes/FakeProcess.cs b/test/vstest.ProgrammerTests/Fakes/FakeProcess.cs index 3d25ead5dc..74fbd2b80b 100644 --- a/test/vstest.ProgrammerTests/Fakes/FakeProcess.cs +++ b/test/vstest.ProgrammerTests/Fakes/FakeProcess.cs @@ -1,39 +1,53 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT license. See LICENSE file in the project root for full license information. +namespace vstest.ProgrammerTests.CommandLine.Fakes; +using Microsoft.VisualStudio.TestPlatform.ObjectModel; using Microsoft.VisualStudio.TestPlatform.PlatformAbstractions; -namespace vstest.ProgrammerTests.CommandLine.Fakes; - internal class FakeProcess { public int Id { get; internal set; } public string Name { get; init; } public string Path { get; } - public string Arguments { get; set; } + public string? Arguments { get; set; } public string WorkingDirectory { get; } public IDictionary EnvironmentVariables { get; } - public Action ErrorCallback { get; } - public Action ExitCallback { get; } - public Action OutputCallback { get; } + // TODO: Throw if already set + public Action? ErrorCallback { get; set; } + // TODO: Throw if already set + public Action? ExitCallback { get; set; } + // TODO: Throw if already set + public Action? OutputCallback { get; set; } public PlatformArchitecture Architecture { get; init; } = PlatformArchitecture.X64; public FakeErrorAggregator FakeErrorAggregator { get; } public string? ErrorOutput { get; init; } public int ExitCode { get; init; } = -1; public bool Exited { get; private set; } + public TestProcessStartInfo TestProcessStartInfo { get; internal set; } - public FakeProcess(FakeErrorAggregator fakeErrorAggregator, string path, string arguments, string workingDirectory, IDictionary environmentVariables, Action errorCallback, Action exitCallBack, Action outputCallback) + public FakeProcess(FakeErrorAggregator fakeErrorAggregator, string path, string? arguments = null, string? workingDirectory = null, IDictionary? environmentVariables = null, Action? errorCallback = null, Action? exitCallBack = null, Action? outputCallback = null) { FakeErrorAggregator = fakeErrorAggregator; Path = path; Name = System.IO.Path.GetFileName(path); Arguments = arguments; - WorkingDirectory = workingDirectory; - EnvironmentVariables = environmentVariables; + WorkingDirectory = workingDirectory ?? System.IO.Path.GetDirectoryName(path) ?? throw new InvalidOperationException($"Path {path} does not have a parent directory."); + EnvironmentVariables = environmentVariables ?? new Dictionary(); ErrorCallback = errorCallback; ExitCallback = exitCallBack; OutputCallback = outputCallback; + + TestProcessStartInfo = new TestProcessStartInfo() + { + FileName = Path, + Arguments = Arguments, + WorkingDirectory = WorkingDirectory, + EnvironmentVariables = EnvironmentVariables, + // TODO: is this even used anywhere + CustomProperties = new Dictionary(), + }; } internal static FakeProcess EnsureFakeProcess(object process) diff --git a/test/vstest.ProgrammerTests/Fakes/FakeProcessHelper.cs b/test/vstest.ProgrammerTests/Fakes/FakeProcessHelper.cs index de6209c5ed..d583fe5092 100644 --- a/test/vstest.ProgrammerTests/Fakes/FakeProcessHelper.cs +++ b/test/vstest.ProgrammerTests/Fakes/FakeProcessHelper.cs @@ -1,12 +1,11 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT license. See LICENSE file in the project root for full license information. +namespace vstest.ProgrammerTests.CommandLine.Fakes; + using Microsoft.VisualStudio.TestPlatform.PlatformAbstractions; using Microsoft.VisualStudio.TestPlatform.PlatformAbstractions.Interfaces; -#pragma warning disable IDE1006 // Naming Styles -namespace vstest.ProgrammerTests.CommandLine.Fakes; - internal class FakeProcessHelper : IProcessHelper { // starting from 100 for no particular reason @@ -84,6 +83,7 @@ public string GetTestEngineDirectory() public object LaunchProcess(string processPath, string arguments, string workingDirectory, IDictionary environmentVariables, Action errorCallback, Action exitCallBack, Action outputCallback) { + // TODO: Throw if setting says we can't start new processes; var process = new FakeProcess(FakeErrorAggregator, processPath, arguments, workingDirectory, environmentVariables, errorCallback, exitCallBack, outputCallback); Processes.Add(process); @@ -111,4 +111,9 @@ public void WaitForProcessExit(object process) { // todo: implement for timeouts? } + + internal void StartFakeProcess(FakeProcess testHostProcess) + { + // TODO: mark the process as started. Do not add a new process if it did not exist. + } } diff --git a/test/vstest.ProgrammerTests/Fakes/FakeTestDllFile.cs b/test/vstest.ProgrammerTests/Fakes/FakeTestDllFile.cs index b9ac4c7198..16301853b3 100644 --- a/test/vstest.ProgrammerTests/Fakes/FakeTestDllFile.cs +++ b/test/vstest.ProgrammerTests/Fakes/FakeTestDllFile.cs @@ -14,11 +14,16 @@ internal class FakeTestDllFile : FakeFile public FrameworkName FrameworkName { get; } public Architecture Architecture { get; } public List> TestResultBatches { get; } + public int TestCount { get; } + public int BatchCount { get; } public FakeTestDllFile(string path, FrameworkName frameworkName, Architecture architecture, List> testResultBatches) : base(path) { FrameworkName = frameworkName; Architecture = architecture; TestResultBatches = testResultBatches; + + TestCount = testResultBatches.SelectMany(tr => tr).Count(); + BatchCount = testResultBatches.Count; } } diff --git a/test/vstest.ProgrammerTests/Fakes/FakeTestRuntimeProvider.cs b/test/vstest.ProgrammerTests/Fakes/FakeTestRuntimeProvider.cs index d470b15fef..0c60660bcf 100644 --- a/test/vstest.ProgrammerTests/Fakes/FakeTestRuntimeProvider.cs +++ b/test/vstest.ProgrammerTests/Fakes/FakeTestRuntimeProvider.cs @@ -7,23 +7,39 @@ namespace vstest.ProgrammerTests.CommandLine.Fakes; using Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.Interfaces; using Microsoft.VisualStudio.TestPlatform.ObjectModel.Host; using Microsoft.VisualStudio.TestPlatform.ObjectModel.Logging; +using Microsoft.VisualStudio.TestPlatform.PlatformAbstractions; internal class FakeTestRuntimeProvider : ITestRuntimeProvider { public FakeProcessHelper FakeProcessHelper { get; } public FakeCommunicationEndpoint FakeCommunicationEndpoint { get; } - public FakeProcess? TestHostProcess { get; private set; } + public FakeErrorAggregator FakeErrorAggregator { get; } + public FakeProcess TestHostProcess { get; private set; } // TODO: make this configurable? public bool Shared => false; - public event EventHandler HostLaunched; - public event EventHandler HostExited; + public event EventHandler? HostLaunched; + public event EventHandler? HostExited; - public FakeTestRuntimeProvider(FakeProcessHelper fakeProcessHelper, FakeCommunicationEndpoint fakeCommunicationEndpoint, FakeErrorAggregator fakeErrorAggregator) + public FakeTestRuntimeProvider(FakeProcessHelper fakeProcessHelper, FakeProcess fakeTestHostProcess, FakeCommunicationEndpoint fakeCommunicationEndpoint, FakeErrorAggregator fakeErrorAggregator) { FakeProcessHelper = fakeProcessHelper; + TestHostProcess = fakeTestHostProcess; + TestHostProcess.ExitCallback = p => + { + // TODO: Validate the process we are passed is actually the same as TestHostProcess + // TODO: Validate we already started the process. + var process = (FakeProcess)p; + if (HostExited != null) + { + // TODO: When we exit, eventually there are no subscribers, maybe we should review if we don't lose the error output sometimes, in unnecessary way + HostExited(this, new HostProviderEventArgs(process.ErrorOutput, process.ExitCode, process.Id)); + } + }; + FakeCommunicationEndpoint = fakeCommunicationEndpoint; + FakeErrorAggregator = fakeErrorAggregator; } public bool CanExecuteCurrentRunConfiguration(string runsettingsXml) @@ -47,7 +63,7 @@ public TestHostConnectionInfo GetTestHostConnectionInfo() public TestProcessStartInfo GetTestHostProcessStartInfo(IEnumerable sources, IDictionary environmentVariables, TestRunnerConnectionInfo connectionInfo) { // TODO: do we need to do more here? How to link testhost to the fake one we "start"? - return new TestProcessStartInfo(); + return TestHostProcess.TestProcessStartInfo; } public IEnumerable GetTestPlatformExtensions(IEnumerable sources, IEnumerable extensions) @@ -73,23 +89,15 @@ public void Initialize(IMessageLogger logger, string runsettingsXml) public Task LaunchTestHostAsync(TestProcessStartInfo testHostStartInfo, CancellationToken cancellationToken) { - TestHostProcess = (FakeProcess)FakeProcessHelper.LaunchProcess( - testHostStartInfo.FileName, - testHostStartInfo.Arguments, - testHostStartInfo.WorkingDirectory, - testHostStartInfo.EnvironmentVariables, - errorCallback: (_, _) => { }, - exitCallBack: p => { - var process = (FakeProcess)p; - if (HostExited != null) - { - // TODO: When we exit, eventually there are no subscribers, maybe we should review if we don't lose the error output sometimes, in unnecessary way - HostExited(this, new HostProviderEventArgs(process.ErrorOutput, process.ExitCode, process.Id)); - } - }, - outputCallback: (_, _) => { } - ); - HostLaunched(this, new HostProviderEventArgs("Fake testhost launched", 0, TestHostProcess.Id)); + if (TestHostProcess.TestProcessStartInfo.FileName != testHostStartInfo.FileName) + throw new InvalidOperationException($"Tried to start a different process than the one associated with this provider: File name is {testHostStartInfo.FileName} is not the same as the fake process associated with this provider {TestHostProcess.TestProcessStartInfo.FileName}."); + + FakeProcessHelper.StartFakeProcess(TestHostProcess); + + if (HostLaunched != null) + { + HostLaunched(this, new HostProviderEventArgs("Fake testhost launched", 0, TestHostProcess.Id)); + } return Task.FromResult(true); } diff --git a/test/vstest.ProgrammerTests/Fakes/FakeTestRuntimeProviderManager.cs b/test/vstest.ProgrammerTests/Fakes/FakeTestRuntimeProviderManager.cs index ba7873fd10..c7d19af3f2 100644 --- a/test/vstest.ProgrammerTests/Fakes/FakeTestRuntimeProviderManager.cs +++ b/test/vstest.ProgrammerTests/Fakes/FakeTestRuntimeProviderManager.cs @@ -11,17 +11,27 @@ namespace vstest.ProgrammerTests.CommandLine.Fakes; internal class FakeTestRuntimeProviderManager : ITestRuntimeProviderManager { - public FakeTestRuntimeProviderManager(List testRuntimeProviders, FakeErrorAggregator fakeErrorAggregator) + public FakeTestRuntimeProviderManager(FakeErrorAggregator fakeErrorAggregator) { - testRuntimeProviders.ForEach(TestRuntimeProviders.Enqueue); FakeErrorAggregator = fakeErrorAggregator; } public ConcurrentQueue TestRuntimeProviders { get; } = new(); - public List ProvidedTestRuntimeProviders { get; } = new(); + public List UsedTestRuntimeProviders { get; } = new(); public FakeErrorAggregator FakeErrorAggregator { get; } + public void AddTestRuntimeProviders(params FakeTestRuntimeProvider[] runtimeProviders) + { + // This is not a bug, I am registering each provider twice because TestPlatform resolves + // them twice for every request that does not run in-process. + foreach (var runtimeProvider in runtimeProviders) + { + TestRuntimeProviders.Enqueue(runtimeProvider); + TestRuntimeProviders.Enqueue(runtimeProvider); + } + } + public ITestRuntimeProvider GetTestHostManagerByRunConfiguration(string runConfiguration) { if (!TestRuntimeProviders.TryDequeue(out var next)) @@ -29,7 +39,7 @@ public ITestRuntimeProvider GetTestHostManagerByRunConfiguration(string runConfi throw new InvalidOperationException("There are no more TestRuntimeProviders to be provided"); } - ProvidedTestRuntimeProviders.Add(next); + UsedTestRuntimeProviders.Add(next); return next; } diff --git a/test/vstest.ProgrammerTests/Fixture.cs b/test/vstest.ProgrammerTests/Fixture.cs index 169aa8358e..86e5b6b203 100644 --- a/test/vstest.ProgrammerTests/Fixture.cs +++ b/test/vstest.ProgrammerTests/Fixture.cs @@ -3,6 +3,18 @@ namespace vstest.ProgrammerTests.CommandLine; +using FluentAssertions; + +using Microsoft.VisualStudio.TestPlatform.Client; +using Microsoft.VisualStudio.TestPlatform.CommandLine; +using Microsoft.VisualStudio.TestPlatform.CommandLine.Publisher; +using Microsoft.VisualStudio.TestPlatform.CommandLine.TestPlatformHelpers; +using Microsoft.VisualStudio.TestPlatform.CommandLineUtilities; +using Microsoft.VisualStudio.TestPlatform.CrossPlatEngine; +using Microsoft.VisualStudio.TestPlatform.CrossPlatEngine.TestRunAttachmentsProcessing; +using Microsoft.VisualStudio.TestPlatform.ObjectModel; +using Microsoft.VisualStudio.TestPlatform.ObjectModel.Client; + using vstest.ProgrammerTests.CommandLine.Fakes; internal class Fixture : IDisposable @@ -11,16 +23,104 @@ internal class Fixture : IDisposable public FakeProcessHelper ProcessHelper { get; } public FakeProcess CurrentProcess { get; } public FakeFileHelper FileHelper { get; } + public FakeTestRuntimeProviderManager TestRuntimeProviderManager { get; } + public FakeTestRunEventsRegistrar TestRunEventsRegistrar { get; } + public TestEngine TestEngine { get; private set; } + public TestPlatform TestPlatform { get; private set; } + public TestRunResultAggregator TestRunResultAggregator { get; private set; } + public FakeTestPlatformEventSource TestPlatformEventSource { get; private set; } + public FakeAssemblyMetadataProvider AssemblyMetadataProvider { get; private set; } + public InferHelper InferHelper { get; private set; } + public FakeDataCollectorAttachmentsProcessorsFactory DataCollectorAttachmentsProcessorsFactory { get; private set; } + public TestRunAttachmentsProcessingManager TestRunAttachmentsProcessingManager { get; private set; } + public TestRequestManager TestRequestManager { get; private set; } + public List ExecutedTests => TestRunEventsRegistrar.RunChangedEvents.SelectMany(er => er.Data.NewTestResults).ToList(); + + public ProtocolConfig ProtocolConfig { get; internal set; } public Fixture() { + // We need to use static class to find the communication endpoint, this clears all the registrations of previous tests. + TestServiceLocator.Clear(); + + CurrentProcess = new FakeProcess(ErrorAggregator, @"X:\fake\vstest.console.exe", string.Empty, null, null, null, null, null); ProcessHelper = new FakeProcessHelper(ErrorAggregator, CurrentProcess); FileHelper = new FakeFileHelper(ErrorAggregator); - + TestRuntimeProviderManager = new FakeTestRuntimeProviderManager(ErrorAggregator); + TestRunEventsRegistrar = new FakeTestRunEventsRegistrar(ErrorAggregator); + ProtocolConfig = new ProtocolConfig(); } public void Dispose() { } + + internal void AddTestHostFixtures(params FakeTestFixtureHost[] testhosts) + { + var providers = testhosts.Select(t => t.FakeTestRuntimeProvider).ToArray(); + TestRuntimeProviderManager.AddTestRuntimeProviders(providers); + } + + internal TestRequestManagerTestHelper BuildTestRequestManager( + int? timeout = DebugOptions.DefaultTimeout, + int? debugTimeout = DebugOptions.DefaultDebugTimeout, + bool? breakOnAbort = DebugOptions.DefaultBreakOnAbort) + { + if (!TestRuntimeProviderManager.TestRuntimeProviders.Any()) + throw new InvalidOperationException("There are runtime providers registered for FakeTestRuntimeProviderManager."); + + + TestEngine = new TestEngine(TestRuntimeProviderManager, ProcessHelper); + TestPlatform = new TestPlatform(TestEngine, FileHelper, TestRuntimeProviderManager); + + TestRunResultAggregator = new TestRunResultAggregator(); + TestPlatformEventSource = new FakeTestPlatformEventSource(ErrorAggregator); + + AssemblyMetadataProvider = new FakeAssemblyMetadataProvider(FileHelper, ErrorAggregator); + InferHelper = new InferHelper(AssemblyMetadataProvider); + + // This is most likely not the correctl place where to cut this off, plugin cache is probably the better place, + // but it is not injected, and I don't want to investigate this now. + DataCollectorAttachmentsProcessorsFactory = new FakeDataCollectorAttachmentsProcessorsFactory(ErrorAggregator); + TestRunAttachmentsProcessingManager = new TestRunAttachmentsProcessingManager(TestPlatformEventSource, DataCollectorAttachmentsProcessorsFactory); + + Task fakeMetricsPublisherTask = Task.FromResult(new FakeMetricsPublisher(ErrorAggregator)); + + var commandLineOptions = CommandLineOptions.Instance; + TestRequestManager testRequestManager = new( + commandLineOptions, + TestPlatform, + TestRunResultAggregator, + TestPlatformEventSource, + InferHelper, + fakeMetricsPublisherTask, + ProcessHelper, + TestRunAttachmentsProcessingManager + ); + + TestRequestManager = testRequestManager; + + return new TestRequestManagerTestHelper(ErrorAggregator, testRequestManager, new DebugOptions + { + Timeout = timeout ?? DebugOptions.DefaultTimeout, + DebugTimeout = debugTimeout ?? DebugOptions.DefaultDebugTimeout, + BreakOnAbort = breakOnAbort ?? DebugOptions.DefaultBreakOnAbort, + }); + } + + internal void AssertNoErrors() + { + ErrorAggregator.Errors.Should().BeEmpty(); + } +} + +internal class DebugOptions +{ + public const int DefaultTimeout = 5; + public const int DefaultDebugTimeout = 50; + public const bool DefaultBreakOnAbort = false; + public int Timeout { get; init; } = DefaultTimeout; + public int DebugTimeout { get; init; } = DefaultDebugTimeout; + public bool BreakOnAbort { get; init; } = DefaultBreakOnAbort; } diff --git a/test/vstest.ProgrammerTests/RunConfiguration.cs b/test/vstest.ProgrammerTests/RunConfiguration.cs new file mode 100644 index 0000000000..ef942da3da --- /dev/null +++ b/test/vstest.ProgrammerTests/RunConfiguration.cs @@ -0,0 +1,9 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +namespace vstest.ProgrammerTests.CommandLine; + +internal static class RunConfiguration +{ + public static ConfigurationEntry MaxParallelLevel { get; } = new(nameof(MaxParallelLevel)); +} diff --git a/test/vstest.ProgrammerTests/TestDlls.cs b/test/vstest.ProgrammerTests/TestDlls.cs new file mode 100644 index 0000000000..29191fb2d5 --- /dev/null +++ b/test/vstest.ProgrammerTests/TestDlls.cs @@ -0,0 +1,9 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +namespace vstest.ProgrammerTests.CommandLine; + +internal class TestDlls +{ + public static string MSTest1 { get; } = $"{nameof(MSTest1)}.dll"; +} diff --git a/test/vstest.ProgrammerTests/TestRequestManagerHelper.cs b/test/vstest.ProgrammerTests/TestRequestManagerHelper.cs new file mode 100644 index 0000000000..c66c64f3be --- /dev/null +++ b/test/vstest.ProgrammerTests/TestRequestManagerHelper.cs @@ -0,0 +1,58 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +namespace vstest.ProgrammerTests.CommandLine; + +using System.Diagnostics; + +using Microsoft.VisualStudio.TestPlatform.CommandLine.TestPlatformHelpers; + +internal class TestRequestManagerTestHelper +{ + private readonly FakeErrorAggregator _errorAggregator; + private TestRequestManager _testRequestManager; + private readonly DebugOptions _debugOptions; + + public TestRequestManagerTestHelper(FakeErrorAggregator errorAggregator, TestRequestManager testRequestManager, DebugOptions debugOptions) + { + _errorAggregator = errorAggregator; + _testRequestManager = testRequestManager; + _debugOptions = debugOptions; + } + + public async Task ExecuteWithAbort(Action testRequsestManagerAction) + { + // We make sure the test is running for the timeout time at max and then we try to abort + // if we aborted we write the error to aggregator + + // Start tasks that waits until it is the right time to call abort + // and continue to starting the method. If that method finishes running on time we cancel this + // wait and don't abort. Otherwise we call abort to start our abort flow. + // + // This abort does not guarantee that we won't hang. If our abort flow is broken then we will + // remain hanging. To have that guarantee it needs to be handled by failfast, or something else that will hang dump us. + // Or a simple timer that kill the process after a given timeout, like a simplified blame hang dumper. + var cancelAbort = new CancellationTokenSource(); + var abortOnTimeout = Task.Run(async () => + { + // Wait until timeout or until we are cancelled. + await Task.Delay(TimeSpan.FromSeconds(Debugger.IsAttached ? _debugOptions.DebugTimeout : _debugOptions.Timeout), cancelAbort.Token); + if (Debugger.IsAttached && _debugOptions.BreakOnAbort) + { + var errors = _errorAggregator.Errors; + // we will abort because we are hanging, look at errors and at concurrent stacks to see where we are hanging. + Debugger.Break(); + } + _errorAggregator.Add(new Exception("errr we aborted")); + _testRequestManager.AbortTestRun(); + }); + + testRequsestManagerAction(_testRequestManager); + + cancelAbort.Cancel(); + if (!abortOnTimeout.IsCanceled) + { + await abortOnTimeout; + } + } +} diff --git a/test/vstest.ProgrammerTests/UnitTest1.cs b/test/vstest.ProgrammerTests/UnitTest1.cs index e29ba2fb50..11b5174e9c 100644 --- a/test/vstest.ProgrammerTests/UnitTest1.cs +++ b/test/vstest.ProgrammerTests/UnitTest1.cs @@ -52,7 +52,7 @@ public async Task GivenAnMSTestAssemblyWith108Tests_WhenTestsAreRun_Then108Tests var fakeErrorAggregator = new FakeErrorAggregator(); var commandLineOptions = CommandLineOptions.Instance; - var fakeCurrentProcess = new FakeProcess(fakeErrorAggregator, @"X:\fake\vstest.console.exe", string.Empty, null, null, null, null, null); + var fakeCurrentProcess = new FakeProcess(fakeErrorAggregator, @"X:\fake\vstest.console.exe"); var fakeProcessHelper = new FakeProcessHelper(fakeErrorAggregator, fakeCurrentProcess); var fakeFileHelper = new FakeFileHelper(fakeErrorAggregator); @@ -95,8 +95,10 @@ public async Task GivenAnMSTestAssemblyWith108Tests_WhenTestsAreRun_Then108Tests var fakeCommunicationEndpoint = new FakeCommunicationEndpoint(new FakeCommunicationChannel(responses, fakeErrorAggregator, 1), fakeErrorAggregator); TestServiceLocator.Clear(); TestServiceLocator.Register(fakeCommunicationEndpoint.TestHostConnectionInfo.Endpoint, fakeCommunicationEndpoint); - var fakeTestRuntimeProvider = new FakeTestRuntimeProvider(fakeProcessHelper, fakeCommunicationEndpoint, fakeErrorAggregator); - var fakeTestRuntimeProviderManager = new FakeTestRuntimeProviderManager(new[] { fakeTestRuntimeProvider, fakeTestRuntimeProvider }.ToList(), fakeErrorAggregator); + var fakeTestHostProcess = new FakeProcess(fakeErrorAggregator, @"C:\temp\testhost.exe"); + var fakeTestRuntimeProvider = new FakeTestRuntimeProvider(fakeProcessHelper, fakeTestHostProcess, fakeCommunicationEndpoint, fakeErrorAggregator); + var fakeTestRuntimeProviderManager = new FakeTestRuntimeProviderManager(fakeErrorAggregator); + fakeTestRuntimeProviderManager.AddTestRuntimeProviders(fakeTestRuntimeProvider); var testEngine = new TestEngine(fakeTestRuntimeProviderManager, fakeProcessHelper); var testPlatform = new TestPlatform(testEngine, fakeFileHelper, fakeTestRuntimeProviderManager); @@ -176,15 +178,9 @@ public async Task GivenAnMSTestAssemblyWith108Tests_WhenTestsAreRun_Then108Tests public async Task GivenMultipleMsTestAssembliesThatUseTheSameTargetFrameworkAndArchitecture_WhenTestsAreRun_ThenAllTestsFromAllAssembliesAreRun() { - // We need to use static class to find the communication endpoint, this clears all the registrations of previous tests. - TestServiceLocator.Clear(); - using var fixture = new FixtureBuilder() - .Build(); - // -- arrange + using var fixture = new Fixture(); - var commandLineOptions = CommandLineOptions.Instance; - // TODO: have mstest1dll canned var mstest1Dll = new FakeTestDllBuilder() .WithPath(@"X:\fake\mstest1.dll") .WithFramework(KnownFramework.Net50) @@ -192,7 +188,7 @@ public async Task GivenMultipleMsTestAssembliesThatUseTheSameTargetFrameworkAndA .WithTestCount(108, 10) .Build(); - var testhost1Process = new FakeProcess(fixture.ErrorAggregator, @"X:\fake\testhost1.exe", string.Empty, null, null, null, null, null); + var testhost1Process = new FakeProcess(fixture.ErrorAggregator, @"X:\fake\testhost1.exe"); var runTests1 = new FakeMessagesBuilder() .VersionCheck(5) @@ -201,7 +197,7 @@ public async Task GivenMultipleMsTestAssembliesThatUseTheSameTargetFrameworkAndA .SessionEnd(FakeMessage.NoResponse, _ => testhost1Process.Exit()) .Build(); - var testhost1 = new FakeTestHostBuilder(fixture) + var testhost1 = new FakeTestHostFixtureBuilder(fixture) .WithTestDll(mstest1Dll) .WithProcess(testhost1Process) .WithResponses(runTests1) @@ -214,7 +210,7 @@ public async Task GivenMultipleMsTestAssembliesThatUseTheSameTargetFrameworkAndA .WithTestCount(50, 8) .Build(); - var testhost2Process = new FakeProcess(fixture.ErrorAggregator, @"X:\fake\testhost1.exe", string.Empty, null, null, null, null, null); + var testhost2Process = new FakeProcess(fixture.ErrorAggregator, @"X:\fake\testhost1.exe"); var runTests2 = new FakeMessagesBuilder() .VersionCheck(5) @@ -223,7 +219,7 @@ public async Task GivenMultipleMsTestAssembliesThatUseTheSameTargetFrameworkAndA .SessionEnd(FakeMessage.NoResponse, _ => testhost1Process.Exit()) .Build(); - var testhost2 = new FakeTestHostBuilder(fixture) + var testhost2 = new FakeTestHostFixtureBuilder(fixture) .WithTestDll(mstest2Dll) .WithProcess(testhost2Process) .WithResponses(runTests2) @@ -231,126 +227,28 @@ public async Task GivenMultipleMsTestAssembliesThatUseTheSameTargetFrameworkAndA // --- - - var fakeTestRuntimeProviderManager = new FakeTestRuntimeProviderManager(new[] { fakeTestRuntimeProvider, fakeTestRuntimeProvider, fakeTestRuntimeProvider2, fakeTestRuntimeProvider2 }.ToList(), fixture.ErrorAggregator); - var testEngine = new TestEngine(fakeTestRuntimeProviderManager, fixture.ProcessHelper); + fixture.AddTestHostFixtures(testhost1, testhost2); - var testPlatform = new TestPlatform(testEngine, fixture.FileHelper, fakeTestRuntimeProviderManager); - - var testRunResultAggregator = new TestRunResultAggregator(); - var fakeTestPlatformEventSource = new FakeTestPlatformEventSource(fixture.ErrorAggregator); - - var fakeAssemblyMetadataProvider = new FakeAssemblyMetadataProvider(fixture.FileHelper, fixture.ErrorAggregator); - var inferHelper = new InferHelper(fakeAssemblyMetadataProvider); - - // This is most likely not the correctl place where to cut this off, plugin cache is probably the better place, - // but it is not injected, and I don't want to investigate this now. - var fakeDataCollectorAttachmentsProcessorsFactory = new FakeDataCollectorAttachmentsProcessorsFactory(fixture.ErrorAggregator); - var testRunAttachmentsProcessingManager = new TestRunAttachmentsProcessingManager(fakeTestPlatformEventSource, fakeDataCollectorAttachmentsProcessorsFactory); - - Task fakeMetricsPublisherTask = Task.FromResult(new FakeMetricsPublisher(fixture.ErrorAggregator)); - TestRequestManager testRequestManager = new( - commandLineOptions, - testPlatform, - testRunResultAggregator, - fakeTestPlatformEventSource, - inferHelper, - fakeMetricsPublisherTask, - fixture.ProcessHelper, - testRunAttachmentsProcessingManager - ); + var testRequestManager = fixture.BuildTestRequestManager(5, 50, true); // -- act - - // TODO: this gives me run configuration that is way too complete, do we a way to generate "bare" runsettings? if not we should add them. Would be also useful to get - // runsettings from parameter set so people can use it - // TODO: TestSessionTimeout gives me way to abort the run without having to cancel it externally, but could probably still lead to hangs if that funtionality is broken - // TODO: few tries later, that is exactly the case when we abort, it still hangs on waiting to complete request, because test run complete was not sent - // var runConfiguration = new Microsoft.VisualStudio.TestPlatform.ObjectModel.RunConfiguration { TestSessionTimeout = 40_000 }.ToXml().OuterXml; var runConfiguration = string.Empty; var testRunRequestPayload = new TestRunRequestPayload { - // TODO: passing null sources and null testcases does not fail fast Sources = new List { mstest1Dll.Path, mstest2Dll.Path }, - // TODO: passing null runsettings does not fail fast, instead it fails in Fakes settings code - // TODO: passing empty string fails in the xml parser code + RunSettings = $"{runConfiguration}" }; - // var fakeTestHostLauncher = new FakeTestHostLauncher(); - var fakeTestRunEventsRegistrar = new FakeTestRunEventsRegistrar(fixture.ErrorAggregator); - var protocolConfig = new ProtocolConfig(); - - // TODO: we make sure the test is running 10 minutes at max and then we try to abort - // if we aborted we write the error to aggregator, this needs to be made into a pattern - // so we can avoid hanging if the run does not complete correctly - var cancelAbort = new CancellationTokenSource(); - var task = Task.Run(async () => - { - await Task.Delay(TimeSpan.FromSeconds(Debugger.IsAttached ? 50 : 5), cancelAbort.Token); - if (Debugger.IsAttached) - { - var errors = fixture.ErrorAggregator.Errors; - // we will abort because we are hanging, look at stacks to see what the problem is - Debugger.Break(); - } - fixture.ErrorAggregator.Add(new Exception("errr we aborted")); - testRequestManager.AbortTestRun(); - - }); - testRequestManager.RunTests(testRunRequestPayload, testHostLauncher: null, fakeTestRunEventsRegistrar, protocolConfig); - cancelAbort.Cancel(); - if (!task.IsCanceled) - { - await task; - } - // pattern end + await testRequestManager.ExecuteWithAbort(tm => tm.RunTests(testRunRequestPayload, testHostLauncher: null, fixture.TestRunEventsRegistrar, fixture.ProtocolConfig)); // -- assert - fixture.ErrorAggregator.Errors.Should().BeEmpty(); - fakeTestRunEventsRegistrar.RunChangedEvents.SelectMany(er => er.Data.NewTestResults).Should().HaveCount(158); + fixture.AssertNoErrors(); + fixture.ExecutedTests.Should().HaveCount(mstest1Dll.TestCount + mstest2Dll.TestCount); } } - -internal class FixtureBuilder -{ - internal Fixture Build() - { - return new Fixture(); - } -} - -internal class CapturedRunSettings -{ - public int MaxParallelLevel { get; internal set; } -} - -internal static class RunConfiguration -{ - public static ConfigurationEntry MaxParallelLevel { get; } = new(nameof(MaxParallelLevel)); -} - -internal class ConfigurationEntry -{ - public ConfigurationEntry(string name) - { - Name = name; - } - - public string Name { get; } - - public string InlinePath => $"RunConfiguration.{Name}"; - - public string FullPath => $"RunSettings.{InlinePath}"; - - public override string ToString() - { - return Name; - } -} - -internal class TestDlls -{ - public static string MSTest1 { get; } = $"{nameof(MSTest1)}.dll"; -} +// Test and improvmement ideas: +// TODO: passing null runsettings does not fail fast, instead it fails in Fakes settings code +// TODO: passing empty string fails in the xml parser code +// TODO: passing null sources and null testcases does not fail fast From 4cd58c7026b711861c36e650ddf2189f7ea6700e Mon Sep 17 00:00:00 2001 From: nohwnd Date: Thu, 17 Feb 2022 16:00:56 +0100 Subject: [PATCH 15/32] use the correct testhost actually. --- test/vstest.ProgrammerTests/UnitTest1.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/vstest.ProgrammerTests/UnitTest1.cs b/test/vstest.ProgrammerTests/UnitTest1.cs index 11b5174e9c..8d51726e18 100644 --- a/test/vstest.ProgrammerTests/UnitTest1.cs +++ b/test/vstest.ProgrammerTests/UnitTest1.cs @@ -216,7 +216,7 @@ public async Task GivenMultipleMsTestAssembliesThatUseTheSameTargetFrameworkAndA .VersionCheck(5) .ExecutionInitialize(FakeMessage.NoResponse) .StartTestExecutionWithSources(mstest2Dll.TestResultBatches) - .SessionEnd(FakeMessage.NoResponse, _ => testhost1Process.Exit()) + .SessionEnd(FakeMessage.NoResponse, _ => testhost2Process.Exit()) .Build(); var testhost2 = new FakeTestHostFixtureBuilder(fixture) From a6386032dfe9da862d5ec05eed512bdb9d3bad5d Mon Sep 17 00:00:00 2001 From: nohwnd Date: Thu, 17 Feb 2022 23:34:04 +0100 Subject: [PATCH 16/32] Test multi tfm --- .../TestRequestSender.cs | 6 +- .../TestServiceLocator.cs | 8 +- .../InferRunSettingsHelper.cs | 1 + test/Intent.Primitives/IRunLogger.cs | 4 +- test/Intent.Primitives/OnlyAttribute.cs | 9 ++ test/Intent/Extensions.cs | 4 +- test/Intent/Runner.cs | 12 +- .../FakeMessagesBuilder.cs | 15 ++- .../FakeTestDllBuilder.cs | 2 +- .../FakeTestHostBuilder.cs | 4 +- .../Fakes/FakeCommunicationEndpoint.cs | 9 +- .../Fakes/FakeFileHelper.cs | 5 + .../Fakes/FakeProcess.cs | 12 ++ .../Fakes/FakeProcessHelper.cs | 7 +- .../Fakes/FakeTestBatchBuilder.cs | 3 + .../Fakes/FakeTestRuntimeProvider.cs | 28 ++++- .../Fakes/FakeTestRuntimeProviderManager.cs | 2 + test/vstest.ProgrammerTests/Fixture.cs | 11 +- test/vstest.ProgrammerTests/Id.cs | 2 +- test/vstest.ProgrammerTests/KnownFramework.cs | 14 ++- .../StringExtensions.cs | 21 ++-- test/vstest.ProgrammerTests/UnitTest1.cs | 112 ++++++++++++++++-- 22 files changed, 246 insertions(+), 45 deletions(-) create mode 100644 test/Intent.Primitives/OnlyAttribute.cs diff --git a/src/Microsoft.TestPlatform.CommunicationUtilities/TestRequestSender.cs b/src/Microsoft.TestPlatform.CommunicationUtilities/TestRequestSender.cs index e1a39fcd28..bd3b5a3642 100644 --- a/src/Microsoft.TestPlatform.CommunicationUtilities/TestRequestSender.cs +++ b/src/Microsoft.TestPlatform.CommunicationUtilities/TestRequestSender.cs @@ -107,7 +107,11 @@ internal TestRequestSender( // resort of getting the dependency into the execution flow. // TODO: I am not sure if we need multiple instances of ICommunicationEndpoint, in that case we should register // and resolve Func and invoke that. - _communicationEndpoint = communicationEndPoint ?? TestServiceLocator.Get(connectionInfo.Endpoint) ?? SetCommunicationEndPoint(); + _communicationEndpoint = communicationEndPoint +#if DEBUG + ?? TestServiceLocator.Get(connectionInfo.Endpoint) +#endif + ?? SetCommunicationEndPoint(); _connectionInfo.Endpoint = connectionInfo.Endpoint; _connectionInfo.Role = connectionInfo.Role == ConnectionRole.Host ? ConnectionRole.Client diff --git a/src/Microsoft.TestPlatform.ObjectModel/TestServiceLocator.cs b/src/Microsoft.TestPlatform.ObjectModel/TestServiceLocator.cs index 9680246f89..075422a65d 100644 --- a/src/Microsoft.TestPlatform.ObjectModel/TestServiceLocator.cs +++ b/src/Microsoft.TestPlatform.ObjectModel/TestServiceLocator.cs @@ -17,10 +17,14 @@ public static void Register(string name, TRegistration instance) Instances.Add(name, instance); } - public static TRegistration Get(string name) + public static TRegistration? Get(string name) { if (!Instances.TryGetValue(name, out var instance)) - throw new InvalidOperationException($"Cannot find an instance for name {name}."); + { + return default; + // TODO: Add enable flag for the whole provider to activate so I can leverage throwing in programmer tests, but not run into it in Playground, or other debug builds. + // throw new InvalidOperationException($"Cannot find an instance for name {name}."); + } #if !NETSTANDARD1_0 Resolves.Add(new Resolve(name, typeof(TRegistration).FullName, Environment.StackTrace)); diff --git a/src/Microsoft.TestPlatform.Utilities/InferRunSettingsHelper.cs b/src/Microsoft.TestPlatform.Utilities/InferRunSettingsHelper.cs index e65aef5e0a..4e7e37a311 100644 --- a/src/Microsoft.TestPlatform.Utilities/InferRunSettingsHelper.cs +++ b/src/Microsoft.TestPlatform.Utilities/InferRunSettingsHelper.cs @@ -499,6 +499,7 @@ private static void AddNodeIfNotPresent(XmlDocument xmlDocument, string nodeP if (root.SelectSingleNode(RunConfigurationNodePath) == null) { + // TODO: When runsettings are incomplete this will silently return, when we run just TestRequestManager we don't get full settings. EqtTrace.Error("InferRunSettingsHelper.UpdateNodeIfNotPresent: Unable to navigate to RunConfiguration. Current node: " + xmlDocument.LocalName); return; } diff --git a/test/Intent.Primitives/IRunLogger.cs b/test/Intent.Primitives/IRunLogger.cs index 20891f65af..474369e78c 100644 --- a/test/Intent.Primitives/IRunLogger.cs +++ b/test/Intent.Primitives/IRunLogger.cs @@ -1,10 +1,10 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT license. See LICENSE file in the project root for full license information. -using System.Reflection; - namespace Intent; +using System.Reflection; + public interface IRunLogger { void WriteTestPassed(MethodInfo m); diff --git a/test/Intent.Primitives/OnlyAttribute.cs b/test/Intent.Primitives/OnlyAttribute.cs new file mode 100644 index 0000000000..3d327e1f55 --- /dev/null +++ b/test/Intent.Primitives/OnlyAttribute.cs @@ -0,0 +1,9 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +namespace Intent; + +[AttributeUsage(AttributeTargets.Assembly | AttributeTargets.Class | AttributeTargets.Constructor | AttributeTargets.Method)] +public class OnlyAttribute : Attribute +{ +} diff --git a/test/Intent/Extensions.cs b/test/Intent/Extensions.cs index 9e777c3162..6ec98807d0 100644 --- a/test/Intent/Extensions.cs +++ b/test/Intent/Extensions.cs @@ -1,10 +1,10 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT license. See LICENSE file in the project root for full license information. -using System.Reflection; - namespace Intent; +using System.Reflection; + public static class Extensions { public static bool IsExcluded(this Assembly asm) diff --git a/test/Intent/Runner.cs b/test/Intent/Runner.cs index 8fb15c7696..1fa12c365f 100644 --- a/test/Intent/Runner.cs +++ b/test/Intent/Runner.cs @@ -1,10 +1,10 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT license. See LICENSE file in the project root for full license information. -using System.Reflection; - namespace Intent; +using System.Reflection; + public class Runner { public static void Run(IEnumerable path, IRunLogger logger) @@ -21,6 +21,14 @@ public static void Run(IEnumerable path, IRunLogger logger) foreach (var t in ts) { var ms = t.GetMethods().SkipExcluded(); + + // This chooses the Only tests only for single assembly and single class, + // to support this full we would have to enumerate all classes and methods first, + // it is easy, I just don't need it right now. + var only = ms.Where(m => m.GetCustomAttribute() != null).ToList(); + if (only.Any()) + ms = only; + foreach (var m in ms) { try diff --git a/test/vstest.ProgrammerTests/FakeMessagesBuilder.cs b/test/vstest.ProgrammerTests/FakeMessagesBuilder.cs index 875433c327..5106f3cc0b 100644 --- a/test/vstest.ProgrammerTests/FakeMessagesBuilder.cs +++ b/test/vstest.ProgrammerTests/FakeMessagesBuilder.cs @@ -45,7 +45,7 @@ internal FakeMessagesBuilder VersionCheck(FakeMessage message) /// /// /// - internal FakeMessagesBuilder VersionCheck(Action beforeAction, FakeMessage message, Action afterAction) + internal FakeMessagesBuilder VersionCheck(FakeMessage message, Action? beforeAction = null, Action? afterAction = null) { AddPairWithFakeMessage(MessageType.VersionCheck, message, beforeAction, afterAction); return this; @@ -57,8 +57,17 @@ internal FakeMessagesBuilder ExecutionInitialize(FakeMessage message) return this; } - internal FakeMessagesBuilder StartTestExecutionWithSources(List> testResultBatches) + internal FakeMessagesBuilder StartTestExecutionWithSources(FakeMessage message, Action? beforeAction = null, Action? afterAction = null) { + AddPairWithFakeMessage(MessageType.StartTestExecutionWithSources, message, beforeAction, afterAction); + return this; + } + + internal FakeMessagesBuilder StartTestExecutionWithSources(List> testResultBatches!!) + { + if (testResultBatches.Count == 0) + throw new InvalidOperationException("There must be at least one batch with at least one test. If you you wish to not respond with any tests, or respond with broken data, create that fake message from scratch."); + var tests = testResultBatches; // this will create as many test stats changes messages, as there are batches -1 // the last batch will be sent as test run complete event @@ -80,7 +89,7 @@ internal FakeMessagesBuilder StartTestExecutionWithSources(List return this; } - + internal FakeMessagesBuilder SessionEnd(FakeMessage fakeMessage) { AddPairWithFakeMessage(MessageType.SessionEnd, fakeMessage); diff --git a/test/vstest.ProgrammerTests/FakeTestDllBuilder.cs b/test/vstest.ProgrammerTests/FakeTestDllBuilder.cs index dc035f9950..df5149fb51 100644 --- a/test/vstest.ProgrammerTests/FakeTestDllBuilder.cs +++ b/test/vstest.ProgrammerTests/FakeTestDllBuilder.cs @@ -11,7 +11,7 @@ namespace vstest.ProgrammerTests.CommandLine; internal class FakeTestDllBuilder { private string _path = @$"X:\fake\mstest_{Guid.NewGuid()}.dll"; - private FrameworkName _framework = KnownFramework.Net50; + private FrameworkName _framework = KnownFramework.Net5; private Architecture _architecture = Architecture.X64; private List>? _testBatches; diff --git a/test/vstest.ProgrammerTests/FakeTestHostBuilder.cs b/test/vstest.ProgrammerTests/FakeTestHostBuilder.cs index b290bf9b83..552c819dd5 100644 --- a/test/vstest.ProgrammerTests/FakeTestHostBuilder.cs +++ b/test/vstest.ProgrammerTests/FakeTestHostBuilder.cs @@ -47,14 +47,12 @@ internal FakeTestFixtureHost Build() var id = Id.Next(); var fakeCommunicationChannel = new FakeCommunicationChannel(_responses, _fixture.ErrorAggregator, id); var fakeCommunicationEndpoint = new FakeCommunicationEndpoint(fakeCommunicationChannel, _fixture.ErrorAggregator); - var fakeTestRuntimeProvider = new FakeTestRuntimeProvider(_fixture.ProcessHelper, _process, fakeCommunicationEndpoint, _fixture.ErrorAggregator); + var fakeTestRuntimeProvider = new FakeTestRuntimeProvider(_fixture.ProcessHelper, _process, _fixture.FileHelper, _dlls, fakeCommunicationEndpoint, _fixture.ErrorAggregator); // This registers the endpoint so we can look it up later using the address, the Id from here is propagated to // testhost connection info, and is used as port in 127.0.0.1:, address so we can lookup the correct channel. TestServiceLocator.Register(fakeCommunicationEndpoint.TestHostConnectionInfo.Endpoint, fakeCommunicationEndpoint); - _dlls.ForEach(_fixture.FileHelper.AddFile); - return new FakeTestFixtureHost(_fixture, id, _dlls, fakeTestRuntimeProvider, fakeCommunicationEndpoint, fakeCommunicationChannel, _process, _responses); } diff --git a/test/vstest.ProgrammerTests/Fakes/FakeCommunicationEndpoint.cs b/test/vstest.ProgrammerTests/Fakes/FakeCommunicationEndpoint.cs index 7e472f7d17..5bbc0dafbf 100644 --- a/test/vstest.ProgrammerTests/Fakes/FakeCommunicationEndpoint.cs +++ b/test/vstest.ProgrammerTests/Fakes/FakeCommunicationEndpoint.cs @@ -55,15 +55,18 @@ public string Start(string endPoint) return endPoint; } + public void Abort() + { + Disconnected?.Invoke(this, new DisconnectedEventArgs()); + _stopped = true; + } + public void Stop() { if (!_stopped) { // Do not allow stop to be called multiple times, because it will end up calling us back and stack overflows. _stopped = true; - - // TODO: notify this in case of error in the process, so we can initiate abort flow - // Disconnected?.Invoke(this, new DisconnectedEventArgs()); } } } diff --git a/test/vstest.ProgrammerTests/Fakes/FakeFileHelper.cs b/test/vstest.ProgrammerTests/Fakes/FakeFileHelper.cs index f9d3a02329..221bba176d 100644 --- a/test/vstest.ProgrammerTests/Fakes/FakeFileHelper.cs +++ b/test/vstest.ProgrammerTests/Fakes/FakeFileHelper.cs @@ -119,6 +119,11 @@ public void WriteAllTextToFile(string filePath, string content) internal void AddFile(T file) where T : FakeFile { + if (Files.Any(f => f.Path.Equals(file.Path, StringComparison.OrdinalIgnoreCase))) + { + throw new InvalidOperationException($"Fake file '{file.Path}' already exists."); + } + Files.Add(file); } } diff --git a/test/vstest.ProgrammerTests/Fakes/FakeProcess.cs b/test/vstest.ProgrammerTests/Fakes/FakeProcess.cs index 74fbd2b80b..d24a705d94 100644 --- a/test/vstest.ProgrammerTests/Fakes/FakeProcess.cs +++ b/test/vstest.ProgrammerTests/Fakes/FakeProcess.cs @@ -24,6 +24,7 @@ internal class FakeProcess public FakeErrorAggregator FakeErrorAggregator { get; } public string? ErrorOutput { get; init; } public int ExitCode { get; init; } = -1; + public bool Started { get; private set; } public bool Exited { get; private set; } public TestProcessStartInfo TestProcessStartInfo { get; internal set; } @@ -63,8 +64,19 @@ internal void SetId(int id) Id = id; } + internal void Start() + { + if (Started) + throw new InvalidOperationException($"Cannot start process {Name} - {Id} because it was already started before."); + + Started = true; + } + internal void Exit() { + if (!Started) + throw new InvalidOperationException($"Cannot exit process {Name} - {Id} because it was not started before."); + // We want to call the exit callback just once. This is behavior inherent to being a real process, // that also exits only once. var exited = Exited; diff --git a/test/vstest.ProgrammerTests/Fakes/FakeProcessHelper.cs b/test/vstest.ProgrammerTests/Fakes/FakeProcessHelper.cs index d583fe5092..cb8797b72f 100644 --- a/test/vstest.ProgrammerTests/Fakes/FakeProcessHelper.cs +++ b/test/vstest.ProgrammerTests/Fakes/FakeProcessHelper.cs @@ -86,6 +86,7 @@ public object LaunchProcess(string processPath, string arguments, string working // TODO: Throw if setting says we can't start new processes; var process = new FakeProcess(FakeErrorAggregator, processPath, arguments, workingDirectory, environmentVariables, errorCallback, exitCallBack, outputCallback); Processes.Add(process); + process.Start(); return process; } @@ -112,8 +113,12 @@ public void WaitForProcessExit(object process) // todo: implement for timeouts? } - internal void StartFakeProcess(FakeProcess testHostProcess) + internal void StartFakeProcess(FakeProcess process) { // TODO: mark the process as started. Do not add a new process if it did not exist. + if (!Processes.Contains(process)) + throw new InvalidOperationException($"Cannot start process {process.Name} - {process.Id} because it was not found in the list of known fake processes."); + + process.Start(); } } diff --git a/test/vstest.ProgrammerTests/Fakes/FakeTestBatchBuilder.cs b/test/vstest.ProgrammerTests/Fakes/FakeTestBatchBuilder.cs index b7edfccb1f..020b00c790 100644 --- a/test/vstest.ProgrammerTests/Fakes/FakeTestBatchBuilder.cs +++ b/test/vstest.ProgrammerTests/Fakes/FakeTestBatchBuilder.cs @@ -48,6 +48,9 @@ internal FakeTestBatchBuilder WithBatchSize(int batchSize) internal List> Build() { + if (TotalCount == 0 || BatchSize == 0) + throw new InvalidOperationException("There must be at least one batch with at least one test."); + var numberOfBatches = Math.DivRem(TotalCount, BatchSize, out int remainder); // TODO: Add adapter uri, and dll name diff --git a/test/vstest.ProgrammerTests/Fakes/FakeTestRuntimeProvider.cs b/test/vstest.ProgrammerTests/Fakes/FakeTestRuntimeProvider.cs index 0c60660bcf..3243c64b92 100644 --- a/test/vstest.ProgrammerTests/Fakes/FakeTestRuntimeProvider.cs +++ b/test/vstest.ProgrammerTests/Fakes/FakeTestRuntimeProvider.cs @@ -7,7 +7,6 @@ namespace vstest.ProgrammerTests.CommandLine.Fakes; using Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.Interfaces; using Microsoft.VisualStudio.TestPlatform.ObjectModel.Host; using Microsoft.VisualStudio.TestPlatform.ObjectModel.Logging; -using Microsoft.VisualStudio.TestPlatform.PlatformAbstractions; internal class FakeTestRuntimeProvider : ITestRuntimeProvider { @@ -15,6 +14,8 @@ internal class FakeTestRuntimeProvider : ITestRuntimeProvider public FakeCommunicationEndpoint FakeCommunicationEndpoint { get; } public FakeErrorAggregator FakeErrorAggregator { get; } public FakeProcess TestHostProcess { get; private set; } + public FakeFileHelper FileHelper { get; } + public List TestDlls { get; } // TODO: make this configurable? public bool Shared => false; @@ -22,10 +23,27 @@ internal class FakeTestRuntimeProvider : ITestRuntimeProvider public event EventHandler? HostLaunched; public event EventHandler? HostExited; - public FakeTestRuntimeProvider(FakeProcessHelper fakeProcessHelper, FakeProcess fakeTestHostProcess, FakeCommunicationEndpoint fakeCommunicationEndpoint, FakeErrorAggregator fakeErrorAggregator) + public FakeTestRuntimeProvider(FakeProcessHelper fakeProcessHelper, FakeProcess fakeTestHostProcess, FakeFileHelper fakeFileHelper, List fakeTestDlls, FakeCommunicationEndpoint fakeCommunicationEndpoint, FakeErrorAggregator fakeErrorAggregator) { FakeProcessHelper = fakeProcessHelper; TestHostProcess = fakeTestHostProcess; + FileHelper = fakeFileHelper; + TestDlls = fakeTestDlls; + FakeCommunicationEndpoint = fakeCommunicationEndpoint; + FakeErrorAggregator = fakeErrorAggregator; + + var architectures = fakeTestDlls.Select(dll => dll.Architecture).Distinct().ToList(); + var frameworks = fakeTestDlls.Select(dll => dll.FrameworkName).Distinct().ToList(); + + if (architectures.Count > 1) + throw new InvalidOperationException($"The provided dlls have more than 1 architecture {architectures.JoinByComma()}. Fake TestRuntimeProvider cannot have dlls with mulitple architectures, because real testhost process can also run just with a single architecture."); + + if (frameworks.Count > 1) + throw new InvalidOperationException($"The provided dlls have more than 1 target framework {frameworks.JoinByComma()}. Fake TestRuntimeProvider cannot have dlls with mulitple target framework, because real testhost process can also run just a single target framework."); + + fakeTestDlls.ForEach(FileHelper.AddFile); + + fakeProcessHelper.AddFakeProcess(fakeTestHostProcess); TestHostProcess.ExitCallback = p => { // TODO: Validate the process we are passed is actually the same as TestHostProcess @@ -37,13 +55,13 @@ public FakeTestRuntimeProvider(FakeProcessHelper fakeProcessHelper, FakeProcess HostExited(this, new HostProviderEventArgs(process.ErrorOutput, process.ExitCode, process.Id)); } }; - - FakeCommunicationEndpoint = fakeCommunicationEndpoint; - FakeErrorAggregator = fakeErrorAggregator; } public bool CanExecuteCurrentRunConfiguration(string runsettingsXml) { + // x86 + // Framework40 + return true; } diff --git a/test/vstest.ProgrammerTests/Fakes/FakeTestRuntimeProviderManager.cs b/test/vstest.ProgrammerTests/Fakes/FakeTestRuntimeProviderManager.cs index c7d19af3f2..45d7360b08 100644 --- a/test/vstest.ProgrammerTests/Fakes/FakeTestRuntimeProviderManager.cs +++ b/test/vstest.ProgrammerTests/Fakes/FakeTestRuntimeProviderManager.cs @@ -34,6 +34,8 @@ public void AddTestRuntimeProviders(params FakeTestRuntimeProvider[] runtimeProv public ITestRuntimeProvider GetTestHostManagerByRunConfiguration(string runConfiguration) { + + if (!TestRuntimeProviders.TryDequeue(out var next)) { throw new InvalidOperationException("There are no more TestRuntimeProviders to be provided"); diff --git a/test/vstest.ProgrammerTests/Fixture.cs b/test/vstest.ProgrammerTests/Fixture.cs index 86e5b6b203..7e74b9a80b 100644 --- a/test/vstest.ProgrammerTests/Fixture.cs +++ b/test/vstest.ProgrammerTests/Fixture.cs @@ -43,7 +43,6 @@ public Fixture() // We need to use static class to find the communication endpoint, this clears all the registrations of previous tests. TestServiceLocator.Clear(); - CurrentProcess = new FakeProcess(ErrorAggregator, @"X:\fake\vstest.console.exe", string.Empty, null, null, null, null, null); ProcessHelper = new FakeProcessHelper(ErrorAggregator, CurrentProcess); FileHelper = new FakeFileHelper(ErrorAggregator); @@ -118,8 +117,14 @@ internal void AssertNoErrors() internal class DebugOptions { public const int DefaultTimeout = 5; - public const int DefaultDebugTimeout = 50; - public const bool DefaultBreakOnAbort = false; + // TODO: This setting is actually quite pointless, because I cannot come up with + // a useful way to abort quickly enough when debugger is attached and I am just running my tests (pressing F5) + // but at the same time not abort when I am in the middle of debugging some behavior. Maybe looking at debugger, + // and asking it if any breakpoints were hit / are set. But that is difficult. + // + // So normally I press F5 to investigate, but Ctrl+F5 (run without debugger), to run tests. + public const int DefaultDebugTimeout = 30 * 60; + public const bool DefaultBreakOnAbort = true; public int Timeout { get; init; } = DefaultTimeout; public int DebugTimeout { get; init; } = DefaultDebugTimeout; public bool BreakOnAbort { get; init; } = DefaultBreakOnAbort; diff --git a/test/vstest.ProgrammerTests/Id.cs b/test/vstest.ProgrammerTests/Id.cs index 1fe4819422..cb62ff30d0 100644 --- a/test/vstest.ProgrammerTests/Id.cs +++ b/test/vstest.ProgrammerTests/Id.cs @@ -3,7 +3,7 @@ namespace vstest.ProgrammerTests.CommandLine; -public class Id +internal class Id { private int _id; diff --git a/test/vstest.ProgrammerTests/KnownFramework.cs b/test/vstest.ProgrammerTests/KnownFramework.cs index 75d044e7bf..732dcfc628 100644 --- a/test/vstest.ProgrammerTests/KnownFramework.cs +++ b/test/vstest.ProgrammerTests/KnownFramework.cs @@ -2,11 +2,23 @@ // Licensed under the MIT license. See LICENSE file in the project root for full license information. namespace vstest.ProgrammerTests.CommandLine; + using System.Runtime.Versioning; internal static class KnownFramework { public static FrameworkName NetCore(int major, int minor = 0) => new($".NETCoreApp,Version={major}.{minor}"); - public static FrameworkName Net50 = NetCore(5); + private static FrameworkName NetFramework(int major, int minor, int patch = 0) => new($".NETFramework,Version={major}.{minor}.{patch}"); + + public static FrameworkName Netcoreapp1 = NetCore(1); + public static FrameworkName Netcoreapp2 = NetCore(2); + public static FrameworkName Netcoreapp21 = NetCore(2, 1); + public static FrameworkName Netcoreapp3 = NetCore(3); + public static FrameworkName Netcoreapp31 = NetCore(3, 1); + public static FrameworkName Net5 = NetCore(5); + public static FrameworkName Net6 = NetCore(6); + public static FrameworkName Net7 = NetCore(7); + + public static FrameworkName Net48 = NetFramework(4, 8); } diff --git a/test/vstest.ProgrammerTests/StringExtensions.cs b/test/vstest.ProgrammerTests/StringExtensions.cs index c0666063db..07d2c2aee0 100644 --- a/test/vstest.ProgrammerTests/StringExtensions.cs +++ b/test/vstest.ProgrammerTests/StringExtensions.cs @@ -3,20 +3,27 @@ namespace vstest.ProgrammerTests.CommandLine; -internal static class StringExtensions +using System.Linq; + +internal static class EnumerableExtensions { - public static string Join(this IEnumerable value, string separator) + public static string JoinBySpace(this IEnumerable value) + { + return value.JoinBy(" "); + } + + public static string JoinByComma(this IEnumerable value) { - return string.Join(separator, value); + return value.JoinBy(", "); } - public static string JoinBySpace(this IEnumerable value) + public static string JoinBy(this IEnumerable value, string delimiter) { - return string.Join(" ", value); + return string.Join(delimiter, value.Select(v => v?.ToString())); } - public static List AsList(this string value) + public static List AsList(this T value) { - return new List { value }; + return new List { value }; } } diff --git a/test/vstest.ProgrammerTests/UnitTest1.cs b/test/vstest.ProgrammerTests/UnitTest1.cs index 8d51726e18..22632a0180 100644 --- a/test/vstest.ProgrammerTests/UnitTest1.cs +++ b/test/vstest.ProgrammerTests/UnitTest1.cs @@ -9,6 +9,8 @@ namespace vstest.ProgrammerTests.CommandLine; using FluentAssertions; using FluentAssertions.Extensions; +using Intent; + using Microsoft.VisualStudio.TestPlatform.Client; using Microsoft.VisualStudio.TestPlatform.CommandLine; using Microsoft.VisualStudio.TestPlatform.CommandLine.Publisher; @@ -20,6 +22,7 @@ namespace vstest.ProgrammerTests.CommandLine; using Microsoft.VisualStudio.TestPlatform.CrossPlatEngine.TestRunAttachmentsProcessing; using Microsoft.VisualStudio.TestPlatform.ObjectModel; using Microsoft.VisualStudio.TestPlatform.ObjectModel.Client; +using Microsoft.VisualStudio.TestPlatform.ObjectModel.Logging; using vstest.ProgrammerTests.CommandLine.Fakes; using vstest.ProgrammerTests.Fakes; @@ -96,7 +99,7 @@ public async Task GivenAnMSTestAssemblyWith108Tests_WhenTestsAreRun_Then108Tests TestServiceLocator.Clear(); TestServiceLocator.Register(fakeCommunicationEndpoint.TestHostConnectionInfo.Endpoint, fakeCommunicationEndpoint); var fakeTestHostProcess = new FakeProcess(fakeErrorAggregator, @"C:\temp\testhost.exe"); - var fakeTestRuntimeProvider = new FakeTestRuntimeProvider(fakeProcessHelper, fakeTestHostProcess, fakeCommunicationEndpoint, fakeErrorAggregator); + var fakeTestRuntimeProvider = new FakeTestRuntimeProvider(fakeProcessHelper, fakeTestHostProcess, fakeFileHelper, mstest1Dll.AsList(), fakeCommunicationEndpoint, fakeErrorAggregator); var fakeTestRuntimeProviderManager = new FakeTestRuntimeProviderManager(fakeErrorAggregator); fakeTestRuntimeProviderManager.AddTestRuntimeProviders(fakeTestRuntimeProvider); var testEngine = new TestEngine(fakeTestRuntimeProviderManager, fakeProcessHelper); @@ -183,7 +186,7 @@ public async Task GivenMultipleMsTestAssembliesThatUseTheSameTargetFrameworkAndA var mstest1Dll = new FakeTestDllBuilder() .WithPath(@"X:\fake\mstest1.dll") - .WithFramework(KnownFramework.Net50) + .WithFramework(KnownFramework.Net5) .WithArchitecture(Architecture.X64) .WithTestCount(108, 10) .Build(); @@ -204,13 +207,13 @@ public async Task GivenMultipleMsTestAssembliesThatUseTheSameTargetFrameworkAndA .Build(); var mstest2Dll = new FakeTestDllBuilder() - .WithPath(@"X:\fake\mstest1.dll") - .WithFramework(KnownFramework.Net50) + .WithPath(@"X:\fake\mstest2.dll") + .WithFramework(KnownFramework.Net5) .WithArchitecture(Architecture.X64) .WithTestCount(50, 8) .Build(); - var testhost2Process = new FakeProcess(fixture.ErrorAggregator, @"X:\fake\testhost1.exe"); + var testhost2Process = new FakeProcess(fixture.ErrorAggregator, @"X:\fake\testhost2.exe"); var runTests2 = new FakeMessagesBuilder() .VersionCheck(5) @@ -225,11 +228,9 @@ public async Task GivenMultipleMsTestAssembliesThatUseTheSameTargetFrameworkAndA .WithResponses(runTests2) .Build(); - // --- - fixture.AddTestHostFixtures(testhost1, testhost2); - var testRequestManager = fixture.BuildTestRequestManager(5, 50, true); + var testRequestManager = fixture.BuildTestRequestManager(); // -- act var runConfiguration = string.Empty; @@ -246,6 +247,101 @@ public async Task GivenMultipleMsTestAssembliesThatUseTheSameTargetFrameworkAndA fixture.AssertNoErrors(); fixture.ExecutedTests.Should().HaveCount(mstest1Dll.TestCount + mstest2Dll.TestCount); } + + [Only] + public async Task GivenMultipleMsTestAssembliesThatUseDifferentTargetFrameworkAndTheSameArchitecture_WhenTestsAreRun_ThenTwoTesthostsAreStartedBothForTheSameTFM() + { + // TODO: make vstest.console not start testhosts for incompatible sources. + + // -- arrange + using var fixture = new Fixture(); + + var mstest1Dll = new FakeTestDllBuilder() + .WithPath(@"X:\fake\mstest1.dll") + .WithFramework(KnownFramework.Net5) // <--- + .WithArchitecture(Architecture.X64) + .WithTestCount(2) + .Build(); + + var testhost1Process = new FakeProcess(fixture.ErrorAggregator, @"X:\fake\testhost1.exe"); + + var runTests1 = new FakeMessagesBuilder() + .VersionCheck(5) + .ExecutionInitialize(FakeMessage.NoResponse) + .StartTestExecutionWithSources(mstest1Dll.TestResultBatches) + .SessionEnd(FakeMessage.NoResponse, _ => testhost1Process.Exit()) + .Build(); + + var testhost1 = new FakeTestHostFixtureBuilder(fixture) + .WithTestDll(mstest1Dll) + .WithProcess(testhost1Process) + .WithResponses(runTests1) + .Build(); + + // -- + + var mstest2Dll = new FakeTestDllBuilder() + .WithPath(@"X:\fake\mstest2.dll") + .WithFramework(KnownFramework.Net48) // <--- + .WithArchitecture(Architecture.X64) + .WithTestCount(1) + .Build(); + + var testhost2Process = new FakeProcess(fixture.ErrorAggregator, @"X:\fake\testhost2.exe"); + + var runTests2 = new FakeMessagesBuilder() + .VersionCheck(5) + .ExecutionInitialize(FakeMessage.NoResponse) + // .StartTestExecutionWithSources(new FakeMessage(MessageType.TestMessage, new TestMessagePayload { MessageLevel = TestMessageLevel.Error, Message = "Loading type failed." }), _ => testhost2Process.Exit()) + .StartTestExecutionWithSources(mstest2Dll.TestResultBatches) + .SessionEnd(FakeMessage.NoResponse, _ => testhost2Process.Exit()) + .Build(); + + var testhost2 = new FakeTestHostFixtureBuilder(fixture) + .WithTestDll(mstest2Dll) + .WithProcess(testhost2Process) + .WithResponses(runTests2) + .Build(); + + fixture.AddTestHostFixtures(testhost1, testhost2); + + var testRequestManager = fixture.BuildTestRequestManager(); + + mstest1Dll.FrameworkName.Should().NotBe(mstest2Dll.FrameworkName); + + // -- act + var runConfiguration = new Microsoft.VisualStudio.TestPlatform.ObjectModel.RunConfiguration { TestSessionTimeout = 40_000 }.ToXml().OuterXml; + var testRunRequestPayload = new TestRunRequestPayload + { + Sources = new List { mstest1Dll.Path, mstest2Dll.Path }, + + RunSettings = $"{runConfiguration}" + }; + + await testRequestManager.ExecuteWithAbort(tm => tm.RunTests(testRunRequestPayload, testHostLauncher: null, fixture.TestRunEventsRegistrar, fixture.ProtocolConfig)); + + // -- assert + fixture.AssertNoErrors(); + // We started both testhosts, even thought we know one of them is incompatible. + fixture.ProcessHelper.Processes.Where(p => p.Started).Should().HaveCount(2); + var startWithSources1 = testhost1.FakeCommunicationChannel.ProcessedMessages.Single(m => m.Request.MessageType == MessageType.StartTestExecutionWithSources); + var startWithSources1Text = startWithSources1.Request.Payload.Select(t => t.ToString()).JoinBySpace(); + // We sent mstest1.dll + startWithSources1Text.Should().Contain("mstest1.dll"); + // And we sent netcoreapp1.0 as the target framework + startWithSources1Text.Should().Contain(KnownFramework.Netcoreapp1.ToString()); + + var startWithSources2 = testhost2.FakeCommunicationChannel.ProcessedMessages.Single(m => m.Request.MessageType == MessageType.StartTestExecutionWithSources); + var startWithSources2Text = startWithSources2.Request.Payload.Select(t => t.ToString()).JoinBySpace(); + // We sent mstest2.dll + startWithSources2Text.Should().Contain("mstest2.dll"); + // And we sent netcoreapp1.0 as the target framework, even though it is incompatible + startWithSources2Text.Should().Contain(KnownFramework.Netcoreapp1.ToString()); + + // In reality, the dll would fail to load, and no tests would run + // in our simulation we sent tests back anyway, so we get all tests results + fixture.ExecutedTests.Should().HaveCount(mstest1Dll.TestCount + mstest2Dll.TestCount); + } } // Test and improvmement ideas: From 2f8f0d6b9ed5cd67a14efebb0279d7d6fad5af71 Mon Sep 17 00:00:00 2001 From: nohwnd Date: Fri, 18 Feb 2022 08:46:58 +0100 Subject: [PATCH 17/32] Add test for the current behavior of tfms --- playground/MSTest1/MSTest1.csproj | 2 +- playground/TestPlatform.Playground/Program.cs | 3 +- .../TestPlatform.Playground.csproj | 2 +- .../Execution/TestRunRequest.cs | 4 - .../CapturedRunSettings.cs | 9 - .../ConfigurationEntry.cs | 23 -- .../FakeMessagesBuilder.cs | 134 ------- .../FakeMessagesBuilder/FromBatches.cs | 18 - .../Fakes/EventRecord.cs | 3 +- .../Fakes/FakeAssemblyMetadataProvider.cs | 2 +- .../Fakes/FakeCommunicationChannel.cs | 338 +++++++++--------- .../Fakes/FakeCommunicationEndpoint.cs | 25 +- ...taCollectorAttachmentsProcessorsFactory.cs | 2 +- .../Fakes/FakeErrorAggregator.cs | 2 +- test/vstest.ProgrammerTests/Fakes/FakeFile.cs | 2 +- .../Fakes/FakeFileHelper.cs | 2 +- .../Fakes/FakeMetricsPublisher.cs | 2 +- .../Fakes/FakeOutput.cs | 5 +- .../Fakes/FakeProcess.cs | 2 +- .../{ => Fakes}/FakeProcessBuilder.cs | 2 +- .../Fakes/FakeProcessHelper.cs | 2 +- .../Fakes/FakeTestBatchBuilder.cs | 18 +- .../{ => Fakes}/FakeTestDllBuilder.cs | 4 +- .../Fakes/FakeTestDllFile.cs | 4 +- .../Fakes/FakeTestExtensionManager.cs | 2 +- .../{ => Fakes}/FakeTestHost.cs | 27 +- .../{ => Fakes}/FakeTestHostBuilder.cs | 17 +- .../Fakes/FakeTestHostLauncher.cs | 2 +- .../Fakes/FakeTestHostProcess.cs | 3 +- .../Fakes/FakeTestHostResponsesBuilder.cs | 138 +++++++ .../Fakes/FakeTestPlatformEventSource.cs | 2 +- .../Fakes/FakeTestRunEventsRegistrar.cs | 2 +- .../Fakes/FakeTestRuntimeProvider.cs | 2 +- .../Fakes/FakeTestRuntimeProviderManager.cs | 5 +- .../{ => Fakes}/Fixture.cs | 6 +- test/vstest.ProgrammerTests/{ => Fakes}/Id.cs | 11 +- .../Fakes/KnownFrameworkNames.cs | 30 ++ .../Fakes/KnownFrameworkStrings.cs | 31 ++ .../{ => Fakes}/OutputMessage.cs | 5 +- .../{ => Fakes}/StringExtensions.cs | 2 +- .../{ => Fakes}/TestRequestManagerHelper.cs | 4 +- test/vstest.ProgrammerTests/KnownFramework.cs | 24 -- .../RunConfiguration.cs | 9 - test/vstest.ProgrammerTests/TestDlls.cs | 9 - test/vstest.ProgrammerTests/UnitTest1.cs | 50 +-- test/vstest.ProgrammerTests/VstestConsole.cs | 35 -- 46 files changed, 482 insertions(+), 544 deletions(-) delete mode 100644 test/vstest.ProgrammerTests/CapturedRunSettings.cs delete mode 100644 test/vstest.ProgrammerTests/ConfigurationEntry.cs delete mode 100644 test/vstest.ProgrammerTests/FakeMessagesBuilder.cs delete mode 100644 test/vstest.ProgrammerTests/FakeMessagesBuilder/FromBatches.cs rename test/vstest.ProgrammerTests/{ => Fakes}/FakeProcessBuilder.cs (84%) rename test/vstest.ProgrammerTests/{ => Fakes}/FakeTestDllBuilder.cs (95%) rename test/vstest.ProgrammerTests/{ => Fakes}/FakeTestHost.cs (54%) rename test/vstest.ProgrammerTests/{ => Fakes}/FakeTestHostBuilder.cs (80%) create mode 100644 test/vstest.ProgrammerTests/Fakes/FakeTestHostResponsesBuilder.cs rename test/vstest.ProgrammerTests/{ => Fakes}/Fixture.cs (97%) rename test/vstest.ProgrammerTests/{ => Fakes}/Id.cs (51%) create mode 100644 test/vstest.ProgrammerTests/Fakes/KnownFrameworkNames.cs create mode 100644 test/vstest.ProgrammerTests/Fakes/KnownFrameworkStrings.cs rename test/vstest.ProgrammerTests/{ => Fakes}/OutputMessage.cs (84%) rename test/vstest.ProgrammerTests/{ => Fakes}/StringExtensions.cs (94%) rename test/vstest.ProgrammerTests/{ => Fakes}/TestRequestManagerHelper.cs (96%) delete mode 100644 test/vstest.ProgrammerTests/KnownFramework.cs delete mode 100644 test/vstest.ProgrammerTests/RunConfiguration.cs delete mode 100644 test/vstest.ProgrammerTests/TestDlls.cs delete mode 100644 test/vstest.ProgrammerTests/VstestConsole.cs diff --git a/playground/MSTest1/MSTest1.csproj b/playground/MSTest1/MSTest1.csproj index e952c0fa11..881b74fec7 100644 --- a/playground/MSTest1/MSTest1.csproj +++ b/playground/MSTest1/MSTest1.csproj @@ -6,7 +6,7 @@ - $(TargetFrameworks);net472 + $(TargetFrameworks);net472;netcoreapp3.1 $(TargetFrameworks);net451 false diff --git a/playground/TestPlatform.Playground/Program.cs b/playground/TestPlatform.Playground/Program.cs index 328d97e44d..4de92bd44c 100644 --- a/playground/TestPlatform.Playground/Program.cs +++ b/playground/TestPlatform.Playground/Program.cs @@ -55,7 +55,8 @@ static void Main(string[] args) "; var sources = new[] { - Path.Combine(playground, "MSTest1", "bin", "Debug", "net472", "MSTest1.dll") + Path.Combine(playground, "MSTest1", "bin", "Debug", "net472", "MSTest1.dll"), + Path.Combine(playground, "MSTest1", "bin", "Debug", "netcoreapp3.1", "MSTest1.dll"), }; var options = new TestPlatformOptions(); diff --git a/playground/TestPlatform.Playground/TestPlatform.Playground.csproj b/playground/TestPlatform.Playground/TestPlatform.Playground.csproj index d73746f16e..f121a18af7 100644 --- a/playground/TestPlatform.Playground/TestPlatform.Playground.csproj +++ b/playground/TestPlatform.Playground/TestPlatform.Playground.csproj @@ -39,7 +39,7 @@ - + diff --git a/src/Microsoft.TestPlatform.Client/Execution/TestRunRequest.cs b/src/Microsoft.TestPlatform.Client/Execution/TestRunRequest.cs index 913ba8348d..325218b656 100644 --- a/src/Microsoft.TestPlatform.Client/Execution/TestRunRequest.cs +++ b/src/Microsoft.TestPlatform.Client/Execution/TestRunRequest.cs @@ -260,10 +260,6 @@ public void Abort() } } - // REVIEW: Added this, review this change. If we call abort, the event is never set, and we end up waiting for it to complete forever. - // I think w - _runCompletionEvent.Set(); - EqtTrace.Info("TestRunRequest.Abort: Aborted."); } diff --git a/test/vstest.ProgrammerTests/CapturedRunSettings.cs b/test/vstest.ProgrammerTests/CapturedRunSettings.cs deleted file mode 100644 index d69ce2370a..0000000000 --- a/test/vstest.ProgrammerTests/CapturedRunSettings.cs +++ /dev/null @@ -1,9 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT license. See LICENSE file in the project root for full license information. - -namespace vstest.ProgrammerTests.CommandLine; - -internal class CapturedRunSettings -{ - public int MaxParallelLevel { get; internal set; } -} diff --git a/test/vstest.ProgrammerTests/ConfigurationEntry.cs b/test/vstest.ProgrammerTests/ConfigurationEntry.cs deleted file mode 100644 index 9bcaab5e07..0000000000 --- a/test/vstest.ProgrammerTests/ConfigurationEntry.cs +++ /dev/null @@ -1,23 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT license. See LICENSE file in the project root for full license information. - -namespace vstest.ProgrammerTests.CommandLine; - -internal class ConfigurationEntry -{ - public ConfigurationEntry(string name) - { - Name = name; - } - - public string Name { get; } - - public string InlinePath => $"RunConfiguration.{Name}"; - - public string FullPath => $"RunSettings.{InlinePath}"; - - public override string ToString() - { - return Name; - } -} diff --git a/test/vstest.ProgrammerTests/FakeMessagesBuilder.cs b/test/vstest.ProgrammerTests/FakeMessagesBuilder.cs deleted file mode 100644 index 5106f3cc0b..0000000000 --- a/test/vstest.ProgrammerTests/FakeMessagesBuilder.cs +++ /dev/null @@ -1,134 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT license. See LICENSE file in the project root for full license information. - -namespace vstest.ProgrammerTests.CommandLine; - -using Microsoft.VisualStudio.TestPlatform.CommunicationUtilities.ObjectModel; -using Microsoft.VisualStudio.TestPlatform.ObjectModel; -using Microsoft.VisualStudio.TestPlatform.ObjectModel.Client; - -using vstest.ProgrammerTests.Fakes; - -/// -/// Builds a list of RequestResponse pairs, with the provided values. Each method is a name of incoming message type. -/// The order in which the builder methods are called determines the order or responses. -/// -internal class FakeMessagesBuilder -{ - private readonly List> _responses = new(); - - /// - /// For VersionCheck message it responds with VersionCheck response that has the given version. - /// - /// - /// - internal FakeMessagesBuilder VersionCheck(int version) - { - AddPairWithValue(MessageType.VersionCheck, version); - return this; - } - - /// - /// For VersionCheck message it responds with the given FakeMessage. - /// - /// Message to respond with, or FakeMessage.NoResponse to not respond. - /// - internal FakeMessagesBuilder VersionCheck(FakeMessage message) - { - AddPairWithFakeMessage(MessageType.VersionCheck, message); - return this; - } - - /// - /// For VersionCheck message it does the given before action and responds with the given FakeMessage and then does the given after action. - /// Use FakeMessage.NoResponse to not respond. - /// - /// - /// - internal FakeMessagesBuilder VersionCheck(FakeMessage message, Action? beforeAction = null, Action? afterAction = null) - { - AddPairWithFakeMessage(MessageType.VersionCheck, message, beforeAction, afterAction); - return this; - } - - internal FakeMessagesBuilder ExecutionInitialize(FakeMessage message) - { - AddPairWithFakeMessage(MessageType.ExecutionInitialize, message); - return this; - } - - internal FakeMessagesBuilder StartTestExecutionWithSources(FakeMessage message, Action? beforeAction = null, Action? afterAction = null) - { - AddPairWithFakeMessage(MessageType.StartTestExecutionWithSources, message, beforeAction, afterAction); - return this; - } - - internal FakeMessagesBuilder StartTestExecutionWithSources(List> testResultBatches!!) - { - if (testResultBatches.Count == 0) - throw new InvalidOperationException("There must be at least one batch with at least one test. If you you wish to not respond with any tests, or respond with broken data, create that fake message from scratch."); - - var tests = testResultBatches; - // this will create as many test stats changes messages, as there are batches -1 - // the last batch will be sent as test run complete event - - // TODO: make the stats agree with the tests below - List changeMessages = tests.Take(tests.Count - 1).Select(batch => - new FakeMessage(MessageType.TestRunStatsChange, - new TestRunChangedEventArgs(new TestRunStatistics(new Dictionary { [TestOutcome.Passed] = batch.Count }), batch, new List()) - )).ToList(); - FakeMessage completedMessage = new FakeMessage(MessageType.ExecutionComplete, new TestRunCompletePayload - { - // TODO: make the stats agree with the tests below - TestRunCompleteArgs = new TestRunCompleteEventArgs(new TestRunStatistics(new Dictionary { [TestOutcome.Passed] = 1 }), false, false, null, new System.Collections.ObjectModel.Collection(), TimeSpan.Zero), - LastRunTests = new TestRunChangedEventArgs(new TestRunStatistics(new Dictionary { [TestOutcome.Passed] = 1 }), tests.Last(), new List()), - }); - List messages = changeMessages.Concat(new[] { completedMessage }).ToList(); - - AddPairWithMultipleFakeMessages(MessageType.StartTestExecutionWithSources, messages); - return this; - } - - - internal FakeMessagesBuilder SessionEnd(FakeMessage fakeMessage) - { - AddPairWithFakeMessage(MessageType.SessionEnd, fakeMessage); - return this; - } - - internal FakeMessagesBuilder SessionEnd(FakeMessage message, Action? beforeAction = null, Action? afterAction = null) - { - AddPairWithFakeMessage(MessageType.SessionEnd, message, beforeAction, afterAction); - return this; - } - - private void AddPairWithValue(string messageType, T value, Action? beforeAction = null, Action? afterAction = null) - { - AddPairWithFakeMessage(messageType, new FakeMessage(messageType, value), beforeAction, afterAction); - } - - private void AddPairWithFakeMessage(string messageType, FakeMessage message, Action? beforeAction = null, Action? afterAction = null) - { - AddPairWithMultipleFakeMessages(messageType, new[] { message }, beforeAction, afterAction); - } - - private void AddPairWithMultipleFakeMessages(string messageType, IEnumerable messages, Action? beforeAction = null, Action? afterAction = null) - { - // TODO: add after actions - Func> callActionAndReturnMessages = m => - { - if (beforeAction != null) - { - beforeAction(m); - } - return messages.ToList(); - }; - - _responses.Add(new RequestResponsePair(messageType, callActionAndReturnMessages)); - } - - internal List> Build() - { - return _responses; - } -} diff --git a/test/vstest.ProgrammerTests/FakeMessagesBuilder/FromBatches.cs b/test/vstest.ProgrammerTests/FakeMessagesBuilder/FromBatches.cs deleted file mode 100644 index f7780d2efb..0000000000 --- a/test/vstest.ProgrammerTests/FakeMessagesBuilder/FromBatches.cs +++ /dev/null @@ -1,18 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT license. See LICENSE file in the project root for full license information. - -namespace FakeMessagesBuilder; - -using System.Collections.Generic; - -using Microsoft.VisualStudio.TestPlatform.ObjectModel; - -internal class FromBatches -{ - private List> _testResultBatches; - - public FromBatches(List> testResultBatches) - { - _testResultBatches = testResultBatches; - } -} \ No newline at end of file diff --git a/test/vstest.ProgrammerTests/Fakes/EventRecord.cs b/test/vstest.ProgrammerTests/Fakes/EventRecord.cs index 9a8dfa91da..7cd6a9630c 100644 --- a/test/vstest.ProgrammerTests/Fakes/EventRecord.cs +++ b/test/vstest.ProgrammerTests/Fakes/EventRecord.cs @@ -3,7 +3,8 @@ #pragma warning disable IDE1006 // Naming Styles -namespace vstest.ProgrammerTests.CommandLine.Fakes; + +namespace vstest.ProgrammerTests.Fakes; internal class EventRecord { diff --git a/test/vstest.ProgrammerTests/Fakes/FakeAssemblyMetadataProvider.cs b/test/vstest.ProgrammerTests/Fakes/FakeAssemblyMetadataProvider.cs index 02f6f57bff..9cf857b788 100644 --- a/test/vstest.ProgrammerTests/Fakes/FakeAssemblyMetadataProvider.cs +++ b/test/vstest.ProgrammerTests/Fakes/FakeAssemblyMetadataProvider.cs @@ -1,7 +1,7 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT license. See LICENSE file in the project root for full license information. -namespace vstest.ProgrammerTests.CommandLine.Fakes; +namespace vstest.ProgrammerTests.Fakes; using System.Runtime.Versioning; diff --git a/test/vstest.ProgrammerTests/Fakes/FakeCommunicationChannel.cs b/test/vstest.ProgrammerTests/Fakes/FakeCommunicationChannel.cs index 0c389c25bf..472b03bb05 100644 --- a/test/vstest.ProgrammerTests/Fakes/FakeCommunicationChannel.cs +++ b/test/vstest.ProgrammerTests/Fakes/FakeCommunicationChannel.cs @@ -11,11 +11,26 @@ namespace vstest.ProgrammerTests.Fakes; using Microsoft.VisualStudio.TestPlatform.CommunicationUtilities.ObjectModel; using Microsoft.VisualStudio.TestPlatform.ObjectModel.Logging; -using vstest.ProgrammerTests.CommandLine; - -internal class FakeCommunicationChannel : ICommunicationChannel +internal abstract class FakeCommunicationChannel : ICommunicationChannel { - // The naming here is a bit confusing when this is implemented in process. + public FakeCommunicationChannel(int id) + { + Id = id; + } + + public int Id { get; } + public CancellationTokenSource CancellationTokenSource = new(); + + public BlockingCollection InQueue { get; } = new(); + public BlockingCollection OutQueue { get; } = new(); + + /// + /// True if we encountered unexpected message (e.g. unknown message, message out of order) or when we sent all our prepared responses, and there were still more requests coming. + /// + public bool Faulted { get; protected set; } + + #region ICommunicationChannel + // The naming for ICommunicationChannel is a bit confusing when this is implemented in-process. // Normally one side in one process would have one end of the communication channel, // and would use Send to pass message to another process. The other side would get notified // about new data by NotifyDataAvailable, read the data there, and send them to other consumers @@ -26,221 +41,204 @@ internal class FakeCommunicationChannel : ICommunicationChannel // them from the queue (NotifyDataAvailable is not used, because we monitor the queue directly here, instead of // in communication manager). And then we reply back to the sender, by invoking MessageReceived. - public BlockingCollection InQueue { get; } = new(); - public BlockingCollection OutQueue { get; } = new(); - public Task ProcessIncomingMessages { get; } - public Task ProcessOutgoingMessages { get; } + public event EventHandler? MessageReceived; - /// - /// True if we encountered unexpected message (e.g. unknown message, message out of order) or when we sent all our prepared responses, and there were still more requests coming. - /// - public bool Faulted { get; private set; } + public void Dispose() + { + CancellationTokenSource.Cancel(); + InQueue.CompleteAdding(); + } + + public Task NotifyDataAvailable() + { + // This is used only by communication manager. vstest.console does not use communication manager. + throw new NotImplementedException(); + } - public List> ProcessedMessages { get; } = new(); + public Task Send(string data) + { + InQueue.Add(data); + return Task.CompletedTask; + } + #endregion + + public void OnMessageReceived(object sender, MessageReceivedEventArgs eventArgs) + { + // This is still a race condition. In real code we solve this via SafeInvoke that does null check + // and catches the exception. In this code I prefer doing it this way, to see if it is fragile. + MessageReceived?.Invoke(this, eventArgs); + } +} + +internal class FakeCommunicationChannel : FakeCommunicationChannel, ICommunicationChannel +{ /// /// Queue of MessageType of the incoming request, and the response that will be sent back. /// - public Queue> NextResponses { get; } = new(); + public Queue> NextResponses { get; } = new(); public FakeErrorAggregator FakeErrorAggregator { get; } public FakeMessage? OutgoingMessage { get; private set; } - public int Id { get; } - - public CancellationTokenSource CancellationTokenSource = new(); - - public event EventHandler? MessageReceived; + public TContext? Context { get; private set; } + public List> ProcessedMessages { get; } = new(); + public Task? ProcessIncomingMessagesTask { get; private set; } + public Task? ProcessOutgoingMessagesTask { get; private set; } - public FakeCommunicationChannel(List> responses, FakeErrorAggregator fakeErrorAggregator, int id) + public FakeCommunicationChannel(List> responses, FakeErrorAggregator fakeErrorAggregator, int id) : base (id) { FakeErrorAggregator = fakeErrorAggregator; - Id = id; responses.ForEach(NextResponses.Enqueue); + } + + public void Start(TContext context) + { + Context = context; + ProcessIncomingMessagesTask = Task.Run(() => ProcessIncomingMessages(context), CancellationTokenSource.Token); + ProcessOutgoingMessagesTask = Task.Run(ProcessOutgoingMessages, CancellationTokenSource.Token); + } - ProcessIncomingMessages = Task.Run(() => + private void ProcessOutgoingMessages() + { + var token = CancellationTokenSource.Token; + while (!token.IsCancellationRequested) { - var token = CancellationTokenSource.Token; - while (!token.IsCancellationRequested) + try { - try - { - var rawMessage = InQueue.Take(token); - var requestMessage = JsonDataSerializer.Instance.DeserializeMessage(rawMessage); + // TODO: better name? this is message that we are currently trying to send + OutgoingMessage = OutQueue.Take(); + OnMessageReceived(this, new MessageReceivedEventArgs { Data = OutgoingMessage.SerializedMessage }); + OutgoingMessage = null; + } + catch (Exception ex) + { + FakeErrorAggregator.Add(ex); + } + } + } - if (Faulted) - { - // We already failed, when there are more requests coming, just save them and respond with error. We want to avoid - // a situation where server ignores our error message and responds with another request, for which we accidentally - // have the right answer in queue. - // - // E.g. We have VersionCheck, TestRunStart prepared, and server sends: VersionCheck, TestInitialize, TestRunStart. - // The first request has a valid response. The next TestInitialize does not have a valid response and errors out, - // but the server ignores it, and sends TestRunStart, which would normally have a prepared response, and lead to - // possibly overlooking the error response to TestInitialize. - // - // With this check in place we will not respond to TestRunStart with success, but with error. - // TODO: Better way to map MessageType and the payload type. - // TODO: simpler way to report error, and add it to the error aggregator - var errorMessage = $"FakeCommunicationChannel: FakeCommunicationChannel: Got message {requestMessage.MessageType}. But a message that was unexptected was received previously and the channel is now faulted. Review {nameof(ProcessedMessages)}, and {nameof(NextResponses)}."; - var exception = new Exception(errorMessage); - FakeErrorAggregator.Add(exception); - var errorResponse = new FakeMessage(MessageType.TestMessage, new TestMessagePayload { MessageLevel = TestMessageLevel.Error, Message = errorMessage }); - ProcessedMessages.Add(new RequestResponsePair(requestMessage, errorResponse)); - Faulted = true; - OutQueue.Add(errorResponse); - } + private void ProcessIncomingMessages(TContext context) + { + var token = CancellationTokenSource.Token; + while (!token.IsCancellationRequested) + { + try + { + var rawMessage = InQueue.Take(token); + var requestMessage = JsonDataSerializer.Instance.DeserializeMessage(rawMessage); - // Just peek at it so we can keep the message on the the queue in case of error. - if (!NextResponses.TryPeek(out var nextResponsePair)) - { - // If there are no more prepared responses then return protocol error. - var errorMessage = $"FakeCommunicationChannel: Got message {requestMessage.MessageType}, but no more requests were expected, because there are no more responses in {nameof(NextResponses)}."; - var exception = new Exception(errorMessage); - FakeErrorAggregator.Add(exception); - var errorResponse = new FakeMessage(MessageType.ProtocolError, new TestMessagePayload { MessageLevel = TestMessageLevel.Error, Message = errorMessage }); - ProcessedMessages.Add(new RequestResponsePair(requestMessage, errorResponse)); - Faulted = true; - OutQueue.Add(errorResponse); - } - else if (nextResponsePair.Request != requestMessage.MessageType) + if (Faulted) + { + // We already failed, when there are more requests coming, just save them and respond with error. We want to avoid + // a situation where server ignores our error message and responds with another request, for which we accidentally + // have the right answer in queue. + // + // E.g. We have VersionCheck, TestRunStart prepared, and server sends: VersionCheck, TestInitialize, TestRunStart. + // The first request has a valid response. The next TestInitialize does not have a valid response and errors out, + // but the server ignores it, and sends TestRunStart, which would normally have a prepared response, and lead to + // possibly overlooking the error response to TestInitialize. + // + // With this check in place we will not respond to TestRunStart with success, but with error. + // TODO: Better way to map MessageType and the payload type. + // TODO: simpler way to report error, and add it to the error aggregator + var errorMessage = $"FakeCommunicationChannel: FakeCommunicationChannel: Got message {requestMessage.MessageType}. But a message that was unexptected was received previously and the channel is now faulted. Review {nameof(ProcessedMessages)}, and {nameof(NextResponses)}."; + var exception = new Exception(errorMessage); + FakeErrorAggregator.Add(exception); + var errorResponse = new FakeMessage(MessageType.TestMessage, new TestMessagePayload { MessageLevel = TestMessageLevel.Error, Message = errorMessage }); + ProcessedMessages.Add(new RequestResponsePair(requestMessage, errorResponse)); + Faulted = true; + OutQueue.Add(errorResponse); + } + + // Just peek at it so we can keep the message on the the queue in case of error. + if (!NextResponses.TryPeek(out var nextResponsePair)) + { + // If there are no more prepared responses then return protocol error. + var errorMessage = $"FakeCommunicationChannel: Got message {requestMessage.MessageType}, but no more requests were expected, because there are no more responses in {nameof(NextResponses)}."; + var exception = new Exception(errorMessage); + FakeErrorAggregator.Add(exception); + var errorResponse = new FakeMessage(MessageType.ProtocolError, new TestMessagePayload { MessageLevel = TestMessageLevel.Error, Message = errorMessage }); + ProcessedMessages.Add(new RequestResponsePair(requestMessage, errorResponse)); + Faulted = true; + OutQueue.Add(errorResponse); + } + else if (nextResponsePair.Request != requestMessage.MessageType) + { + // If the incoming message does not match what we expected return protocol error. The lsat message will remain in the + // NextResponses queue. + var errorMessage = $"FakeCommunicationChannel: Excpected message {nextResponsePair.Request} but got {requestMessage.MessageType}."; + var exception = new Exception(errorMessage); + FakeErrorAggregator.Add(exception); + var errorResponse = new FakeMessage(MessageType.ProtocolError, new TestMessagePayload { MessageLevel = TestMessageLevel.Error, Message = errorMessage }); + ProcessedMessages.Add(new RequestResponsePair(requestMessage, errorResponse)); + Faulted = true; + OutQueue.Add(errorResponse); + } + else + { + var responsePair = NextResponses.Dequeue(); + if (responsePair.Debug && Debugger.IsAttached) { - // If the incoming message does not match what we expected return protocol error. The lsat message will remain in the - // NextResponses queue. - var errorMessage = $"FakeCommunicationChannel: Excpected message {nextResponsePair.Request} but got {requestMessage.MessageType}."; - var exception = new Exception(errorMessage); - FakeErrorAggregator.Add(exception); - var errorResponse = new FakeMessage(MessageType.ProtocolError, new TestMessagePayload { MessageLevel = TestMessageLevel.Error, Message = errorMessage }); - ProcessedMessages.Add(new RequestResponsePair(requestMessage, errorResponse)); - Faulted = true; - OutQueue.Add(errorResponse); + // We are about to send an interesting message + Debugger.Break(); } - else - { - var responsePair = NextResponses.Dequeue(); - if (responsePair.Debug && Debugger.IsAttached) - { - // We are about to send an interesting message - Debugger.Break(); - } - // TODO: passing the raw message in, is strange - var responses = responsePair.GetResponse(rawMessage)!; - ProcessedMessages.Add(new RequestResponsePair(requestMessage, responses)); + // TODO: passing the raw message in, is strange + responsePair.BeforeAction?.Invoke(context); + var responses = responsePair.Responses; + ProcessedMessages.Add(new RequestResponsePair(requestMessage, responses, false)); - foreach (var response in responses) + foreach (var response in responses) + { + // If we created a pair with NoResponse message, we won't send that back to the server. + if (response != FakeMessage.NoResponse) { - // If we created a pair with NoResponse message, we won't send that back to the server. - if (response != FakeMessage.NoResponse) - { - OutQueue.Add(response); - } + OutQueue.Add(response); } } - } - catch (Exception ex) - { - FakeErrorAggregator.Add(ex); + + responsePair.AfterAction?.Invoke(context); } } - - }, CancellationTokenSource.Token); - - ProcessOutgoingMessages = Task.Run(() => - { - var token = CancellationTokenSource.Token; - while (!token.IsCancellationRequested) + catch (Exception ex) { - try - { - // TODO: better name? this is message that we are currently trying to send - OutgoingMessage = OutQueue.Take(); - // This is still a race condition. In real code we solve this via SafeInvoke that does null check - // and catches the exception. In this code I prefer doing it this way, to see if it is fragile. - if (MessageReceived != null) - { - MessageReceived(this, new MessageReceivedEventArgs { Data = OutgoingMessage.SerializedMessage }); - } - OutgoingMessage = null; - } - catch (Exception ex) - { - FakeErrorAggregator.Add(ex); - } + FakeErrorAggregator.Add(ex); } - - }, CancellationTokenSource.Token); - } - - public void Dispose() - { - CancellationTokenSource.Cancel(); - InQueue.CompleteAdding(); - } - - public Task NotifyDataAvailable() - { - // This is used only by communication manager. vstest.console does not use communication manager. - throw new NotImplementedException(); - } - - public Task Send(string data) - { - InQueue.Add(data); - return Task.CompletedTask; + } } } -internal class RequestResponsePair where T : class +internal class RequestResponsePair where TRequest : class { - public RequestResponsePair(T request, U response, bool debug = false) + public RequestResponsePair(TRequest request, TResponse response, bool debug = false) { Request = request; - Responses = new List { response }; + Responses = new List { response }; Debug = debug; } - public RequestResponsePair(T request, IEnumerable responses, bool debug = false) + public RequestResponsePair(TRequest request, IEnumerable responses, bool debug = false) { Request = request; Responses = responses.ToList(); Debug = debug; } - public RequestResponsePair(T request, Func> responseFactory, bool debug = false) + public RequestResponsePair(TRequest request, IEnumerable responses, Action? beforeAction = null, Action? afterAction = null, bool debug = false) { Request = request; - ResponseFactory = responseFactory; + Responses = responses.ToList(); + BeforeAction = beforeAction; + AfterAction = afterAction; Debug = debug; } - public T Request { get; } - - // TODO: make this Expression>? ResponseFactory { get; } - public List Responses { get; private set; } + public TRequest Request { get; } - public bool Debug { get; init; } - - // TODO: Let's sleep on this and see if I understand tomorrow what I was trying to do, because this has too many usages now... - // One day later I do remember it, but I am still not convinced. But let's keep that for now. The idea here is to get either a canned - // response or a response generated based on the incoming request (e.g. version comes in that is 3, response is lower version (2)). - // both of these could be done by just executing Func, but that is not readable during debug time. Or maybe some variation on Either<> - // but that seems as a very foreign concept to common C#. - public List GetResponse(T? request = null) - { - if (ResponseFactory != null) - { - // TODO: what am I doing wrong? (Why do I need that '!' ? I assign Request in both ctors and don't have setter, is the null coalescing not - // supposed to propagate non-nullable type to the target type? - // TODO: I don't like rewriting the response here. This is yet another sign that I am possibly mixing concepts in this class. - Responses = ResponseFactory(request ?? Request!); - return Responses; - } - else - { - // TODO: split this class to two that has the same parent, so we are sure we have a response in any case. - return Responses!; - } - } + // TODO: make this Expression< so we can get some info about what this is doing when looking directly at this instance + public Action? BeforeAction { get; } + public Action? AfterAction { get; } + public List Responses { get; } + public bool Debug { get; } } /// diff --git a/test/vstest.ProgrammerTests/Fakes/FakeCommunicationEndpoint.cs b/test/vstest.ProgrammerTests/Fakes/FakeCommunicationEndpoint.cs index 5bbc0dafbf..07c53e9ae4 100644 --- a/test/vstest.ProgrammerTests/Fakes/FakeCommunicationEndpoint.cs +++ b/test/vstest.ProgrammerTests/Fakes/FakeCommunicationEndpoint.cs @@ -1,14 +1,12 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT license. See LICENSE file in the project root for full license information. -namespace vstest.ProgrammerTests.CommandLine; +namespace vstest.ProgrammerTests.Fakes; using Microsoft.VisualStudio.TestPlatform.CommunicationUtilities.Interfaces; using Microsoft.VisualStudio.TestPlatform.ObjectModel; using Microsoft.VisualStudio.TestPlatform.Utilities; -using vstest.ProgrammerTests.Fakes; - internal class FakeCommunicationEndpoint : ICommunicationEndPoint { private bool _stopped; @@ -29,6 +27,19 @@ public FakeCommunicationEndpoint(FakeCommunicationChannel fakeCommunicationChann public FakeCommunicationChannel Channel { get; } public TestHostConnectionInfo TestHostConnectionInfo { get; } + /// + /// Notify the caller that we disconnected, this happens if process exits unexpectedly and leads to abort flow. + /// In success case use Stop instead, to just "close" the channel, because the other side already disconnected from us + /// and told us to tear down. + /// + public void Disconnect() + { + Disconnected?.Invoke(this, new DisconnectedEventArgs()); + _stopped = true; + } + + #region ICommunicationEndPoint + public event EventHandler? Connected; public event EventHandler? Disconnected; @@ -55,12 +66,6 @@ public string Start(string endPoint) return endPoint; } - public void Abort() - { - Disconnected?.Invoke(this, new DisconnectedEventArgs()); - _stopped = true; - } - public void Stop() { if (!_stopped) @@ -69,4 +74,6 @@ public void Stop() _stopped = true; } } + + #endregion } diff --git a/test/vstest.ProgrammerTests/Fakes/FakeDataCollectorAttachmentsProcessorsFactory.cs b/test/vstest.ProgrammerTests/Fakes/FakeDataCollectorAttachmentsProcessorsFactory.cs index ffcf56fa4a..e6a83a2455 100644 --- a/test/vstest.ProgrammerTests/Fakes/FakeDataCollectorAttachmentsProcessorsFactory.cs +++ b/test/vstest.ProgrammerTests/Fakes/FakeDataCollectorAttachmentsProcessorsFactory.cs @@ -6,7 +6,7 @@ using Microsoft.VisualStudio.TestPlatform.ObjectModel.Logging; #pragma warning disable IDE1006 // Naming Styles -namespace vstest.ProgrammerTests.CommandLine.Fakes; +namespace vstest.ProgrammerTests.Fakes; internal class FakeDataCollectorAttachmentsProcessorsFactory : IDataCollectorAttachmentsProcessorsFactory { diff --git a/test/vstest.ProgrammerTests/Fakes/FakeErrorAggregator.cs b/test/vstest.ProgrammerTests/Fakes/FakeErrorAggregator.cs index db5f70b4f3..25d04fc094 100644 --- a/test/vstest.ProgrammerTests/Fakes/FakeErrorAggregator.cs +++ b/test/vstest.ProgrammerTests/Fakes/FakeErrorAggregator.cs @@ -1,7 +1,7 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT license. See LICENSE file in the project root for full license information. -namespace vstest.ProgrammerTests.CommandLine; +namespace vstest.ProgrammerTests.Fakes; internal class FakeErrorAggregator { diff --git a/test/vstest.ProgrammerTests/Fakes/FakeFile.cs b/test/vstest.ProgrammerTests/Fakes/FakeFile.cs index 4df355eeb1..e9764eab4b 100644 --- a/test/vstest.ProgrammerTests/Fakes/FakeFile.cs +++ b/test/vstest.ProgrammerTests/Fakes/FakeFile.cs @@ -1,7 +1,7 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT license. See LICENSE file in the project root for full license information. -namespace vstest.ProgrammerTests.CommandLine.Fakes; +namespace vstest.ProgrammerTests.Fakes; internal class FakeFile { diff --git a/test/vstest.ProgrammerTests/Fakes/FakeFileHelper.cs b/test/vstest.ProgrammerTests/Fakes/FakeFileHelper.cs index 221bba176d..9a7cce020e 100644 --- a/test/vstest.ProgrammerTests/Fakes/FakeFileHelper.cs +++ b/test/vstest.ProgrammerTests/Fakes/FakeFileHelper.cs @@ -1,7 +1,7 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT license. See LICENSE file in the project root for full license information. -namespace vstest.ProgrammerTests.CommandLine.Fakes; +namespace vstest.ProgrammerTests.Fakes; using Microsoft.VisualStudio.TestPlatform.Utilities.Helpers.Interfaces; diff --git a/test/vstest.ProgrammerTests/Fakes/FakeMetricsPublisher.cs b/test/vstest.ProgrammerTests/Fakes/FakeMetricsPublisher.cs index bf86207444..acdec54735 100644 --- a/test/vstest.ProgrammerTests/Fakes/FakeMetricsPublisher.cs +++ b/test/vstest.ProgrammerTests/Fakes/FakeMetricsPublisher.cs @@ -4,7 +4,7 @@ using Microsoft.VisualStudio.TestPlatform.CommandLine.Publisher; #pragma warning disable IDE1006 // Naming Styles -namespace vstest.ProgrammerTests.CommandLine.Fakes; +namespace vstest.ProgrammerTests.Fakes; internal class FakeMetricsPublisher : IMetricsPublisher { diff --git a/test/vstest.ProgrammerTests/Fakes/FakeOutput.cs b/test/vstest.ProgrammerTests/Fakes/FakeOutput.cs index 60cc095311..9e254330fc 100644 --- a/test/vstest.ProgrammerTests/Fakes/FakeOutput.cs +++ b/test/vstest.ProgrammerTests/Fakes/FakeOutput.cs @@ -1,13 +1,12 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT license. See LICENSE file in the project root for full license information. +namespace vstest.ProgrammerTests.Fakes; + using System.Text; using Microsoft.VisualStudio.TestPlatform.Utilities; -#pragma warning disable IDE1006 // Naming Styles -namespace vstest.ProgrammerTests.CommandLine.Fakes; - internal class FakeOutput : IOutput { public FakeOutput() diff --git a/test/vstest.ProgrammerTests/Fakes/FakeProcess.cs b/test/vstest.ProgrammerTests/Fakes/FakeProcess.cs index d24a705d94..3f1eb2ea96 100644 --- a/test/vstest.ProgrammerTests/Fakes/FakeProcess.cs +++ b/test/vstest.ProgrammerTests/Fakes/FakeProcess.cs @@ -1,7 +1,7 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT license. See LICENSE file in the project root for full license information. -namespace vstest.ProgrammerTests.CommandLine.Fakes; +namespace vstest.ProgrammerTests.Fakes; using Microsoft.VisualStudio.TestPlatform.ObjectModel; using Microsoft.VisualStudio.TestPlatform.PlatformAbstractions; diff --git a/test/vstest.ProgrammerTests/FakeProcessBuilder.cs b/test/vstest.ProgrammerTests/Fakes/FakeProcessBuilder.cs similarity index 84% rename from test/vstest.ProgrammerTests/FakeProcessBuilder.cs rename to test/vstest.ProgrammerTests/Fakes/FakeProcessBuilder.cs index 6bfd4efd3f..4780e5bed3 100644 --- a/test/vstest.ProgrammerTests/FakeProcessBuilder.cs +++ b/test/vstest.ProgrammerTests/Fakes/FakeProcessBuilder.cs @@ -1,7 +1,7 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT license. See LICENSE file in the project root for full license information. -namespace vstest.ProgrammerTests.CommandLine; +namespace vstest.ProgrammerTests.Fakes; internal class FakeProcessBuilder { diff --git a/test/vstest.ProgrammerTests/Fakes/FakeProcessHelper.cs b/test/vstest.ProgrammerTests/Fakes/FakeProcessHelper.cs index cb8797b72f..8ae7432228 100644 --- a/test/vstest.ProgrammerTests/Fakes/FakeProcessHelper.cs +++ b/test/vstest.ProgrammerTests/Fakes/FakeProcessHelper.cs @@ -1,7 +1,7 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT license. See LICENSE file in the project root for full license information. -namespace vstest.ProgrammerTests.CommandLine.Fakes; +namespace vstest.ProgrammerTests.Fakes; using Microsoft.VisualStudio.TestPlatform.PlatformAbstractions; using Microsoft.VisualStudio.TestPlatform.PlatformAbstractions.Interfaces; diff --git a/test/vstest.ProgrammerTests/Fakes/FakeTestBatchBuilder.cs b/test/vstest.ProgrammerTests/Fakes/FakeTestBatchBuilder.cs index 020b00c790..f8f7e21127 100644 --- a/test/vstest.ProgrammerTests/Fakes/FakeTestBatchBuilder.cs +++ b/test/vstest.ProgrammerTests/Fakes/FakeTestBatchBuilder.cs @@ -1,9 +1,7 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT license. See LICENSE file in the project root for full license information. -namespace vstest.ProgrammerTests.CommandLine; - -using System.Runtime.CompilerServices; +namespace vstest.ProgrammerTests.Fakes; using Microsoft.VisualStudio.TestPlatform.ObjectModel; @@ -12,6 +10,7 @@ internal class FakeTestBatchBuilder public int TotalCount { get; private set; } public TimeSpan Duration { get; private set; } public int BatchSize { get; private set; } + public static List> Empty => new(); public FakeTestBatchBuilder() { @@ -48,17 +47,18 @@ internal FakeTestBatchBuilder WithBatchSize(int batchSize) internal List> Build() { - if (TotalCount == 0 || BatchSize == 0) - throw new InvalidOperationException("There must be at least one batch with at least one test."); + if (BatchSize == 0 && TotalCount != 0) + throw new InvalidOperationException("Batch size cannot be 0, unless TotalCount is also 0. Splitting non-zero amount of tests into 0 sized batches does not make sense."); + + if (TotalCount == 0) + return Empty; var numberOfBatches = Math.DivRem(TotalCount, BatchSize, out int remainder); // TODO: Add adapter uri, and dll name // TODO: set duration - var batches = - Enumerable.Range(0, numberOfBatches) - .Select(batchNumber => - Enumerable.Range(0, BatchSize) + var batches = Enumerable.Range(0, numberOfBatches) + .Select(batchNumber => Enumerable.Range(0, BatchSize) .Select((index) => new TestResult(new TestCase($"Test{batchNumber}-{index}", new Uri("some://uri"), "DummySourceFileName"))).ToList()).ToList(); if (remainder > 0) diff --git a/test/vstest.ProgrammerTests/FakeTestDllBuilder.cs b/test/vstest.ProgrammerTests/Fakes/FakeTestDllBuilder.cs similarity index 95% rename from test/vstest.ProgrammerTests/FakeTestDllBuilder.cs rename to test/vstest.ProgrammerTests/Fakes/FakeTestDllBuilder.cs index df5149fb51..036869a762 100644 --- a/test/vstest.ProgrammerTests/FakeTestDllBuilder.cs +++ b/test/vstest.ProgrammerTests/Fakes/FakeTestDllBuilder.cs @@ -1,7 +1,7 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT license. See LICENSE file in the project root for full license information. -namespace vstest.ProgrammerTests.CommandLine; +namespace vstest.ProgrammerTests.Fakes; using System; using System.Runtime.Versioning; @@ -11,7 +11,7 @@ namespace vstest.ProgrammerTests.CommandLine; internal class FakeTestDllBuilder { private string _path = @$"X:\fake\mstest_{Guid.NewGuid()}.dll"; - private FrameworkName _framework = KnownFramework.Net5; + private FrameworkName _framework = KnownFrameworkNames.Net5; private Architecture _architecture = Architecture.X64; private List>? _testBatches; diff --git a/test/vstest.ProgrammerTests/Fakes/FakeTestDllFile.cs b/test/vstest.ProgrammerTests/Fakes/FakeTestDllFile.cs index 16301853b3..1239292cd7 100644 --- a/test/vstest.ProgrammerTests/Fakes/FakeTestDllFile.cs +++ b/test/vstest.ProgrammerTests/Fakes/FakeTestDllFile.cs @@ -1,14 +1,12 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT license. See LICENSE file in the project root for full license information. -namespace vstest.ProgrammerTests.CommandLine; +namespace vstest.ProgrammerTests.Fakes; using System.Runtime.Versioning; using Microsoft.VisualStudio.TestPlatform.ObjectModel; -using vstest.ProgrammerTests.CommandLine.Fakes; - internal class FakeTestDllFile : FakeFile { public FrameworkName FrameworkName { get; } diff --git a/test/vstest.ProgrammerTests/Fakes/FakeTestExtensionManager.cs b/test/vstest.ProgrammerTests/Fakes/FakeTestExtensionManager.cs index b503278332..bddc6fe679 100644 --- a/test/vstest.ProgrammerTests/Fakes/FakeTestExtensionManager.cs +++ b/test/vstest.ProgrammerTests/Fakes/FakeTestExtensionManager.cs @@ -4,7 +4,7 @@ using Microsoft.VisualStudio.TestPlatform.ObjectModel.Engine; #pragma warning disable IDE1006 // Naming Styles -namespace vstest.ProgrammerTests.CommandLine.Fakes; +namespace vstest.ProgrammerTests.Fakes; internal class FakeTestExtensionManager : ITestExtensionManager { diff --git a/test/vstest.ProgrammerTests/FakeTestHost.cs b/test/vstest.ProgrammerTests/Fakes/FakeTestHost.cs similarity index 54% rename from test/vstest.ProgrammerTests/FakeTestHost.cs rename to test/vstest.ProgrammerTests/Fakes/FakeTestHost.cs index e054d50b5e..ea425831b4 100644 --- a/test/vstest.ProgrammerTests/FakeTestHost.cs +++ b/test/vstest.ProgrammerTests/Fakes/FakeTestHost.cs @@ -1,33 +1,25 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT license. See LICENSE file in the project root for full license information. -namespace vstest.ProgrammerTests.CommandLine; - -using vstest.ProgrammerTests.CommandLine.Fakes; -using vstest.ProgrammerTests.Fakes; - -internal class FakeTestFixtureHost +namespace vstest.ProgrammerTests.Fakes; +internal class FakeTestHostFixture { - private readonly Fixture _fixture; - public int Id { get; } public List Dlls { get; } public FakeTestRuntimeProvider FakeTestRuntimeProvider { get; } public FakeCommunicationEndpoint FakeCommunicationEndpoint { get; } - public FakeCommunicationChannel FakeCommunicationChannel { get; } - public List> Responses { get; } + public FakeCommunicationChannel FakeCommunicationChannel { get; } + public List> Responses { get; } public FakeProcess Process { get; internal set; } - public FakeTestFixtureHost( - Fixture fixture, + public FakeTestHostFixture( int id, List dlls, FakeTestRuntimeProvider fakeTestRuntimeProvider, FakeCommunicationEndpoint fakeCommunicationEndpoint, - FakeCommunicationChannel fakeCommunicationChannel, + FakeCommunicationChannel fakeCommunicationChannel, FakeProcess process, - List> responses) + List> responses) { - _fixture = fixture; Id = id; Dlls = dlls; FakeTestRuntimeProvider = fakeTestRuntimeProvider; @@ -35,5 +27,10 @@ public FakeTestFixtureHost( FakeCommunicationChannel = fakeCommunicationChannel; Process = process; Responses = responses; + + // The channel will pass back this whole fixture as context for every processed request so we can + // refer back to any part of testhost in message responses. E.g. to abort the channel, or exit + // testhost before or after answering. + fakeCommunicationChannel.Start(this); } } diff --git a/test/vstest.ProgrammerTests/FakeTestHostBuilder.cs b/test/vstest.ProgrammerTests/Fakes/FakeTestHostBuilder.cs similarity index 80% rename from test/vstest.ProgrammerTests/FakeTestHostBuilder.cs rename to test/vstest.ProgrammerTests/Fakes/FakeTestHostBuilder.cs index 552c819dd5..33a9dfc201 100644 --- a/test/vstest.ProgrammerTests/FakeTestHostBuilder.cs +++ b/test/vstest.ProgrammerTests/Fakes/FakeTestHostBuilder.cs @@ -1,28 +1,25 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT license. See LICENSE file in the project root for full license information. -namespace vstest.ProgrammerTests.CommandLine; +namespace vstest.ProgrammerTests.Fakes; using System; using Microsoft.VisualStudio.TestPlatform.CommunicationUtilities.Interfaces; using Microsoft.VisualStudio.TestPlatform.ObjectModel; -using vstest.ProgrammerTests.CommandLine.Fakes; -using vstest.ProgrammerTests.Fakes; - internal class FakeTestHostFixtureBuilder { // This will be also used as a port number, don't start from 0 // it skips some paths in the real code, because port 0 has special meaning. - private static readonly Id Id = new(1000); + private static readonly SequentialId Id = new(1000); private readonly Fixture _fixture; // TODO: this would correctly be any test holding container, but let's not get ahead of myself. private readonly List _dlls = new(); private FakeProcess? _process; - private List>? _responses; + private List>? _responses; public FakeTestHostFixtureBuilder(Fixture fixture) { @@ -35,7 +32,7 @@ internal FakeTestHostFixtureBuilder WithTestDll(FakeTestDllFile dll) return this; } - internal FakeTestFixtureHost Build() + internal FakeTestHostFixture Build() { if (_responses == null) @@ -45,7 +42,7 @@ internal FakeTestFixtureHost Build() throw new InvalidOperationException("Add some process to the testhost by using WithProcess."); var id = Id.Next(); - var fakeCommunicationChannel = new FakeCommunicationChannel(_responses, _fixture.ErrorAggregator, id); + var fakeCommunicationChannel = new FakeCommunicationChannel(_responses, _fixture.ErrorAggregator, id); var fakeCommunicationEndpoint = new FakeCommunicationEndpoint(fakeCommunicationChannel, _fixture.ErrorAggregator); var fakeTestRuntimeProvider = new FakeTestRuntimeProvider(_fixture.ProcessHelper, _process, _fixture.FileHelper, _dlls, fakeCommunicationEndpoint, _fixture.ErrorAggregator); @@ -53,7 +50,7 @@ internal FakeTestFixtureHost Build() // testhost connection info, and is used as port in 127.0.0.1:, address so we can lookup the correct channel. TestServiceLocator.Register(fakeCommunicationEndpoint.TestHostConnectionInfo.Endpoint, fakeCommunicationEndpoint); - return new FakeTestFixtureHost(_fixture, id, _dlls, fakeTestRuntimeProvider, fakeCommunicationEndpoint, fakeCommunicationChannel, _process, _responses); + return new FakeTestHostFixture(id, _dlls, fakeTestRuntimeProvider, fakeCommunicationEndpoint, fakeCommunicationChannel, _process, _responses); } internal FakeTestHostFixtureBuilder WithProcess(FakeProcess process) @@ -62,7 +59,7 @@ internal FakeTestHostFixtureBuilder WithProcess(FakeProcess process) return this; } - internal FakeTestHostFixtureBuilder WithResponses(List> responses) + internal FakeTestHostFixtureBuilder WithResponses(List> responses) { _responses = responses; return this; diff --git a/test/vstest.ProgrammerTests/Fakes/FakeTestHostLauncher.cs b/test/vstest.ProgrammerTests/Fakes/FakeTestHostLauncher.cs index b7e318a3c0..120c0d049d 100644 --- a/test/vstest.ProgrammerTests/Fakes/FakeTestHostLauncher.cs +++ b/test/vstest.ProgrammerTests/Fakes/FakeTestHostLauncher.cs @@ -5,7 +5,7 @@ using Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.Interfaces; #pragma warning disable IDE1006 // Naming Styles -namespace vstest.ProgrammerTests.CommandLine.Fakes; +namespace vstest.ProgrammerTests.Fakes; internal class FakeTestHostLauncher : ITestHostLauncher { diff --git a/test/vstest.ProgrammerTests/Fakes/FakeTestHostProcess.cs b/test/vstest.ProgrammerTests/Fakes/FakeTestHostProcess.cs index f78ef8ae11..120170351d 100644 --- a/test/vstest.ProgrammerTests/Fakes/FakeTestHostProcess.cs +++ b/test/vstest.ProgrammerTests/Fakes/FakeTestHostProcess.cs @@ -3,7 +3,8 @@ #pragma warning disable IDE1006 // Naming Styles -namespace vstest.ProgrammerTests.CommandLine.Fakes; + +namespace vstest.ProgrammerTests.Fakes; // TODO: was used in first test but is not correct design //internal class FakeTestHostProcess : FakeProcess diff --git a/test/vstest.ProgrammerTests/Fakes/FakeTestHostResponsesBuilder.cs b/test/vstest.ProgrammerTests/Fakes/FakeTestHostResponsesBuilder.cs new file mode 100644 index 0000000000..056d0e31bc --- /dev/null +++ b/test/vstest.ProgrammerTests/Fakes/FakeTestHostResponsesBuilder.cs @@ -0,0 +1,138 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +namespace vstest.ProgrammerTests.Fakes; + +using Microsoft.VisualStudio.TestPlatform.CommunicationUtilities.ObjectModel; +using Microsoft.VisualStudio.TestPlatform.ObjectModel; +using Microsoft.VisualStudio.TestPlatform.ObjectModel.Client; + +/// +/// Builds a list of RequestResponse pairs, with the provided values. Each method is a name of incoming message type. +/// The order in which the builder methods are called determines the order or responses. +/// +internal class FakeTestHostResponsesBuilder +{ + private readonly List> _responses = new(); + + /// + /// For VersionCheck message it responds with VersionCheck response that has the given version. + /// + /// + /// + internal FakeTestHostResponsesBuilder VersionCheck(int version) + { + AddPairWithValue(MessageType.VersionCheck, version); + return this; + } + + /// + /// For VersionCheck message it responds with the given FakeMessage. + /// + /// Message to respond with, or FakeMessage.NoResponse to not respond. + /// + internal FakeTestHostResponsesBuilder VersionCheck(FakeMessage message) + { + AddPairWithFakeMessage(MessageType.VersionCheck, message); + return this; + } + + /// + /// For VersionCheck message it does the given before action and responds with the given FakeMessage and then does the given after action. + /// Use FakeMessage.NoResponse to not respond. + /// + /// + /// + internal FakeTestHostResponsesBuilder VersionCheck(FakeMessage message, Action? beforeAction = null, Action? afterAction = null) + { + AddPairWithFakeMessage(MessageType.VersionCheck, message, beforeAction, afterAction); + return this; + } + + internal FakeTestHostResponsesBuilder ExecutionInitialize(FakeMessage message) + { + AddPairWithFakeMessage(MessageType.ExecutionInitialize, message); + return this; + } + + internal FakeTestHostResponsesBuilder StartTestExecutionWithSources(FakeMessage message, Action? beforeAction = null, Action? afterAction = null) + { + AddPairWithFakeMessage(MessageType.StartTestExecutionWithSources, message, beforeAction, afterAction); + return this; + } + + internal FakeTestHostResponsesBuilder StartTestExecutionWithSources(List> testResultBatches!!) + { + var tests = testResultBatches; + + List messages; + if (testResultBatches.Count == 0) + { + // this will create as many test stats changes messages, as there are batches -1 + // the last batch will be sent as test run complete event + + // TODO: make the stats agree with the tests below + List changeMessages = tests.Take(tests.Count - 1).Select(batch => + new FakeMessage(MessageType.TestRunStatsChange, + new TestRunChangedEventArgs(new TestRunStatistics(new Dictionary { [TestOutcome.Passed] = batch.Count }), batch, new List()) + )).ToList(); + + // TODO: This is finicky because the statistics processor expects the dictionary to not be null + FakeMessage completedMessage = new FakeMessage(MessageType.ExecutionComplete, new TestRunCompletePayload + { + // TODO: make the stats agree with the tests below + TestRunCompleteArgs = new TestRunCompleteEventArgs(new TestRunStatistics(new Dictionary { [TestOutcome.Passed] = 1 }), false, false, null, new System.Collections.ObjectModel.Collection(), TimeSpan.Zero), + LastRunTests = new TestRunChangedEventArgs(new TestRunStatistics(new Dictionary { [TestOutcome.Passed] = 1 }), tests.Last(), new List()), + }); + messages = changeMessages.Concat(new[] { completedMessage }).ToList(); + } + else + { + var completedMessage = new FakeMessage(MessageType.ExecutionComplete, new TestRunCompletePayload + { + // TODO: make the stats agree with the tests below + TestRunCompleteArgs = new TestRunCompleteEventArgs(new TestRunStatistics(new Dictionary { [TestOutcome.Passed] = 0 }), false, false, null, new System.Collections.ObjectModel.Collection(), TimeSpan.Zero), + LastRunTests = new TestRunChangedEventArgs(new TestRunStatistics(new Dictionary { [TestOutcome.Passed] = 0 }), tests.Last(), new List()), + }); + + messages = completedMessage.AsList(); + } + + + AddPairWithMultipleFakeMessages(MessageType.StartTestExecutionWithSources, messages); + return this; + } + + + internal FakeTestHostResponsesBuilder SessionEnd(FakeMessage fakeMessage) + { + AddPairWithFakeMessage(MessageType.SessionEnd, fakeMessage); + return this; + } + + internal FakeTestHostResponsesBuilder SessionEnd(FakeMessage message, Action? beforeAction = null, Action? afterAction = null) + { + AddPairWithFakeMessage(MessageType.SessionEnd, message, beforeAction, afterAction); + return this; + } + + private void AddPairWithValue(string messageType, T value, Action? beforeAction = null, Action? afterAction = null) + { + AddPairWithFakeMessage(messageType, new FakeMessage(messageType, value), beforeAction, afterAction); + } + + private void AddPairWithFakeMessage(string messageType, FakeMessage message, Action? beforeAction = null, Action? afterAction = null) + { + AddPairWithMultipleFakeMessages(messageType, new[] { message }, beforeAction, afterAction); + } + + private void AddPairWithMultipleFakeMessages(string messageType, IEnumerable messages, Action? beforeAction = null, Action? afterAction = null) + { + _responses.Add(new RequestResponsePair(messageType, messages, beforeAction, afterAction)); + } + + internal List> Build() + { + return _responses; + } +} diff --git a/test/vstest.ProgrammerTests/Fakes/FakeTestPlatformEventSource.cs b/test/vstest.ProgrammerTests/Fakes/FakeTestPlatformEventSource.cs index 02ee19f378..ed4b085149 100644 --- a/test/vstest.ProgrammerTests/Fakes/FakeTestPlatformEventSource.cs +++ b/test/vstest.ProgrammerTests/Fakes/FakeTestPlatformEventSource.cs @@ -4,7 +4,7 @@ using Microsoft.VisualStudio.TestPlatform.CoreUtilities.Tracing.Interfaces; #pragma warning disable IDE1006 // Naming Styles -namespace vstest.ProgrammerTests.CommandLine.Fakes; +namespace vstest.ProgrammerTests.Fakes; internal class FakeTestPlatformEventSource : ITestPlatformEventSource { diff --git a/test/vstest.ProgrammerTests/Fakes/FakeTestRunEventsRegistrar.cs b/test/vstest.ProgrammerTests/Fakes/FakeTestRunEventsRegistrar.cs index ddd8c5d57a..b9b24c1078 100644 --- a/test/vstest.ProgrammerTests/Fakes/FakeTestRunEventsRegistrar.cs +++ b/test/vstest.ProgrammerTests/Fakes/FakeTestRunEventsRegistrar.cs @@ -6,7 +6,7 @@ using Microsoft.VisualStudio.TestPlatform.ObjectModel.Logging; #pragma warning disable IDE1006 // Naming Styles -namespace vstest.ProgrammerTests.CommandLine.Fakes; +namespace vstest.ProgrammerTests.Fakes; internal class FakeTestRunEventsRegistrar : ITestRunEventsRegistrar { diff --git a/test/vstest.ProgrammerTests/Fakes/FakeTestRuntimeProvider.cs b/test/vstest.ProgrammerTests/Fakes/FakeTestRuntimeProvider.cs index 3243c64b92..4365e31a08 100644 --- a/test/vstest.ProgrammerTests/Fakes/FakeTestRuntimeProvider.cs +++ b/test/vstest.ProgrammerTests/Fakes/FakeTestRuntimeProvider.cs @@ -1,7 +1,7 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT license. See LICENSE file in the project root for full license information. -namespace vstest.ProgrammerTests.CommandLine.Fakes; +namespace vstest.ProgrammerTests.Fakes; using Microsoft.VisualStudio.TestPlatform.ObjectModel; using Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.Interfaces; diff --git a/test/vstest.ProgrammerTests/Fakes/FakeTestRuntimeProviderManager.cs b/test/vstest.ProgrammerTests/Fakes/FakeTestRuntimeProviderManager.cs index 45d7360b08..10f0257484 100644 --- a/test/vstest.ProgrammerTests/Fakes/FakeTestRuntimeProviderManager.cs +++ b/test/vstest.ProgrammerTests/Fakes/FakeTestRuntimeProviderManager.cs @@ -2,7 +2,8 @@ // Licensed under the MIT license. See LICENSE file in the project root for full license information. #pragma warning disable IDE1006 // Naming Styles -namespace vstest.ProgrammerTests.CommandLine.Fakes; + +namespace vstest.ProgrammerTests.Fakes; using System.Collections.Concurrent; @@ -34,7 +35,7 @@ public void AddTestRuntimeProviders(params FakeTestRuntimeProvider[] runtimeProv public ITestRuntimeProvider GetTestHostManagerByRunConfiguration(string runConfiguration) { - + if (!TestRuntimeProviders.TryDequeue(out var next)) { diff --git a/test/vstest.ProgrammerTests/Fixture.cs b/test/vstest.ProgrammerTests/Fakes/Fixture.cs similarity index 97% rename from test/vstest.ProgrammerTests/Fixture.cs rename to test/vstest.ProgrammerTests/Fakes/Fixture.cs index 7e74b9a80b..5fd7028b4e 100644 --- a/test/vstest.ProgrammerTests/Fixture.cs +++ b/test/vstest.ProgrammerTests/Fakes/Fixture.cs @@ -1,7 +1,7 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT license. See LICENSE file in the project root for full license information. -namespace vstest.ProgrammerTests.CommandLine; +namespace vstest.ProgrammerTests.Fakes; using FluentAssertions; @@ -15,8 +15,6 @@ namespace vstest.ProgrammerTests.CommandLine; using Microsoft.VisualStudio.TestPlatform.ObjectModel; using Microsoft.VisualStudio.TestPlatform.ObjectModel.Client; -using vstest.ProgrammerTests.CommandLine.Fakes; - internal class Fixture : IDisposable { public FakeErrorAggregator ErrorAggregator { get; } = new(); @@ -55,7 +53,7 @@ public void Dispose() } - internal void AddTestHostFixtures(params FakeTestFixtureHost[] testhosts) + internal void AddTestHostFixtures(params FakeTestHostFixture[] testhosts) { var providers = testhosts.Select(t => t.FakeTestRuntimeProvider).ToArray(); TestRuntimeProviderManager.AddTestRuntimeProviders(providers); diff --git a/test/vstest.ProgrammerTests/Id.cs b/test/vstest.ProgrammerTests/Fakes/Id.cs similarity index 51% rename from test/vstest.ProgrammerTests/Id.cs rename to test/vstest.ProgrammerTests/Fakes/Id.cs index cb62ff30d0..815245110b 100644 --- a/test/vstest.ProgrammerTests/Id.cs +++ b/test/vstest.ProgrammerTests/Fakes/Id.cs @@ -1,17 +1,20 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT license. See LICENSE file in the project root for full license information. -namespace vstest.ProgrammerTests.CommandLine; +namespace vstest.ProgrammerTests.Fakes; -internal class Id +/// +/// A sequential Id that starts from 0 or a given number. Put this in a static field in your class, and call Next to get the next Id. +/// +internal class SequentialId { private int _id; - public Id () : this(0) + public SequentialId() : this(0) { } - public Id (int firstId) + public SequentialId(int firstId) { _id = firstId; } diff --git a/test/vstest.ProgrammerTests/Fakes/KnownFrameworkNames.cs b/test/vstest.ProgrammerTests/Fakes/KnownFrameworkNames.cs new file mode 100644 index 0000000000..bdb800d888 --- /dev/null +++ b/test/vstest.ProgrammerTests/Fakes/KnownFrameworkNames.cs @@ -0,0 +1,30 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +namespace vstest.ProgrammerTests.Fakes; + +using System.Runtime.Versioning; + +internal static class KnownFrameworkNames +{ + public static FrameworkName Netcoreapp1 = new(KnownFrameworkStrings.Netcoreapp1); + public static FrameworkName Netcoreapp2 = new(KnownFrameworkStrings.Netcoreapp2); + public static FrameworkName Netcoreapp21 = new(KnownFrameworkStrings.Netcoreapp21); + public static FrameworkName Netcoreapp3 = new(KnownFrameworkStrings.Netcoreapp3); + public static FrameworkName Netcoreapp31 = new(KnownFrameworkStrings.Netcoreapp31); + public static FrameworkName Net5 = new(KnownFrameworkStrings.Net5); + public static FrameworkName Net6 = new(KnownFrameworkStrings.Net6); + public static FrameworkName Net7 = new(KnownFrameworkStrings.Net7); + + public static FrameworkName Net4 = new(KnownFrameworkStrings.Net4); + public static FrameworkName Net45 = new(KnownFrameworkStrings.Net45); + public static FrameworkName Net451 = new(KnownFrameworkStrings.Net451); + public static FrameworkName Net452 = new(KnownFrameworkStrings.Net452); + public static FrameworkName Net46 = new(KnownFrameworkStrings.Net46); + public static FrameworkName Net461 = new(KnownFrameworkStrings.Net461); + public static FrameworkName Net462 = new(KnownFrameworkStrings.Net462); + public static FrameworkName Net47 = new(KnownFrameworkStrings.Net47); + public static FrameworkName Net471 = new(KnownFrameworkStrings.Net471); + public static FrameworkName Net472 = new(KnownFrameworkStrings.Net472); + public static FrameworkName Net48 = new(KnownFrameworkStrings.Net48); +} diff --git a/test/vstest.ProgrammerTests/Fakes/KnownFrameworkStrings.cs b/test/vstest.ProgrammerTests/Fakes/KnownFrameworkStrings.cs new file mode 100644 index 0000000000..9605a8ff59 --- /dev/null +++ b/test/vstest.ProgrammerTests/Fakes/KnownFrameworkStrings.cs @@ -0,0 +1,31 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +namespace vstest.ProgrammerTests.Fakes; + +internal static class KnownFrameworkStrings +{ + public static string NetCore(int major, int minor = 0) => $".NETCoreApp,Version=v{major}.{minor}"; + private static string NetFramework(int major, int minor, int patch = 0) => $".NETFramework,Version=v{major}.{minor}.{patch}"; + + public static string Netcoreapp1 = NetCore(1); + public static string Netcoreapp2 = NetCore(2); + public static string Netcoreapp21 = NetCore(2, 1); + public static string Netcoreapp3 = NetCore(3); + public static string Netcoreapp31 = NetCore(3, 1); + public static string Net5 = NetCore(5); + public static string Net6 = NetCore(6); + public static string Net7 = NetCore(7); + + public static string Net4 = NetFramework(4,0); + public static string Net45 = NetFramework(4, 5); + public static string Net451 = NetFramework(4, 5, 1); + public static string Net452 = NetFramework(4, 5, 2); + public static string Net46 = NetFramework(4, 6); + public static string Net461 = NetFramework(4, 6, 1); + public static string Net462 = NetFramework(4, 6, 2); + public static string Net47 = NetFramework(4, 7); + public static string Net471 = NetFramework(4, 7, 1); + public static string Net472 = NetFramework(4, 7, 2); + public static string Net48 = NetFramework(4, 8); +} diff --git a/test/vstest.ProgrammerTests/OutputMessage.cs b/test/vstest.ProgrammerTests/Fakes/OutputMessage.cs similarity index 84% rename from test/vstest.ProgrammerTests/OutputMessage.cs rename to test/vstest.ProgrammerTests/Fakes/OutputMessage.cs index 4ee4fbd94b..498f7fcd51 100644 --- a/test/vstest.ProgrammerTests/OutputMessage.cs +++ b/test/vstest.ProgrammerTests/Fakes/OutputMessage.cs @@ -1,10 +1,9 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT license. See LICENSE file in the project root for full license information. -using Microsoft.VisualStudio.TestPlatform.Utilities; +namespace vstest.ProgrammerTests.Fakes; -#pragma warning disable IDE1006 // Naming Styles -namespace vstest.ProgrammerTests.CommandLine; +using Microsoft.VisualStudio.TestPlatform.Utilities; internal class OutputMessage { diff --git a/test/vstest.ProgrammerTests/StringExtensions.cs b/test/vstest.ProgrammerTests/Fakes/StringExtensions.cs similarity index 94% rename from test/vstest.ProgrammerTests/StringExtensions.cs rename to test/vstest.ProgrammerTests/Fakes/StringExtensions.cs index 07d2c2aee0..fcca5fbef0 100644 --- a/test/vstest.ProgrammerTests/StringExtensions.cs +++ b/test/vstest.ProgrammerTests/Fakes/StringExtensions.cs @@ -1,7 +1,7 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT license. See LICENSE file in the project root for full license information. -namespace vstest.ProgrammerTests.CommandLine; +namespace vstest.ProgrammerTests.Fakes; using System.Linq; diff --git a/test/vstest.ProgrammerTests/TestRequestManagerHelper.cs b/test/vstest.ProgrammerTests/Fakes/TestRequestManagerHelper.cs similarity index 96% rename from test/vstest.ProgrammerTests/TestRequestManagerHelper.cs rename to test/vstest.ProgrammerTests/Fakes/TestRequestManagerHelper.cs index c66c64f3be..bf83c8599e 100644 --- a/test/vstest.ProgrammerTests/TestRequestManagerHelper.cs +++ b/test/vstest.ProgrammerTests/Fakes/TestRequestManagerHelper.cs @@ -1,7 +1,7 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT license. See LICENSE file in the project root for full license information. -namespace vstest.ProgrammerTests.CommandLine; +namespace vstest.ProgrammerTests.Fakes; using System.Diagnostics; @@ -10,7 +10,7 @@ namespace vstest.ProgrammerTests.CommandLine; internal class TestRequestManagerTestHelper { private readonly FakeErrorAggregator _errorAggregator; - private TestRequestManager _testRequestManager; + private readonly TestRequestManager _testRequestManager; private readonly DebugOptions _debugOptions; public TestRequestManagerTestHelper(FakeErrorAggregator errorAggregator, TestRequestManager testRequestManager, DebugOptions debugOptions) diff --git a/test/vstest.ProgrammerTests/KnownFramework.cs b/test/vstest.ProgrammerTests/KnownFramework.cs deleted file mode 100644 index 732dcfc628..0000000000 --- a/test/vstest.ProgrammerTests/KnownFramework.cs +++ /dev/null @@ -1,24 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT license. See LICENSE file in the project root for full license information. - -namespace vstest.ProgrammerTests.CommandLine; - -using System.Runtime.Versioning; - -internal static class KnownFramework -{ - public static FrameworkName NetCore(int major, int minor = 0) => new($".NETCoreApp,Version={major}.{minor}"); - - private static FrameworkName NetFramework(int major, int minor, int patch = 0) => new($".NETFramework,Version={major}.{minor}.{patch}"); - - public static FrameworkName Netcoreapp1 = NetCore(1); - public static FrameworkName Netcoreapp2 = NetCore(2); - public static FrameworkName Netcoreapp21 = NetCore(2, 1); - public static FrameworkName Netcoreapp3 = NetCore(3); - public static FrameworkName Netcoreapp31 = NetCore(3, 1); - public static FrameworkName Net5 = NetCore(5); - public static FrameworkName Net6 = NetCore(6); - public static FrameworkName Net7 = NetCore(7); - - public static FrameworkName Net48 = NetFramework(4, 8); -} diff --git a/test/vstest.ProgrammerTests/RunConfiguration.cs b/test/vstest.ProgrammerTests/RunConfiguration.cs deleted file mode 100644 index ef942da3da..0000000000 --- a/test/vstest.ProgrammerTests/RunConfiguration.cs +++ /dev/null @@ -1,9 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT license. See LICENSE file in the project root for full license information. - -namespace vstest.ProgrammerTests.CommandLine; - -internal static class RunConfiguration -{ - public static ConfigurationEntry MaxParallelLevel { get; } = new(nameof(MaxParallelLevel)); -} diff --git a/test/vstest.ProgrammerTests/TestDlls.cs b/test/vstest.ProgrammerTests/TestDlls.cs deleted file mode 100644 index 29191fb2d5..0000000000 --- a/test/vstest.ProgrammerTests/TestDlls.cs +++ /dev/null @@ -1,9 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT license. See LICENSE file in the project root for full license information. - -namespace vstest.ProgrammerTests.CommandLine; - -internal class TestDlls -{ - public static string MSTest1 { get; } = $"{nameof(MSTest1)}.dll"; -} diff --git a/test/vstest.ProgrammerTests/UnitTest1.cs b/test/vstest.ProgrammerTests/UnitTest1.cs index 22632a0180..616150b57c 100644 --- a/test/vstest.ProgrammerTests/UnitTest1.cs +++ b/test/vstest.ProgrammerTests/UnitTest1.cs @@ -1,7 +1,7 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT license. See LICENSE file in the project root for full license information. -namespace vstest.ProgrammerTests.CommandLine; +namespace vstest.ProgrammerTests; using System.Diagnostics; using System.Runtime.Versioning; @@ -24,7 +24,6 @@ namespace vstest.ProgrammerTests.CommandLine; using Microsoft.VisualStudio.TestPlatform.ObjectModel.Client; using Microsoft.VisualStudio.TestPlatform.ObjectModel.Logging; -using vstest.ProgrammerTests.CommandLine.Fakes; using vstest.ProgrammerTests.Fakes; // exluded from run @@ -80,22 +79,20 @@ public async Task GivenAnMSTestAssemblyWith108Tests_WhenTestsAreRun_Then108Tests LastRunTests = new TestRunChangedEventArgs(new TestRunStatistics(new Dictionary { [TestOutcome.Passed] = 1 }), tests.Last(), new List()), }); List messages = changeMessages.Concat(new[] { completedMessage }).ToList(); - var responses = new List> { - new RequestResponsePair(MessageType.VersionCheck, new FakeMessage(MessageType.VersionCheck, 5)), - new RequestResponsePair(MessageType.ExecutionInitialize, FakeMessage.NoResponse), - new RequestResponsePair(MessageType.StartTestExecutionWithSources, messages), - new RequestResponsePair(MessageType.SessionEnd, message => + var responses = new List> { + new RequestResponsePair(MessageType.VersionCheck, new FakeMessage(MessageType.VersionCheck, 5)), + new RequestResponsePair(MessageType.ExecutionInitialize, FakeMessage.NoResponse), + new RequestResponsePair(MessageType.StartTestExecutionWithSources, messages, false), + new RequestResponsePair(MessageType.SessionEnd, new [] { FakeMessage.NoResponse }, message => { // TODO: how do we associate this to the correct process? var fp = fakeProcessHelper.Processes.Last(); fakeProcessHelper.TerminateProcess(fp); - - return new List { FakeMessage.NoResponse }; }), }; - var fakeCommunicationEndpoint = new FakeCommunicationEndpoint(new FakeCommunicationChannel(responses, fakeErrorAggregator, 1), fakeErrorAggregator); + var fakeCommunicationEndpoint = new FakeCommunicationEndpoint(new FakeCommunicationChannel(responses, fakeErrorAggregator, 1), fakeErrorAggregator); TestServiceLocator.Clear(); TestServiceLocator.Register(fakeCommunicationEndpoint.TestHostConnectionInfo.Endpoint, fakeCommunicationEndpoint); var fakeTestHostProcess = new FakeProcess(fakeErrorAggregator, @"C:\temp\testhost.exe"); @@ -186,14 +183,14 @@ public async Task GivenMultipleMsTestAssembliesThatUseTheSameTargetFrameworkAndA var mstest1Dll = new FakeTestDllBuilder() .WithPath(@"X:\fake\mstest1.dll") - .WithFramework(KnownFramework.Net5) + .WithFramework(KnownFrameworkNames.Net5) .WithArchitecture(Architecture.X64) .WithTestCount(108, 10) .Build(); var testhost1Process = new FakeProcess(fixture.ErrorAggregator, @"X:\fake\testhost1.exe"); - var runTests1 = new FakeMessagesBuilder() + var runTests1 = new FakeTestHostResponsesBuilder() .VersionCheck(5) .ExecutionInitialize(FakeMessage.NoResponse) .StartTestExecutionWithSources(mstest1Dll.TestResultBatches) @@ -208,14 +205,14 @@ public async Task GivenMultipleMsTestAssembliesThatUseTheSameTargetFrameworkAndA var mstest2Dll = new FakeTestDllBuilder() .WithPath(@"X:\fake\mstest2.dll") - .WithFramework(KnownFramework.Net5) + .WithFramework(KnownFrameworkNames.Net5) .WithArchitecture(Architecture.X64) .WithTestCount(50, 8) .Build(); var testhost2Process = new FakeProcess(fixture.ErrorAggregator, @"X:\fake\testhost2.exe"); - var runTests2 = new FakeMessagesBuilder() + var runTests2 = new FakeTestHostResponsesBuilder() .VersionCheck(5) .ExecutionInitialize(FakeMessage.NoResponse) .StartTestExecutionWithSources(mstest2Dll.TestResultBatches) @@ -258,18 +255,18 @@ public async Task GivenMultipleMsTestAssembliesThatUseDifferentTargetFrameworkAn var mstest1Dll = new FakeTestDllBuilder() .WithPath(@"X:\fake\mstest1.dll") - .WithFramework(KnownFramework.Net5) // <--- + .WithFramework(KnownFrameworkNames.Net5) // <--- .WithArchitecture(Architecture.X64) .WithTestCount(2) .Build(); var testhost1Process = new FakeProcess(fixture.ErrorAggregator, @"X:\fake\testhost1.exe"); - var runTests1 = new FakeMessagesBuilder() + var runTests1 = new FakeTestHostResponsesBuilder() .VersionCheck(5) .ExecutionInitialize(FakeMessage.NoResponse) .StartTestExecutionWithSources(mstest1Dll.TestResultBatches) - .SessionEnd(FakeMessage.NoResponse, _ => testhost1Process.Exit()) + .SessionEnd(FakeMessage.NoResponse, afterAction: _ => testhost1Process.Exit()) .Build(); var testhost1 = new FakeTestHostFixtureBuilder(fixture) @@ -282,17 +279,17 @@ public async Task GivenMultipleMsTestAssembliesThatUseDifferentTargetFrameworkAn var mstest2Dll = new FakeTestDllBuilder() .WithPath(@"X:\fake\mstest2.dll") - .WithFramework(KnownFramework.Net48) // <--- + .WithFramework(KnownFrameworkNames.Net48) // <--- .WithArchitecture(Architecture.X64) .WithTestCount(1) .Build(); var testhost2Process = new FakeProcess(fixture.ErrorAggregator, @"X:\fake\testhost2.exe"); - var runTests2 = new FakeMessagesBuilder() + var container = new List(); + var runTests2 = new FakeTestHostResponsesBuilder() .VersionCheck(5) .ExecutionInitialize(FakeMessage.NoResponse) - // .StartTestExecutionWithSources(new FakeMessage(MessageType.TestMessage, new TestMessagePayload { MessageLevel = TestMessageLevel.Error, Message = "Loading type failed." }), _ => testhost2Process.Exit()) .StartTestExecutionWithSources(mstest2Dll.TestResultBatches) .SessionEnd(FakeMessage.NoResponse, _ => testhost2Process.Exit()) .Build(); @@ -303,6 +300,8 @@ public async Task GivenMultipleMsTestAssembliesThatUseDifferentTargetFrameworkAn .WithResponses(runTests2) .Build(); + container.Add(testhost2); + fixture.AddTestHostFixtures(testhost1, testhost2); var testRequestManager = fixture.BuildTestRequestManager(); @@ -310,7 +309,9 @@ public async Task GivenMultipleMsTestAssembliesThatUseDifferentTargetFrameworkAn mstest1Dll.FrameworkName.Should().NotBe(mstest2Dll.FrameworkName); // -- act - var runConfiguration = new Microsoft.VisualStudio.TestPlatform.ObjectModel.RunConfiguration { TestSessionTimeout = 40_000 }.ToXml().OuterXml; + // TODO: Building whole default runconfiguration is needed here, because TestRequestManager does not ensure the basic settings are populated, + // and all methods that populate them just silently fail, so TestHostProvider does not get any useful settings. + var runConfiguration = new RunConfiguration().ToXml().OuterXml; var testRunRequestPayload = new TestRunRequestPayload { Sources = new List { mstest1Dll.Path, mstest2Dll.Path }, @@ -329,14 +330,14 @@ public async Task GivenMultipleMsTestAssembliesThatUseDifferentTargetFrameworkAn // We sent mstest1.dll startWithSources1Text.Should().Contain("mstest1.dll"); // And we sent netcoreapp1.0 as the target framework - startWithSources1Text.Should().Contain(KnownFramework.Netcoreapp1.ToString()); + startWithSources1Text.Should().Contain(KnownFrameworkStrings.Netcoreapp1); var startWithSources2 = testhost2.FakeCommunicationChannel.ProcessedMessages.Single(m => m.Request.MessageType == MessageType.StartTestExecutionWithSources); var startWithSources2Text = startWithSources2.Request.Payload.Select(t => t.ToString()).JoinBySpace(); // We sent mstest2.dll startWithSources2Text.Should().Contain("mstest2.dll"); // And we sent netcoreapp1.0 as the target framework, even though it is incompatible - startWithSources2Text.Should().Contain(KnownFramework.Netcoreapp1.ToString()); + startWithSources2Text.Should().Contain(KnownFrameworkStrings.Netcoreapp1); // In reality, the dll would fail to load, and no tests would run // in our simulation we sent tests back anyway, so we get all tests results @@ -348,3 +349,6 @@ public async Task GivenMultipleMsTestAssembliesThatUseDifferentTargetFrameworkAn // TODO: passing null runsettings does not fail fast, instead it fails in Fakes settings code // TODO: passing empty string fails in the xml parser code // TODO: passing null sources and null testcases does not fail fast + + // TODO: Just calling Exit, Close won't stop the run, we will keep waiting for test run to complete, I think in real life when we exit then Disconnected will be called on the vstest.console side, leading to abort flow. + //.StartTestExecutionWithSources(new FakeMessage(MessageType.TestMessage, new TestMessagePayload { MessageLevel = TestMessageLevel.Error, Message = "Loading type failed." }), afterAction: f => { /*f.Process.Exit();*/ f.FakeCommunicationEndpoint.Disconnect(); }) diff --git a/test/vstest.ProgrammerTests/VstestConsole.cs b/test/vstest.ProgrammerTests/VstestConsole.cs deleted file mode 100644 index a079e99919..0000000000 --- a/test/vstest.ProgrammerTests/VstestConsole.cs +++ /dev/null @@ -1,35 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT license. See LICENSE file in the project root for full license information. - -namespace vstest.ProgrammerTests.CommandLine; - -using Microsoft.VisualStudio.TestPlatform.CommandLine; -using vstest.ProgrammerTests.CommandLine.Fakes; - -internal class VstestConsoleBuilder -{ - public List Sources { get; } = new(); - - public List Arguments { get; } = new(); - - internal FakeOutput Output { get; } = new(); - - internal VstestConsoleBuilder WithSource(params string[] sources) - { - Sources.AddRange(sources); - return this; - } - - internal VstestConsoleBuilder WithArguments(params string[] arguments) - { - Arguments.AddRange(arguments); - return this; - } - - internal void Execute() - { - var commandLine = new[] { Sources, Arguments }.SelectMany(s => s).JoinBySpace(); - // vstest.console - var console = new Executor(Output).Execute(commandLine); - } -} From 48d6a0b6ed80f6de695f78b6d1594d08f4609505 Mon Sep 17 00:00:00 2001 From: nohwnd Date: Fri, 18 Feb 2022 08:53:42 +0100 Subject: [PATCH 18/32] Check warning. --- test/vstest.ProgrammerTests/UnitTest1.cs | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/test/vstest.ProgrammerTests/UnitTest1.cs b/test/vstest.ProgrammerTests/UnitTest1.cs index 616150b57c..9ee06ed822 100644 --- a/test/vstest.ProgrammerTests/UnitTest1.cs +++ b/test/vstest.ProgrammerTests/UnitTest1.cs @@ -323,6 +323,10 @@ public async Task GivenMultipleMsTestAssembliesThatUseDifferentTargetFrameworkAn // -- assert fixture.AssertNoErrors(); + // We unify the frameworks to netcoreapp1.0 (because the vstest.console dll we are loading is built for netcoreapp and prefers netcoreapp), and because the + // behavior is to choose the common oldest framework. We then log warning about incompatible sources. + fixture.TestRunEventsRegistrar.LoggedWarnings.Should().ContainMatch($"Test run detected DLL(s) which were built for different framework and platform versions*{KnownFrameworkNames.Netcoreapp1}*"); + // We started both testhosts, even thought we know one of them is incompatible. fixture.ProcessHelper.Processes.Where(p => p.Started).Should().HaveCount(2); var startWithSources1 = testhost1.FakeCommunicationChannel.ProcessedMessages.Single(m => m.Request.MessageType == MessageType.StartTestExecutionWithSources); @@ -349,6 +353,5 @@ public async Task GivenMultipleMsTestAssembliesThatUseDifferentTargetFrameworkAn // TODO: passing null runsettings does not fail fast, instead it fails in Fakes settings code // TODO: passing empty string fails in the xml parser code // TODO: passing null sources and null testcases does not fail fast - - // TODO: Just calling Exit, Close won't stop the run, we will keep waiting for test run to complete, I think in real life when we exit then Disconnected will be called on the vstest.console side, leading to abort flow. - //.StartTestExecutionWithSources(new FakeMessage(MessageType.TestMessage, new TestMessagePayload { MessageLevel = TestMessageLevel.Error, Message = "Loading type failed." }), afterAction: f => { /*f.Process.Exit();*/ f.FakeCommunicationEndpoint.Disconnect(); }) +// TODO: Just calling Exit, Close won't stop the run, we will keep waiting for test run to complete, I think in real life when we exit then Disconnected will be called on the vstest.console side, leading to abort flow. +//.StartTestExecutionWithSources(new FakeMessage(MessageType.TestMessage, new TestMessagePayload { MessageLevel = TestMessageLevel.Error, Message = "Loading type failed." }), afterAction: f => { /*f.Process.Exit();*/ f.FakeCommunicationEndpoint.Disconnect(); }) From 1c3ca5cd44970816c82502b9f17d8570ddd24d20 Mon Sep 17 00:00:00 2001 From: nohwnd Date: Fri, 18 Feb 2022 10:18:03 +0100 Subject: [PATCH 19/32] Allow 0 tests to be returned. --- .../TestServiceLocator.cs | 15 ++++++- .../Intent.Primitives.csproj | 4 ++ test/Intent/Intent.csproj | 4 ++ .../Fakes/FakeTestHostResponsesBuilder.cs | 11 ++---- test/vstest.ProgrammerTests/Program.cs | 3 +- test/vstest.ProgrammerTests/UnitTest1.cs | 39 ++++--------------- 6 files changed, 34 insertions(+), 42 deletions(-) diff --git a/src/Microsoft.TestPlatform.ObjectModel/TestServiceLocator.cs b/src/Microsoft.TestPlatform.ObjectModel/TestServiceLocator.cs index 075422a65d..6264588d08 100644 --- a/src/Microsoft.TestPlatform.ObjectModel/TestServiceLocator.cs +++ b/src/Microsoft.TestPlatform.ObjectModel/TestServiceLocator.cs @@ -3,16 +3,23 @@ namespace Microsoft.VisualStudio.TestPlatform.ObjectModel; +// We don't want this in our shipped code. Build only for debug until I am able to remove it. +#if DEBUG + +#if !NETSTANDARD1_0 using System; +#endif using System.Collections.Generic; -// TODO: Make this internal, I am just trying to have easier time trying this out. +#pragma warning disable RS0016 // Add public types and members to the declared API +#pragma warning disable RS0037 // Enable tracking of nullability of reference types in the declared API + public static class TestServiceLocator { public static Dictionary Instances { get; } = new Dictionary(); public static List Resolves { get; } = new(); - public static void Register(string name, TRegistration instance) + public static void Register(string name, TRegistration instance) where TRegistration : notnull { Instances.Add(name, instance); } @@ -53,3 +60,7 @@ public Resolve(string name, string type, string stackTrace) public string Type { get; } public string StackTrace { get; } } + +#pragma warning restore RS0037 // Enable tracking of nullability of reference types in the declared API +#pragma warning restore RS0016 // Add public types and members to the declared API +#endif diff --git a/test/Intent.Primitives/Intent.Primitives.csproj b/test/Intent.Primitives/Intent.Primitives.csproj index 132c02c59c..93ec3f6e75 100644 --- a/test/Intent.Primitives/Intent.Primitives.csproj +++ b/test/Intent.Primitives/Intent.Primitives.csproj @@ -1,9 +1,13 @@ + $(MSBuildThisFileDirectory)..\..\ net6.0 enable enable + True + $(TestPlatformRoot)\scripts\key.snk + True diff --git a/test/Intent/Intent.csproj b/test/Intent/Intent.csproj index bb711c9256..97b464df79 100644 --- a/test/Intent/Intent.csproj +++ b/test/Intent/Intent.csproj @@ -1,10 +1,14 @@ + $(MSBuildThisFileDirectory)..\..\ Exe net6.0 enable enable + True + True + $(TestPlatformRoot)\scripts\key.snk diff --git a/test/vstest.ProgrammerTests/Fakes/FakeTestHostResponsesBuilder.cs b/test/vstest.ProgrammerTests/Fakes/FakeTestHostResponsesBuilder.cs index 056d0e31bc..2f3302e565 100644 --- a/test/vstest.ProgrammerTests/Fakes/FakeTestHostResponsesBuilder.cs +++ b/test/vstest.ProgrammerTests/Fakes/FakeTestHostResponsesBuilder.cs @@ -63,16 +63,14 @@ internal FakeTestHostResponsesBuilder StartTestExecutionWithSources(FakeMessage internal FakeTestHostResponsesBuilder StartTestExecutionWithSources(List> testResultBatches!!) { - var tests = testResultBatches; - List messages; - if (testResultBatches.Count == 0) + if (testResultBatches.Count != 0) { // this will create as many test stats changes messages, as there are batches -1 // the last batch will be sent as test run complete event // TODO: make the stats agree with the tests below - List changeMessages = tests.Take(tests.Count - 1).Select(batch => + List changeMessages = testResultBatches.Take(testResultBatches.Count - 1).Select(batch => new FakeMessage(MessageType.TestRunStatsChange, new TestRunChangedEventArgs(new TestRunStatistics(new Dictionary { [TestOutcome.Passed] = batch.Count }), batch, new List()) )).ToList(); @@ -82,7 +80,7 @@ internal FakeTestHostResponsesBuilder StartTestExecutionWithSources(List { [TestOutcome.Passed] = 1 }), false, false, null, new System.Collections.ObjectModel.Collection(), TimeSpan.Zero), - LastRunTests = new TestRunChangedEventArgs(new TestRunStatistics(new Dictionary { [TestOutcome.Passed] = 1 }), tests.Last(), new List()), + LastRunTests = new TestRunChangedEventArgs(new TestRunStatistics(new Dictionary { [TestOutcome.Passed] = 1 }), testResultBatches.Last(), new List()), }); messages = changeMessages.Concat(new[] { completedMessage }).ToList(); } @@ -90,9 +88,8 @@ internal FakeTestHostResponsesBuilder StartTestExecutionWithSources(List(MessageType.ExecutionComplete, new TestRunCompletePayload { - // TODO: make the stats agree with the tests below TestRunCompleteArgs = new TestRunCompleteEventArgs(new TestRunStatistics(new Dictionary { [TestOutcome.Passed] = 0 }), false, false, null, new System.Collections.ObjectModel.Collection(), TimeSpan.Zero), - LastRunTests = new TestRunChangedEventArgs(new TestRunStatistics(new Dictionary { [TestOutcome.Passed] = 0 }), tests.Last(), new List()), + LastRunTests = new TestRunChangedEventArgs(new TestRunStatistics(new Dictionary { [TestOutcome.Passed] = 0 }), new List(), new List()), }); messages = completedMessage.AsList(); diff --git a/test/vstest.ProgrammerTests/Program.cs b/test/vstest.ProgrammerTests/Program.cs index 6de575dca8..33f8f4d538 100644 --- a/test/vstest.ProgrammerTests/Program.cs +++ b/test/vstest.ProgrammerTests/Program.cs @@ -1,9 +1,10 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT license. See LICENSE file in the project root for full license information. +namespace vstest.ProgrammerTests; + using System.Reflection; -namespace vstest.ProgrammerTests; internal class Program { static void Main() diff --git a/test/vstest.ProgrammerTests/UnitTest1.cs b/test/vstest.ProgrammerTests/UnitTest1.cs index 9ee06ed822..e49b9084a2 100644 --- a/test/vstest.ProgrammerTests/UnitTest1.cs +++ b/test/vstest.ProgrammerTests/UnitTest1.cs @@ -22,30 +22,9 @@ namespace vstest.ProgrammerTests; using Microsoft.VisualStudio.TestPlatform.CrossPlatEngine.TestRunAttachmentsProcessing; using Microsoft.VisualStudio.TestPlatform.ObjectModel; using Microsoft.VisualStudio.TestPlatform.ObjectModel.Client; -using Microsoft.VisualStudio.TestPlatform.ObjectModel.Logging; using vstest.ProgrammerTests.Fakes; -// exluded from run -internal class InlineRunSettingsTests -{ - public void GivenInlineRunsettingsWhenCallingVstestConsoleThenTheyPropagateToTestHost() - { - //using Fixture fixture = new(); - //fixture.VstestConsole - // .WithSource(TestDlls.MSTest1) - // .WithArguments($" -- {RunConfiguration.MaxParallelLevel.InlinePath}=3") - // .Execute(); - - //fixture.Processes.Should().HaveCount(1); - //var process = fixture.Processes.First(); - //process.Should().BeAssignableTo(); - //var testhost = (FakeTestHostProcess)process; - //testhost.RunSettings.Should().NotBeNull(); - //testhost.RunSettings!.MaxParallelLevel.Should().Be(3); - } -} - public class TestDiscoveryTests { public async Task GivenAnMSTestAssemblyWith108Tests_WhenTestsAreRun_Then108TestsAreExecuted() @@ -66,7 +45,6 @@ public async Task GivenAnMSTestAssemblyWith108Tests_WhenTestsAreRun_Then108Tests .WithBatchSize(10) .Build(); var mstest1Dll = new FakeTestDllFile(@"X:\fake\mstest1.dll", new FrameworkName(".NETCoreApp,Version=v5.0"), Architecture.X64, tests); - fakeFileHelper.AddFile(mstest1Dll); List changeMessages = tests.Take(tests.Count - 1).Select(batch => // TODO: make the stats agree with the tests below new FakeMessage(MessageType.TestRunStatsChange, @@ -91,8 +69,9 @@ public async Task GivenAnMSTestAssemblyWith108Tests_WhenTestsAreRun_Then108Tests }), }; - - var fakeCommunicationEndpoint = new FakeCommunicationEndpoint(new FakeCommunicationChannel(responses, fakeErrorAggregator, 1), fakeErrorAggregator); + var fakeCommunicationChannel = new FakeCommunicationChannel(responses, fakeErrorAggregator, 1); + fakeCommunicationChannel.Start(new object()); + var fakeCommunicationEndpoint = new FakeCommunicationEndpoint(fakeCommunicationChannel, fakeErrorAggregator); TestServiceLocator.Clear(); TestServiceLocator.Register(fakeCommunicationEndpoint.TestHostConnectionInfo.Endpoint, fakeCommunicationEndpoint); var fakeTestHostProcess = new FakeProcess(fakeErrorAggregator, @"C:\temp\testhost.exe"); @@ -245,7 +224,6 @@ public async Task GivenMultipleMsTestAssembliesThatUseTheSameTargetFrameworkAndA fixture.ExecutedTests.Should().HaveCount(mstest1Dll.TestCount + mstest2Dll.TestCount); } - [Only] public async Task GivenMultipleMsTestAssembliesThatUseDifferentTargetFrameworkAndTheSameArchitecture_WhenTestsAreRun_ThenTwoTesthostsAreStartedBothForTheSameTFM() { // TODO: make vstest.console not start testhosts for incompatible sources. @@ -281,12 +259,13 @@ public async Task GivenMultipleMsTestAssembliesThatUseDifferentTargetFrameworkAn .WithPath(@"X:\fake\mstest2.dll") .WithFramework(KnownFrameworkNames.Net48) // <--- .WithArchitecture(Architecture.X64) - .WithTestCount(1) + // In reality, the dll would fail to load, and no tests would run from this dll, + // we simulate that by making it have 0 tests. + .WithTestCount(0) .Build(); var testhost2Process = new FakeProcess(fixture.ErrorAggregator, @"X:\fake\testhost2.exe"); - var container = new List(); var runTests2 = new FakeTestHostResponsesBuilder() .VersionCheck(5) .ExecutionInitialize(FakeMessage.NoResponse) @@ -300,8 +279,6 @@ public async Task GivenMultipleMsTestAssembliesThatUseDifferentTargetFrameworkAn .WithResponses(runTests2) .Build(); - container.Add(testhost2); - fixture.AddTestHostFixtures(testhost1, testhost2); var testRequestManager = fixture.BuildTestRequestManager(); @@ -343,9 +320,7 @@ public async Task GivenMultipleMsTestAssembliesThatUseDifferentTargetFrameworkAn // And we sent netcoreapp1.0 as the target framework, even though it is incompatible startWithSources2Text.Should().Contain(KnownFrameworkStrings.Netcoreapp1); - // In reality, the dll would fail to load, and no tests would run - // in our simulation we sent tests back anyway, so we get all tests results - fixture.ExecutedTests.Should().HaveCount(mstest1Dll.TestCount + mstest2Dll.TestCount); + fixture.ExecutedTests.Should().HaveCount(mstest1Dll.TestCount); } } From 1ea876bd60f7a3782c85a9a75adfff1e65ff1b8b Mon Sep 17 00:00:00 2001 From: nohwnd Date: Fri, 18 Feb 2022 13:17:51 +0100 Subject: [PATCH 20/32] Add comment --- .../Hosting/ITestRuntimeProviderManager.cs | 3 ++- test/vstest.ProgrammerTests/UnitTest1.cs | 5 ++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/src/Microsoft.TestPlatform.Common/Hosting/ITestRuntimeProviderManager.cs b/src/Microsoft.TestPlatform.Common/Hosting/ITestRuntimeProviderManager.cs index 023edd18b8..57366c4669 100644 --- a/src/Microsoft.TestPlatform.Common/Hosting/ITestRuntimeProviderManager.cs +++ b/src/Microsoft.TestPlatform.Common/Hosting/ITestRuntimeProviderManager.cs @@ -1,9 +1,10 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT license. See LICENSE file in the project root for full license information. +namespace Microsoft.VisualStudio.TestPlatform.Common.Hosting; + using Microsoft.VisualStudio.TestPlatform.ObjectModel.Host; -namespace Microsoft.VisualStudio.TestPlatform.Common.Hosting; internal interface ITestRuntimeProviderManager { ITestRuntimeProvider GetTestHostManagerByRunConfiguration(string runConfiguration); diff --git a/test/vstest.ProgrammerTests/UnitTest1.cs b/test/vstest.ProgrammerTests/UnitTest1.cs index e49b9084a2..86a5c5852f 100644 --- a/test/vstest.ProgrammerTests/UnitTest1.cs +++ b/test/vstest.ProgrammerTests/UnitTest1.cs @@ -25,6 +25,9 @@ namespace vstest.ProgrammerTests; using vstest.ProgrammerTests.Fakes; +// Tests are run by Intent library that is executed from our Program.Main. To debug press F5 in VS, and maybe mark just a single test with [Only]. +// To just run, press Ctrl+F5 to run without debugging. It will use short timeout for abort in case something is wrong with your test. + public class TestDiscoveryTests { public async Task GivenAnMSTestAssemblyWith108Tests_WhenTestsAreRun_Then108TestsAreExecuted() @@ -195,7 +198,7 @@ public async Task GivenMultipleMsTestAssembliesThatUseTheSameTargetFrameworkAndA .VersionCheck(5) .ExecutionInitialize(FakeMessage.NoResponse) .StartTestExecutionWithSources(mstest2Dll.TestResultBatches) - .SessionEnd(FakeMessage.NoResponse, _ => testhost2Process.Exit()) + .SessionEnd(FakeMessage.NoResponse, f => f.Process.Exit()) .Build(); var testhost2 = new FakeTestHostFixtureBuilder(fixture) From b3d128d46c2115ab10ddf207c1a766d89ebf3d44 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Jare=C5=A1?= Date: Fri, 18 Feb 2022 04:34:56 -0800 Subject: [PATCH 21/32] Apply suggestions from code review --- .../DataCollectionRequestHandler.cs | 1 - .../SocketClient.cs | 2 +- .../TestRequestSender.cs | 8 ++------ .../Client/Parallel/ParallelRunDataAggregator.cs | 2 +- .../Client/ProxyExecutionManager.cs | 2 +- .../Client/ProxyOperationManager.cs | 2 +- test/Intent/Runner.cs | 2 +- ...crosoft.TestPlatform.TestHostProvider.UnitTests.csproj | 2 +- 8 files changed, 8 insertions(+), 13 deletions(-) diff --git a/src/Microsoft.TestPlatform.CommunicationUtilities/DataCollectionRequestHandler.cs b/src/Microsoft.TestPlatform.CommunicationUtilities/DataCollectionRequestHandler.cs index c0179c544b..63bae93f18 100644 --- a/src/Microsoft.TestPlatform.CommunicationUtilities/DataCollectionRequestHandler.cs +++ b/src/Microsoft.TestPlatform.CommunicationUtilities/DataCollectionRequestHandler.cs @@ -160,7 +160,6 @@ public static DataCollectionRequestHandler Create( // the .Instance. This is a very complicated way of solving the circular dependency, // and should be replaced by adding a property to Message and assigning it. // .Instance can then be removed. - if (Instance == null) { lock (SyncObject) diff --git a/src/Microsoft.TestPlatform.CommunicationUtilities/SocketClient.cs b/src/Microsoft.TestPlatform.CommunicationUtilities/SocketClient.cs index ca521b3fa2..a2a1120f57 100644 --- a/src/Microsoft.TestPlatform.CommunicationUtilities/SocketClient.cs +++ b/src/Microsoft.TestPlatform.CommunicationUtilities/SocketClient.cs @@ -106,7 +106,7 @@ private void OnServerConnected(Task connectAsyncTask) private void StopOnError(Exception error) { EqtTrace.Info("SocketClient.PrivateStop: Stop communication from server endpoint: {0}, error:{1}", _endPoint, error); - // TODO: this is here to prevent stack overflow. + // This is here to prevent stack overflow. if (!_stopped) { // Do not allow stop to be called multiple times. diff --git a/src/Microsoft.TestPlatform.CommunicationUtilities/TestRequestSender.cs b/src/Microsoft.TestPlatform.CommunicationUtilities/TestRequestSender.cs index bd3b5a3642..870c8396a7 100644 --- a/src/Microsoft.TestPlatform.CommunicationUtilities/TestRequestSender.cs +++ b/src/Microsoft.TestPlatform.CommunicationUtilities/TestRequestSender.cs @@ -29,8 +29,6 @@ namespace Microsoft.VisualStudio.TestPlatform.CommunicationUtilities; public class TestRequestSender : ITestRequestSender { // Time to wait for test host exit - // DONOTMERGE: this was 10s, I made it 1 second, because it makes my tests pass faster when I get in error state - // REVIEW: this was 10s, I made it 1 seconds private const int ClientProcessExitWaitTimeout = 10 * 1000; private readonly IDataSerializer _dataSerializer; @@ -105,8 +103,6 @@ internal TestRequestSender( // TODO: In various places TestRequest sender is instantiated, and we can't easily inject the factory, so this is last // resort of getting the dependency into the execution flow. - // TODO: I am not sure if we need multiple instances of ICommunicationEndpoint, in that case we should register - // and resolve Func and invoke that. _communicationEndpoint = communicationEndPoint #if DEBUG ?? TestServiceLocator.Get(connectionInfo.Endpoint) @@ -174,7 +170,7 @@ public int InitializeCommunication() // Server start returns the listener port // return int.Parse(this.communicationServer.Start()); var endpoint = _communicationEndpoint.Start(_connectionInfo.Endpoint); - // TODO: This is forcing us to use ip and port for communication + // TODO: This is forcing us to use IP address and port for communication return endpoint.GetIpEndPoint().Port; } @@ -656,7 +652,7 @@ private string GetAbortErrorMessage(Exception exception, bool getClientError) EqtTrace.Verbose("TestRequestSender: GetAbortErrorMessage: Client has disconnected. Wait for standard error."); // Wait for test host to exit for a moment - // TODO: this timeout is 10 seconds, make it also configurable like the other famous + // TODO: this timeout is 10 seconds, make it also configurable like the other famous timeout that is 100ms if (_clientExited.Wait(_clientExitedWaitTime)) { // Set a default message of test host process exited and additionally specify the error if we were able to get it diff --git a/src/Microsoft.TestPlatform.CrossPlatEngine/Client/Parallel/ParallelRunDataAggregator.cs b/src/Microsoft.TestPlatform.CrossPlatEngine/Client/Parallel/ParallelRunDataAggregator.cs index 82fb4c08ce..6a7e86d2c3 100644 --- a/src/Microsoft.TestPlatform.CrossPlatEngine/Client/Parallel/ParallelRunDataAggregator.cs +++ b/src/Microsoft.TestPlatform.CrossPlatEngine/Client/Parallel/ParallelRunDataAggregator.cs @@ -72,7 +72,7 @@ public ITestRunStatistics GetAggregatedRunStats() { foreach (var runStats in _testRunStatsList) { - // TODO: we get nullref here if the stats are empty. Should that be okay or not? + // TODO: we get nullref here if the stats are empty. foreach (var outcome in runStats.Stats.Keys) { if (!testOutcomeMap.ContainsKey(outcome)) diff --git a/src/Microsoft.TestPlatform.CrossPlatEngine/Client/ProxyExecutionManager.cs b/src/Microsoft.TestPlatform.CrossPlatEngine/Client/ProxyExecutionManager.cs index 3b9e7e5a99..47886ec36b 100644 --- a/src/Microsoft.TestPlatform.CrossPlatEngine/Client/ProxyExecutionManager.cs +++ b/src/Microsoft.TestPlatform.CrossPlatEngine/Client/ProxyExecutionManager.cs @@ -359,7 +359,7 @@ public void HandleTestRunStatsChange(TestRunChangedEventArgs testRunChangedArgs) /// public void HandleRawMessage(string rawMessage) { - // TODO: perf - why do we have to deserialize the messages here only to read that this is + // TODO: PERF: - why do we have to deserialize the messages here only to read that this is // execution complete? Why can't we act on it somewhere else where the result of deserialization is not // thrown away? var message = _dataSerializer.DeserializeMessage(rawMessage); diff --git a/src/Microsoft.TestPlatform.CrossPlatEngine/Client/ProxyOperationManager.cs b/src/Microsoft.TestPlatform.CrossPlatEngine/Client/ProxyOperationManager.cs index ae140c8268..88f0c020aa 100644 --- a/src/Microsoft.TestPlatform.CrossPlatEngine/Client/ProxyOperationManager.cs +++ b/src/Microsoft.TestPlatform.CrossPlatEngine/Client/ProxyOperationManager.cs @@ -416,7 +416,7 @@ private void CompatIssueWithVersionCheckAndRunsettings() var properties = TestHostManager.GetType().GetRuntimeProperties(); // The field is actually defaulting to true, so this is just a complicated way to set or not set - // this to true (modern testhosts shoul have it set to true). Bad thing about this is that we are checking + // this to true (modern testhosts should have it set to true). Bad thing about this is that we are checking // internal "undocumented" property. Good thing is that if you don't implement it you get the modern behavior. var versionCheckProperty = properties.FirstOrDefault(p => string.Equals(p.Name, _versionCheckPropertyName, StringComparison.OrdinalIgnoreCase)); if (versionCheckProperty != null) diff --git a/test/Intent/Runner.cs b/test/Intent/Runner.cs index 1fa12c365f..e349dde222 100644 --- a/test/Intent/Runner.cs +++ b/test/Intent/Runner.cs @@ -22,7 +22,7 @@ public static void Run(IEnumerable path, IRunLogger logger) { var ms = t.GetMethods().SkipExcluded(); - // This chooses the Only tests only for single assembly and single class, + // TODO: This chooses the Only tests only for single assembly and single class, // to support this full we would have to enumerate all classes and methods first, // it is easy, I just don't need it right now. var only = ms.Where(m => m.GetCustomAttribute() != null).ToList(); diff --git a/test/Microsoft.TestPlatform.TestHostProvider.UnitTests/Microsoft.TestPlatform.TestHostProvider.UnitTests.csproj b/test/Microsoft.TestPlatform.TestHostProvider.UnitTests/Microsoft.TestPlatform.TestHostProvider.UnitTests.csproj index c799e07028..e098a6461c 100644 --- a/test/Microsoft.TestPlatform.TestHostProvider.UnitTests/Microsoft.TestPlatform.TestHostProvider.UnitTests.csproj +++ b/test/Microsoft.TestPlatform.TestHostProvider.UnitTests/Microsoft.TestPlatform.TestHostProvider.UnitTests.csproj @@ -8,7 +8,7 @@ Microsoft.TestPlatform.TestHostProvider.UnitTests - netcoreapp3.1;net472 + netcoreapp2.1;net472 netcoreapp3.1 Exe From ab696c9eed8f9fd15f5159c0244410aa9b70ea11 Mon Sep 17 00:00:00 2001 From: nohwnd Date: Fri, 18 Feb 2022 13:35:51 +0100 Subject: [PATCH 22/32] Revert playground --- playground/MSTest1/MSTest1.csproj | 2 +- playground/TestPlatform.Playground/Program.cs | 3 +-- .../TestPlatform.Playground/TestPlatform.Playground.csproj | 2 +- 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/playground/MSTest1/MSTest1.csproj b/playground/MSTest1/MSTest1.csproj index 881b74fec7..e952c0fa11 100644 --- a/playground/MSTest1/MSTest1.csproj +++ b/playground/MSTest1/MSTest1.csproj @@ -6,7 +6,7 @@ - $(TargetFrameworks);net472;netcoreapp3.1 + $(TargetFrameworks);net472 $(TargetFrameworks);net451 false diff --git a/playground/TestPlatform.Playground/Program.cs b/playground/TestPlatform.Playground/Program.cs index 4de92bd44c..328d97e44d 100644 --- a/playground/TestPlatform.Playground/Program.cs +++ b/playground/TestPlatform.Playground/Program.cs @@ -55,8 +55,7 @@ static void Main(string[] args) "; var sources = new[] { - Path.Combine(playground, "MSTest1", "bin", "Debug", "net472", "MSTest1.dll"), - Path.Combine(playground, "MSTest1", "bin", "Debug", "netcoreapp3.1", "MSTest1.dll"), + Path.Combine(playground, "MSTest1", "bin", "Debug", "net472", "MSTest1.dll") }; var options = new TestPlatformOptions(); diff --git a/playground/TestPlatform.Playground/TestPlatform.Playground.csproj b/playground/TestPlatform.Playground/TestPlatform.Playground.csproj index f121a18af7..d73746f16e 100644 --- a/playground/TestPlatform.Playground/TestPlatform.Playground.csproj +++ b/playground/TestPlatform.Playground/TestPlatform.Playground.csproj @@ -39,7 +39,7 @@ - + From 183b711b3bea862e7ddc7ad23c2d4886bf97d6f1 Mon Sep 17 00:00:00 2001 From: nohwnd Date: Fri, 18 Feb 2022 13:46:50 +0100 Subject: [PATCH 23/32] Fix protected changes --- src/Microsoft.TestPlatform.Client/TestPlatform.cs | 2 +- src/Microsoft.TestPlatform.CrossPlatEngine/TestEngine.cs | 6 ++++++ src/vstest.console/CommandLine/TestRunResultAggregator.cs | 2 +- 3 files changed, 8 insertions(+), 2 deletions(-) diff --git a/src/Microsoft.TestPlatform.Client/TestPlatform.cs b/src/Microsoft.TestPlatform.Client/TestPlatform.cs index a125a70930..c19ddb65af 100644 --- a/src/Microsoft.TestPlatform.Client/TestPlatform.cs +++ b/src/Microsoft.TestPlatform.Client/TestPlatform.cs @@ -66,7 +66,7 @@ public TestPlatform() /// The test engine. /// The file helper. /// The data. - internal TestPlatform( + protected internal TestPlatform( ITestEngine testEngine, IFileHelper filehelper, ITestRuntimeProviderManager testHostProviderManager) diff --git a/src/Microsoft.TestPlatform.CrossPlatEngine/TestEngine.cs b/src/Microsoft.TestPlatform.CrossPlatEngine/TestEngine.cs index e3810008fe..d8d5373ef8 100644 --- a/src/Microsoft.TestPlatform.CrossPlatEngine/TestEngine.cs +++ b/src/Microsoft.TestPlatform.CrossPlatEngine/TestEngine.cs @@ -42,6 +42,12 @@ public class TestEngine : ITestEngine { } + protected internal TestEngine( + TestRuntimeProviderManager testHostProviderManager, + IProcessHelper processHelper) : this((ITestRuntimeProviderManager) testHostProviderManager, processHelper) + { + } + internal TestEngine( ITestRuntimeProviderManager testHostProviderManager, IProcessHelper processHelper) diff --git a/src/vstest.console/CommandLine/TestRunResultAggregator.cs b/src/vstest.console/CommandLine/TestRunResultAggregator.cs index 39ea714dfd..05b0505558 100644 --- a/src/vstest.console/CommandLine/TestRunResultAggregator.cs +++ b/src/vstest.console/CommandLine/TestRunResultAggregator.cs @@ -20,7 +20,7 @@ internal class TestRunResultAggregator /// Initializes the TestRunResultAggregator /// /// Constructor is private since the factory method should be used to get the instance. - internal TestRunResultAggregator() + protected internal TestRunResultAggregator() { // Outcome is passed until we see a failure. Outcome = TestOutcome.Passed; From 346fe306366cfec9dce951b3a52958f9ca28008a Mon Sep 17 00:00:00 2001 From: nohwnd Date: Fri, 18 Feb 2022 13:53:34 +0100 Subject: [PATCH 24/32] Remove using --- test/vstest.ProgrammerTests/UnitTest1.cs | 2 -- 1 file changed, 2 deletions(-) diff --git a/test/vstest.ProgrammerTests/UnitTest1.cs b/test/vstest.ProgrammerTests/UnitTest1.cs index 86a5c5852f..dec07721fe 100644 --- a/test/vstest.ProgrammerTests/UnitTest1.cs +++ b/test/vstest.ProgrammerTests/UnitTest1.cs @@ -9,8 +9,6 @@ namespace vstest.ProgrammerTests; using FluentAssertions; using FluentAssertions.Extensions; -using Intent; - using Microsoft.VisualStudio.TestPlatform.Client; using Microsoft.VisualStudio.TestPlatform.CommandLine; using Microsoft.VisualStudio.TestPlatform.CommandLine.Publisher; From a3635da4c8de43a07f674d4f6b229f4cc7a0f3b7 Mon Sep 17 00:00:00 2001 From: nohwnd Date: Fri, 18 Feb 2022 14:12:27 +0100 Subject: [PATCH 25/32] More cosmetic issues --- .../Fakes/DebugOptions.cs | 20 +++++ .../Fakes/EventRecord.cs | 3 - .../Fakes/FakeCommunicationChannel.cs | 80 ------------------- ...taCollectorAttachmentsProcessorsFactory.cs | 5 +- .../Fakes/FakeMessage.cs | 53 ++++++++++++ .../Fakes/FakeMetricsPublisher.cs | 5 +- .../Fakes/FakeOutput.cs | 4 - .../Fakes/FakeProcessBuilder.cs | 11 --- .../Fakes/FakeTestHostBuilder.cs | 2 + .../Fakes/FakeTestHostLauncher.cs | 5 +- .../Fakes/FakeTestHostProcess.cs | 17 ---- .../Fakes/FakeTestPlatformEventSource.cs | 5 +- .../Fakes/FakeTestRunEventsRegistrar.cs | 5 +- .../Fakes/FakeTestRuntimeProviderManager.cs | 2 - test/vstest.ProgrammerTests/Fakes/Fixture.cs | 37 +++------ .../Fakes/RequestResponsePair.cs | 38 +++++++++ test/vstest.ProgrammerTests/UnitTest1.cs | 2 + 17 files changed, 137 insertions(+), 157 deletions(-) create mode 100644 test/vstest.ProgrammerTests/Fakes/DebugOptions.cs create mode 100644 test/vstest.ProgrammerTests/Fakes/FakeMessage.cs delete mode 100644 test/vstest.ProgrammerTests/Fakes/FakeProcessBuilder.cs delete mode 100644 test/vstest.ProgrammerTests/Fakes/FakeTestHostProcess.cs create mode 100644 test/vstest.ProgrammerTests/Fakes/RequestResponsePair.cs diff --git a/test/vstest.ProgrammerTests/Fakes/DebugOptions.cs b/test/vstest.ProgrammerTests/Fakes/DebugOptions.cs new file mode 100644 index 0000000000..8b5e254c5e --- /dev/null +++ b/test/vstest.ProgrammerTests/Fakes/DebugOptions.cs @@ -0,0 +1,20 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +namespace vstest.ProgrammerTests.Fakes; + +internal class DebugOptions +{ + public const int DefaultTimeout = 5; + // TODO: This setting is actually quite pointless, because I cannot come up with + // a useful way to abort quickly enough when debugger is attached and I am just running my tests (pressing F5) + // but at the same time not abort when I am in the middle of debugging some behavior. Maybe looking at debugger, + // and asking it if any breakpoints were hit / are set. But that is difficult. + // + // So normally I press F5 to investigate, but Ctrl+F5 (run without debugger), to run tests. + public const int DefaultDebugTimeout = 30 * 60; + public const bool DefaultBreakOnAbort = true; + public int Timeout { get; init; } = DefaultTimeout; + public int DebugTimeout { get; init; } = DefaultDebugTimeout; + public bool BreakOnAbort { get; init; } = DefaultBreakOnAbort; +} diff --git a/test/vstest.ProgrammerTests/Fakes/EventRecord.cs b/test/vstest.ProgrammerTests/Fakes/EventRecord.cs index 7cd6a9630c..0bfcd03776 100644 --- a/test/vstest.ProgrammerTests/Fakes/EventRecord.cs +++ b/test/vstest.ProgrammerTests/Fakes/EventRecord.cs @@ -1,9 +1,6 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT license. See LICENSE file in the project root for full license information. - -#pragma warning disable IDE1006 // Naming Styles - namespace vstest.ProgrammerTests.Fakes; internal class EventRecord diff --git a/test/vstest.ProgrammerTests/Fakes/FakeCommunicationChannel.cs b/test/vstest.ProgrammerTests/Fakes/FakeCommunicationChannel.cs index 472b03bb05..3939a64c15 100644 --- a/test/vstest.ProgrammerTests/Fakes/FakeCommunicationChannel.cs +++ b/test/vstest.ProgrammerTests/Fakes/FakeCommunicationChannel.cs @@ -207,83 +207,3 @@ private void ProcessIncomingMessages(TContext context) } } -internal class RequestResponsePair where TRequest : class -{ - public RequestResponsePair(TRequest request, TResponse response, bool debug = false) - { - Request = request; - Responses = new List { response }; - Debug = debug; - } - - public RequestResponsePair(TRequest request, IEnumerable responses, bool debug = false) - { - Request = request; - Responses = responses.ToList(); - Debug = debug; - } - - public RequestResponsePair(TRequest request, IEnumerable responses, Action? beforeAction = null, Action? afterAction = null, bool debug = false) - { - Request = request; - Responses = responses.ToList(); - BeforeAction = beforeAction; - AfterAction = afterAction; - Debug = debug; - } - - public TRequest Request { get; } - - // TODO: make this Expression< so we can get some info about what this is doing when looking directly at this instance - public Action? BeforeAction { get; } - public Action? AfterAction { get; } - public List Responses { get; } - public bool Debug { get; } -} - -/// -/// A class like Message / VersionedMessage that is easier to create and review during debugging. -/// -internal sealed class FakeMessage : FakeMessage -{ - public FakeMessage(string messageType, T payload, int version = 0) - { - MessageType = messageType; - Payload = payload; - Version = version; - SerializedMessage = JsonDataSerializer.Instance.SerializePayload(MessageType, payload, version); - } - - /// - /// Message identifier, usually coming from the MessageType class. - /// - public string MessageType { get; } - - /// - /// The payload that this message is holding. - /// - public T Payload { get; } - - /// - /// Version of the message to allow the internal serializer to choose the correct serialization strategy. - /// - public int Version { get; } -} - -/// -/// Marker for Fake message so we can put put all FakeMessages into one collection, without making it too wide. -/// -internal abstract class FakeMessage -{ - /// - /// The message serialized using the default JsonDataSerializer. - /// - // TODO: Is there a better way to ensure that is is not null, we will always set it in the inherited types, but it would be nice to have warning if we did not. - // And adding constructor makes it difficult to use the serializer, especially if we wanted to the serializer dynamic and not a static instance. - public string SerializedMessage { get; init; } = string.Empty; - - /// - /// - /// - public static FakeMessage NoResponse { get; } = new FakeMessage("NoResponse", 0); -} diff --git a/test/vstest.ProgrammerTests/Fakes/FakeDataCollectorAttachmentsProcessorsFactory.cs b/test/vstest.ProgrammerTests/Fakes/FakeDataCollectorAttachmentsProcessorsFactory.cs index e6a83a2455..664dd699a1 100644 --- a/test/vstest.ProgrammerTests/Fakes/FakeDataCollectorAttachmentsProcessorsFactory.cs +++ b/test/vstest.ProgrammerTests/Fakes/FakeDataCollectorAttachmentsProcessorsFactory.cs @@ -1,13 +1,12 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT license. See LICENSE file in the project root for full license information. +namespace vstest.ProgrammerTests.Fakes; + using Microsoft.VisualStudio.TestPlatform.ObjectModel; using Microsoft.VisualStudio.TestPlatform.ObjectModel.Engine; using Microsoft.VisualStudio.TestPlatform.ObjectModel.Logging; -#pragma warning disable IDE1006 // Naming Styles -namespace vstest.ProgrammerTests.Fakes; - internal class FakeDataCollectorAttachmentsProcessorsFactory : IDataCollectorAttachmentsProcessorsFactory { public FakeDataCollectorAttachmentsProcessorsFactory(FakeErrorAggregator fakeErrorAggregator) diff --git a/test/vstest.ProgrammerTests/Fakes/FakeMessage.cs b/test/vstest.ProgrammerTests/Fakes/FakeMessage.cs new file mode 100644 index 0000000000..ffbeb8666d --- /dev/null +++ b/test/vstest.ProgrammerTests/Fakes/FakeMessage.cs @@ -0,0 +1,53 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +namespace vstest.ProgrammerTests.Fakes; + +using Microsoft.VisualStudio.TestPlatform.CommunicationUtilities; + +/// +/// Marker for Fake message so we can put put all FakeMessages into one collection, without making it too wide. +/// +internal abstract class FakeMessage +{ + /// + /// The message serialized using the default JsonDataSerializer. + /// + // TODO: Is there a better way to ensure that is is not null, we will always set it in the inherited types, but it would be nice to have warning if we did not. + // And adding constructor makes it difficult to use the serializer, especially if we wanted to the serializer dynamic and not a static instance. + public string SerializedMessage { get; init; } = string.Empty; + + /// + /// + /// + public static FakeMessage NoResponse { get; } = new FakeMessage("NoResponse", 0); +} + +/// +/// A class like Message / VersionedMessage that is easier to create and review during debugging. +/// +internal sealed class FakeMessage : FakeMessage +{ + public FakeMessage(string messageType, T payload, int version = 0) + { + MessageType = messageType; + Payload = payload; + Version = version; + SerializedMessage = JsonDataSerializer.Instance.SerializePayload(MessageType, payload, version); + } + + /// + /// Message identifier, usually coming from the MessageType class. + /// + public string MessageType { get; } + + /// + /// The payload that this message is holding. + /// + public T Payload { get; } + + /// + /// Version of the message to allow the internal serializer to choose the correct serialization strategy. + /// + public int Version { get; } +} diff --git a/test/vstest.ProgrammerTests/Fakes/FakeMetricsPublisher.cs b/test/vstest.ProgrammerTests/Fakes/FakeMetricsPublisher.cs index acdec54735..39d06384f4 100644 --- a/test/vstest.ProgrammerTests/Fakes/FakeMetricsPublisher.cs +++ b/test/vstest.ProgrammerTests/Fakes/FakeMetricsPublisher.cs @@ -1,11 +1,10 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT license. See LICENSE file in the project root for full license information. -using Microsoft.VisualStudio.TestPlatform.CommandLine.Publisher; - -#pragma warning disable IDE1006 // Naming Styles namespace vstest.ProgrammerTests.Fakes; +using Microsoft.VisualStudio.TestPlatform.CommandLine.Publisher; + internal class FakeMetricsPublisher : IMetricsPublisher { public FakeMetricsPublisher(FakeErrorAggregator fakeErrorAggregator) diff --git a/test/vstest.ProgrammerTests/Fakes/FakeOutput.cs b/test/vstest.ProgrammerTests/Fakes/FakeOutput.cs index 9e254330fc..ce1d7fbe6e 100644 --- a/test/vstest.ProgrammerTests/Fakes/FakeOutput.cs +++ b/test/vstest.ProgrammerTests/Fakes/FakeOutput.cs @@ -9,10 +9,6 @@ namespace vstest.ProgrammerTests.Fakes; internal class FakeOutput : IOutput { - public FakeOutput() - { - } - public List Messages { get; } = new(); public StringBuilder CurrentLine { get; } = new(); public List Lines { get; } = new(); diff --git a/test/vstest.ProgrammerTests/Fakes/FakeProcessBuilder.cs b/test/vstest.ProgrammerTests/Fakes/FakeProcessBuilder.cs deleted file mode 100644 index 4780e5bed3..0000000000 --- a/test/vstest.ProgrammerTests/Fakes/FakeProcessBuilder.cs +++ /dev/null @@ -1,11 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT license. See LICENSE file in the project root for full license information. - -namespace vstest.ProgrammerTests.Fakes; - -internal class FakeProcessBuilder -{ - public FakeProcessBuilder() - { - } -} \ No newline at end of file diff --git a/test/vstest.ProgrammerTests/Fakes/FakeTestHostBuilder.cs b/test/vstest.ProgrammerTests/Fakes/FakeTestHostBuilder.cs index 33a9dfc201..7be196b79c 100644 --- a/test/vstest.ProgrammerTests/Fakes/FakeTestHostBuilder.cs +++ b/test/vstest.ProgrammerTests/Fakes/FakeTestHostBuilder.cs @@ -46,9 +46,11 @@ internal FakeTestHostFixture Build() var fakeCommunicationEndpoint = new FakeCommunicationEndpoint(fakeCommunicationChannel, _fixture.ErrorAggregator); var fakeTestRuntimeProvider = new FakeTestRuntimeProvider(_fixture.ProcessHelper, _process, _fixture.FileHelper, _dlls, fakeCommunicationEndpoint, _fixture.ErrorAggregator); +#if DEBUG // This registers the endpoint so we can look it up later using the address, the Id from here is propagated to // testhost connection info, and is used as port in 127.0.0.1:, address so we can lookup the correct channel. TestServiceLocator.Register(fakeCommunicationEndpoint.TestHostConnectionInfo.Endpoint, fakeCommunicationEndpoint); +# endif return new FakeTestHostFixture(id, _dlls, fakeTestRuntimeProvider, fakeCommunicationEndpoint, fakeCommunicationChannel, _process, _responses); } diff --git a/test/vstest.ProgrammerTests/Fakes/FakeTestHostLauncher.cs b/test/vstest.ProgrammerTests/Fakes/FakeTestHostLauncher.cs index 120c0d049d..8492a7bb4a 100644 --- a/test/vstest.ProgrammerTests/Fakes/FakeTestHostLauncher.cs +++ b/test/vstest.ProgrammerTests/Fakes/FakeTestHostLauncher.cs @@ -1,12 +1,11 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT license. See LICENSE file in the project root for full license information. +namespace vstest.ProgrammerTests.Fakes; + using Microsoft.VisualStudio.TestPlatform.ObjectModel; using Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.Interfaces; -#pragma warning disable IDE1006 // Naming Styles -namespace vstest.ProgrammerTests.Fakes; - internal class FakeTestHostLauncher : ITestHostLauncher { private readonly FakeProcessHelper _fakeProcessHelper; diff --git a/test/vstest.ProgrammerTests/Fakes/FakeTestHostProcess.cs b/test/vstest.ProgrammerTests/Fakes/FakeTestHostProcess.cs deleted file mode 100644 index 120170351d..0000000000 --- a/test/vstest.ProgrammerTests/Fakes/FakeTestHostProcess.cs +++ /dev/null @@ -1,17 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT license. See LICENSE file in the project root for full license information. - - -#pragma warning disable IDE1006 // Naming Styles - -namespace vstest.ProgrammerTests.Fakes; - -// TODO: was used in first test but is not correct design -//internal class FakeTestHostProcess : FakeProcess -//{ -// public FakeTestHostProcess(string commandLine, string arguments, FakeErrorAggregator fakeErrorAggregator) : base(commandLine, arguments, fakeErrorAggregator) -// { -// } - -// public CapturedRunSettings? RunSettings { get; internal set; } -//} diff --git a/test/vstest.ProgrammerTests/Fakes/FakeTestPlatformEventSource.cs b/test/vstest.ProgrammerTests/Fakes/FakeTestPlatformEventSource.cs index ed4b085149..cde02328ba 100644 --- a/test/vstest.ProgrammerTests/Fakes/FakeTestPlatformEventSource.cs +++ b/test/vstest.ProgrammerTests/Fakes/FakeTestPlatformEventSource.cs @@ -1,11 +1,10 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT license. See LICENSE file in the project root for full license information. -using Microsoft.VisualStudio.TestPlatform.CoreUtilities.Tracing.Interfaces; - -#pragma warning disable IDE1006 // Naming Styles namespace vstest.ProgrammerTests.Fakes; +using Microsoft.VisualStudio.TestPlatform.CoreUtilities.Tracing.Interfaces; + internal class FakeTestPlatformEventSource : ITestPlatformEventSource { public FakeTestPlatformEventSource(FakeErrorAggregator fakeErrorAggregator) diff --git a/test/vstest.ProgrammerTests/Fakes/FakeTestRunEventsRegistrar.cs b/test/vstest.ProgrammerTests/Fakes/FakeTestRunEventsRegistrar.cs index b9b24c1078..5610c9a9fd 100644 --- a/test/vstest.ProgrammerTests/Fakes/FakeTestRunEventsRegistrar.cs +++ b/test/vstest.ProgrammerTests/Fakes/FakeTestRunEventsRegistrar.cs @@ -1,13 +1,12 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT license. See LICENSE file in the project root for full license information. +namespace vstest.ProgrammerTests.Fakes; + using Microsoft.VisualStudio.TestPlatform.Common.Interfaces; using Microsoft.VisualStudio.TestPlatform.ObjectModel.Client; using Microsoft.VisualStudio.TestPlatform.ObjectModel.Logging; -#pragma warning disable IDE1006 // Naming Styles -namespace vstest.ProgrammerTests.Fakes; - internal class FakeTestRunEventsRegistrar : ITestRunEventsRegistrar { public FakeTestRunEventsRegistrar(FakeErrorAggregator fakeErrorAggregator) diff --git a/test/vstest.ProgrammerTests/Fakes/FakeTestRuntimeProviderManager.cs b/test/vstest.ProgrammerTests/Fakes/FakeTestRuntimeProviderManager.cs index 10f0257484..9668bd3f99 100644 --- a/test/vstest.ProgrammerTests/Fakes/FakeTestRuntimeProviderManager.cs +++ b/test/vstest.ProgrammerTests/Fakes/FakeTestRuntimeProviderManager.cs @@ -1,8 +1,6 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT license. See LICENSE file in the project root for full license information. -#pragma warning disable IDE1006 // Naming Styles - namespace vstest.ProgrammerTests.Fakes; using System.Collections.Concurrent; diff --git a/test/vstest.ProgrammerTests/Fakes/Fixture.cs b/test/vstest.ProgrammerTests/Fakes/Fixture.cs index 5fd7028b4e..ac2125d235 100644 --- a/test/vstest.ProgrammerTests/Fakes/Fixture.cs +++ b/test/vstest.ProgrammerTests/Fakes/Fixture.cs @@ -23,23 +23,26 @@ internal class Fixture : IDisposable public FakeFileHelper FileHelper { get; } public FakeTestRuntimeProviderManager TestRuntimeProviderManager { get; } public FakeTestRunEventsRegistrar TestRunEventsRegistrar { get; } - public TestEngine TestEngine { get; private set; } - public TestPlatform TestPlatform { get; private set; } - public TestRunResultAggregator TestRunResultAggregator { get; private set; } - public FakeTestPlatformEventSource TestPlatformEventSource { get; private set; } - public FakeAssemblyMetadataProvider AssemblyMetadataProvider { get; private set; } - public InferHelper InferHelper { get; private set; } - public FakeDataCollectorAttachmentsProcessorsFactory DataCollectorAttachmentsProcessorsFactory { get; private set; } - public TestRunAttachmentsProcessingManager TestRunAttachmentsProcessingManager { get; private set; } - public TestRequestManager TestRequestManager { get; private set; } + public TestEngine? TestEngine { get; private set; } + public TestPlatform? TestPlatform { get; private set; } + public TestRunResultAggregator? TestRunResultAggregator { get; private set; } + public FakeTestPlatformEventSource? TestPlatformEventSource { get; private set; } + public FakeAssemblyMetadataProvider? AssemblyMetadataProvider { get; private set; } + public InferHelper? InferHelper { get; private set; } + public FakeDataCollectorAttachmentsProcessorsFactory? DataCollectorAttachmentsProcessorsFactory { get; private set; } + public TestRunAttachmentsProcessingManager? TestRunAttachmentsProcessingManager { get; private set; } + public TestRequestManager? TestRequestManager { get; private set; } public List ExecutedTests => TestRunEventsRegistrar.RunChangedEvents.SelectMany(er => er.Data.NewTestResults).ToList(); public ProtocolConfig ProtocolConfig { get; internal set; } public Fixture() { +// This type is compiled only in DEBUG, and won't exist otherwise. +#if DEBUG // We need to use static class to find the communication endpoint, this clears all the registrations of previous tests. TestServiceLocator.Clear(); +#endif CurrentProcess = new FakeProcess(ErrorAggregator, @"X:\fake\vstest.console.exe", string.Empty, null, null, null, null, null); ProcessHelper = new FakeProcessHelper(ErrorAggregator, CurrentProcess); @@ -111,19 +114,3 @@ internal void AssertNoErrors() ErrorAggregator.Errors.Should().BeEmpty(); } } - -internal class DebugOptions -{ - public const int DefaultTimeout = 5; - // TODO: This setting is actually quite pointless, because I cannot come up with - // a useful way to abort quickly enough when debugger is attached and I am just running my tests (pressing F5) - // but at the same time not abort when I am in the middle of debugging some behavior. Maybe looking at debugger, - // and asking it if any breakpoints were hit / are set. But that is difficult. - // - // So normally I press F5 to investigate, but Ctrl+F5 (run without debugger), to run tests. - public const int DefaultDebugTimeout = 30 * 60; - public const bool DefaultBreakOnAbort = true; - public int Timeout { get; init; } = DefaultTimeout; - public int DebugTimeout { get; init; } = DefaultDebugTimeout; - public bool BreakOnAbort { get; init; } = DefaultBreakOnAbort; -} diff --git a/test/vstest.ProgrammerTests/Fakes/RequestResponsePair.cs b/test/vstest.ProgrammerTests/Fakes/RequestResponsePair.cs new file mode 100644 index 0000000000..5384f9cace --- /dev/null +++ b/test/vstest.ProgrammerTests/Fakes/RequestResponsePair.cs @@ -0,0 +1,38 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +namespace vstest.ProgrammerTests.Fakes; + +internal class RequestResponsePair where TRequest : class +{ + public RequestResponsePair(TRequest request, TResponse response, bool debug = false) + { + Request = request; + Responses = new List { response }; + Debug = debug; + } + + public RequestResponsePair(TRequest request, IEnumerable responses, bool debug = false) + { + Request = request; + Responses = responses.ToList(); + Debug = debug; + } + + public RequestResponsePair(TRequest request, IEnumerable responses, Action? beforeAction = null, Action? afterAction = null, bool debug = false) + { + Request = request; + Responses = responses.ToList(); + BeforeAction = beforeAction; + AfterAction = afterAction; + Debug = debug; + } + + public TRequest Request { get; } + + // TODO: make this Expression< so we can get some info about what this is doing when looking directly at this instance + public Action? BeforeAction { get; } + public Action? AfterAction { get; } + public List Responses { get; } + public bool Debug { get; } +} diff --git a/test/vstest.ProgrammerTests/UnitTest1.cs b/test/vstest.ProgrammerTests/UnitTest1.cs index dec07721fe..bf2d9a7997 100644 --- a/test/vstest.ProgrammerTests/UnitTest1.cs +++ b/test/vstest.ProgrammerTests/UnitTest1.cs @@ -73,8 +73,10 @@ public async Task GivenAnMSTestAssemblyWith108Tests_WhenTestsAreRun_Then108Tests var fakeCommunicationChannel = new FakeCommunicationChannel(responses, fakeErrorAggregator, 1); fakeCommunicationChannel.Start(new object()); var fakeCommunicationEndpoint = new FakeCommunicationEndpoint(fakeCommunicationChannel, fakeErrorAggregator); +#if DEBUG TestServiceLocator.Clear(); TestServiceLocator.Register(fakeCommunicationEndpoint.TestHostConnectionInfo.Endpoint, fakeCommunicationEndpoint); +#endif var fakeTestHostProcess = new FakeProcess(fakeErrorAggregator, @"C:\temp\testhost.exe"); var fakeTestRuntimeProvider = new FakeTestRuntimeProvider(fakeProcessHelper, fakeTestHostProcess, fakeFileHelper, mstest1Dll.AsList(), fakeCommunicationEndpoint, fakeErrorAggregator); var fakeTestRuntimeProviderManager = new FakeTestRuntimeProviderManager(fakeErrorAggregator); From 3c31acf5f3a636adba3b4e523e2a5ad1075ebaed Mon Sep 17 00:00:00 2001 From: nohwnd Date: Fri, 18 Feb 2022 15:58:59 +0100 Subject: [PATCH 26/32] Fix build issues --- test/vstest.ProgrammerTests/Fakes/DebugOptions.cs | 3 +++ .../vstest.ProgrammerTests/Fakes/FakeTestExtensionManager.cs | 5 ++--- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/test/vstest.ProgrammerTests/Fakes/DebugOptions.cs b/test/vstest.ProgrammerTests/Fakes/DebugOptions.cs index 8b5e254c5e..9e461e0093 100644 --- a/test/vstest.ProgrammerTests/Fakes/DebugOptions.cs +++ b/test/vstest.ProgrammerTests/Fakes/DebugOptions.cs @@ -1,7 +1,10 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT license. See LICENSE file in the project root for full license information. +#pragma warning disable IDE1006 // Naming Styles +// For some reason only this occurence of vstest is flagged in build, and no other. namespace vstest.ProgrammerTests.Fakes; +#pragma warning restore IDE1006 // Naming Styles internal class DebugOptions { diff --git a/test/vstest.ProgrammerTests/Fakes/FakeTestExtensionManager.cs b/test/vstest.ProgrammerTests/Fakes/FakeTestExtensionManager.cs index bddc6fe679..1f5adbbbf7 100644 --- a/test/vstest.ProgrammerTests/Fakes/FakeTestExtensionManager.cs +++ b/test/vstest.ProgrammerTests/Fakes/FakeTestExtensionManager.cs @@ -1,11 +1,10 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT license. See LICENSE file in the project root for full license information. -using Microsoft.VisualStudio.TestPlatform.ObjectModel.Engine; - -#pragma warning disable IDE1006 // Naming Styles namespace vstest.ProgrammerTests.Fakes; +using Microsoft.VisualStudio.TestPlatform.ObjectModel.Engine; + internal class FakeTestExtensionManager : ITestExtensionManager { public void ClearExtensions() From 5b12b0c1ea990be2261ded9ab7f093739bb2d697 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Jare=C5=A1?= Date: Mon, 21 Feb 2022 11:20:55 +0100 Subject: [PATCH 27/32] Apply suggestions from code review MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Amaury Levé --- .../TestRequestSender.cs | 1 - test/Intent.Primitives/TestResult.cs | 2 +- test/Intent/Extensions.cs | 10 +++++----- 3 files changed, 6 insertions(+), 7 deletions(-) diff --git a/src/Microsoft.TestPlatform.CommunicationUtilities/TestRequestSender.cs b/src/Microsoft.TestPlatform.CommunicationUtilities/TestRequestSender.cs index 342ecafcb7..e5c78bb33f 100644 --- a/src/Microsoft.TestPlatform.CommunicationUtilities/TestRequestSender.cs +++ b/src/Microsoft.TestPlatform.CommunicationUtilities/TestRequestSender.cs @@ -743,7 +743,6 @@ private ICommunicationEndPoint SetCommunicationEndPoint() } else { - EqtTrace.Verbose("TestRequestSender is acting as server."); return new SocketServer(); } diff --git a/test/Intent.Primitives/TestResult.cs b/test/Intent.Primitives/TestResult.cs index 5d6408d736..053167718f 100644 --- a/test/Intent.Primitives/TestResult.cs +++ b/test/Intent.Primitives/TestResult.cs @@ -5,7 +5,7 @@ namespace Intent; public enum TestResult { - None = 0, + None, Passed, Failed, Error, diff --git a/test/Intent/Extensions.cs b/test/Intent/Extensions.cs index 6ec98807d0..1406680ff2 100644 --- a/test/Intent/Extensions.cs +++ b/test/Intent/Extensions.cs @@ -25,11 +25,11 @@ public static List SkipNonPublic(this IEnumerable e) public static List SkipExcluded(this IEnumerable e) { return e.Where(i => - i.Name != nameof(object.ToString) - && i.Name != nameof(object.GetType) - && i.Name != nameof(object.GetHashCode) - && i.Name != nameof(object.Equals) - && i.GetCustomAttribute() == null).ToList(); + i.Name != nameof(object.ToString) + && i.Name != nameof(object.GetType) + && i.Name != nameof(object.GetHashCode) + && i.Name != nameof(object.Equals) + && i.GetCustomAttribute() == null).ToList(); } From da614470a9a3c61a006a541c3913ad3b17b5ece4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Jare=C5=A1?= Date: Mon, 21 Feb 2022 11:43:17 +0100 Subject: [PATCH 28/32] Apply suggestions from code review MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Amaury Levé --- test/vstest.ProgrammerTests/Fakes/FakeCommunicationChannel.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/vstest.ProgrammerTests/Fakes/FakeCommunicationChannel.cs b/test/vstest.ProgrammerTests/Fakes/FakeCommunicationChannel.cs index 3939a64c15..3d26712557 100644 --- a/test/vstest.ProgrammerTests/Fakes/FakeCommunicationChannel.cs +++ b/test/vstest.ProgrammerTests/Fakes/FakeCommunicationChannel.cs @@ -19,7 +19,7 @@ public FakeCommunicationChannel(int id) } public int Id { get; } - public CancellationTokenSource CancellationTokenSource = new(); + public CancellationTokenSource CancellationTokenSource { get; } = new(); public BlockingCollection InQueue { get; } = new(); public BlockingCollection OutQueue { get; } = new(); From 35c8809c24558b1cdc797ffb1b908eb4aa4c468a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Jare=C5=A1?= Date: Mon, 21 Feb 2022 11:49:58 +0100 Subject: [PATCH 29/32] Update test/Intent/Extensions.cs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Amaury Levé --- test/Intent/Extensions.cs | 2 -- 1 file changed, 2 deletions(-) diff --git a/test/Intent/Extensions.cs b/test/Intent/Extensions.cs index 1406680ff2..78a9bef318 100644 --- a/test/Intent/Extensions.cs +++ b/test/Intent/Extensions.cs @@ -31,6 +31,4 @@ public static List SkipExcluded(this IEnumerable e) && i.Name != nameof(object.Equals) && i.GetCustomAttribute() == null).ToList(); } - - } From 3f29dea21b324adffc9d2b42386e584e0c93bd72 Mon Sep 17 00:00:00 2001 From: nohwnd Date: Mon, 21 Feb 2022 12:29:36 +0100 Subject: [PATCH 30/32] Fix review comments --- test/Intent/Runner.cs | 34 +++++++++---------- .../Fakes/FakeAssemblyMetadataProvider.cs | 4 +-- .../Fakes/FakeFileHelper.cs | 20 ++++++++++- .../Fakes/FakeProcessHelper.cs | 5 ++- .../Fakes/FakeTestRuntimeProvider.cs | 2 +- test/vstest.ProgrammerTests/Fakes/Fixture.cs | 2 ++ .../Fakes/{Id.cs => SequentialId.cs} | 0 test/vstest.ProgrammerTests/UnitTest1.cs | 4 +++ 8 files changed, 47 insertions(+), 24 deletions(-) rename test/vstest.ProgrammerTests/Fakes/{Id.cs => SequentialId.cs} (100%) diff --git a/test/Intent/Runner.cs b/test/Intent/Runner.cs index e349dde222..3bab3db2c7 100644 --- a/test/Intent/Runner.cs +++ b/test/Intent/Runner.cs @@ -7,52 +7,52 @@ namespace Intent; public class Runner { - public static void Run(IEnumerable path, IRunLogger logger) + public static void Run(IEnumerable paths, IRunLogger logger) { - foreach (var p in path) + foreach (var path in paths) { try { - var asm = Assembly.LoadFrom(p); - if (asm.IsExcluded()) + var assembly = Assembly.LoadFrom(path); + if (assembly.IsExcluded()) continue; - var ts = asm.GetTypes().SkipNonPublic().SkipExcluded(); - foreach (var t in ts) + var types = assembly.GetTypes().SkipNonPublic().SkipExcluded(); + foreach (var type in types) { - var ms = t.GetMethods().SkipExcluded(); + var methods = type.GetMethods().SkipExcluded(); // TODO: This chooses the Only tests only for single assembly and single class, // to support this full we would have to enumerate all classes and methods first, // it is easy, I just don't need it right now. - var only = ms.Where(m => m.GetCustomAttribute() != null).ToList(); - if (only.Any()) - ms = only; + var methodsWithOnly = methods.Where(m => m.GetCustomAttribute() != null).ToList(); + if (methodsWithOnly.Count > 0) + methods = methodsWithOnly; - foreach (var m in ms) + foreach (var method in methods) { try { - var i = Activator.CreateInstance(t); - var result = m.Invoke(i, Array.Empty()); - if (result is Task task) + var instance = Activator.CreateInstance(type); + var testResult = method.Invoke(instance, Array.Empty()); + if (testResult is Task task) { // When the result is a task we need to await it. // TODO: this can be improved with await, imho task.GetAwaiter().GetResult(); }; - logger.WriteTestPassed(m); + logger.WriteTestPassed(method); } catch (Exception ex) { if (ex is TargetInvocationException tex && tex.InnerException != null) { - logger.WriteTestFailure(m, tex.InnerException); + logger.WriteTestFailure(method, tex.InnerException); } else { - logger.WriteTestFailure(m, ex); + logger.WriteTestFailure(method, ex); } } } diff --git a/test/vstest.ProgrammerTests/Fakes/FakeAssemblyMetadataProvider.cs b/test/vstest.ProgrammerTests/Fakes/FakeAssemblyMetadataProvider.cs index 9cf857b788..fef8088fd0 100644 --- a/test/vstest.ProgrammerTests/Fakes/FakeAssemblyMetadataProvider.cs +++ b/test/vstest.ProgrammerTests/Fakes/FakeAssemblyMetadataProvider.cs @@ -22,13 +22,13 @@ public FakeAssemblyMetadataProvider(FakeFileHelper fakeFileHelper, FakeErrorAggr public Architecture GetArchitecture(string filePath) { - var file = (FakeTestDllFile)FakeFileHelper.Files.Single(f => f.Path == filePath); + var file = FakeFileHelper.GetFakeFile(filePath); return file.Architecture; } public FrameworkName GetFrameWork(string filePath) { - var file = (FakeTestDllFile)FakeFileHelper.Files.Single(f => f.Path == filePath); + var file = FakeFileHelper.GetFakeFile(filePath); return file.FrameworkName; } } diff --git a/test/vstest.ProgrammerTests/Fakes/FakeFileHelper.cs b/test/vstest.ProgrammerTests/Fakes/FakeFileHelper.cs index 9a7cce020e..30ebab8f53 100644 --- a/test/vstest.ProgrammerTests/Fakes/FakeFileHelper.cs +++ b/test/vstest.ProgrammerTests/Fakes/FakeFileHelper.cs @@ -117,7 +117,7 @@ public void WriteAllTextToFile(string filePath, string content) throw new NotImplementedException(); } - internal void AddFile(T file) where T : FakeFile + internal void AddFakeFile(T file) where T : FakeFile { if (Files.Any(f => f.Path.Equals(file.Path, StringComparison.OrdinalIgnoreCase))) { @@ -126,4 +126,22 @@ internal void AddFile(T file) where T : FakeFile Files.Add(file); } + + internal T GetFakeFile(string path) where T : FakeFile + { + var matchingFiles = Files.Where(f => f.Path == path).ToList(); + if (matchingFiles.Count == 0) + throw new FileNotFoundException($"Fake file {path}, was not found. Check if file was previously added to FakeFileHelper."); + + // TODO: The public collection of files should probably be made readonly / immutable, and internally be made a concurrent dictionary, because it does not make + // sense to have more than 1 file object with the same name, and we check for that in AddFakeFile anyway. + if (matchingFiles.Count > 1) + throw new InvalidOperationException($"Fake file {path}, exists more than once. Are you modifying the Files collection in FakeFileHelper manually?"); + + var file = matchingFiles.Single(); + if (file is not T result) + throw new InvalidOperationException($"Fake file {path}, was supposed to be a {typeof(T)}, but was {file.GetType()}."); + + return result; + } } diff --git a/test/vstest.ProgrammerTests/Fakes/FakeProcessHelper.cs b/test/vstest.ProgrammerTests/Fakes/FakeProcessHelper.cs index 8ae7432228..2085202009 100644 --- a/test/vstest.ProgrammerTests/Fakes/FakeProcessHelper.cs +++ b/test/vstest.ProgrammerTests/Fakes/FakeProcessHelper.cs @@ -11,11 +11,10 @@ internal class FakeProcessHelper : IProcessHelper // starting from 100 for no particular reason // I want to avoid processId 0 and 1 as they are // "reserved" on Windows (0) and Linux (both 0 and 1) - private int _lastProcessId = 100; + private static readonly SequentialId IdSource = new(100); public FakeProcess CurrentProcess { get; } public List Processes { get; } = new(); - public int LastProcessId => _lastProcessId; public FakeErrorAggregator FakeErrorAggregator { get; } @@ -28,7 +27,7 @@ public FakeProcessHelper(FakeErrorAggregator fakeErrorAggregator, FakeProcess cu public void AddFakeProcess(FakeProcess process) { - var id = Interlocked.Increment(ref _lastProcessId); + var id = IdSource.Next(); process.SetId(id); Processes.Add(process); } diff --git a/test/vstest.ProgrammerTests/Fakes/FakeTestRuntimeProvider.cs b/test/vstest.ProgrammerTests/Fakes/FakeTestRuntimeProvider.cs index 4365e31a08..c68d50fdab 100644 --- a/test/vstest.ProgrammerTests/Fakes/FakeTestRuntimeProvider.cs +++ b/test/vstest.ProgrammerTests/Fakes/FakeTestRuntimeProvider.cs @@ -41,7 +41,7 @@ public FakeTestRuntimeProvider(FakeProcessHelper fakeProcessHelper, FakeProcess if (frameworks.Count > 1) throw new InvalidOperationException($"The provided dlls have more than 1 target framework {frameworks.JoinByComma()}. Fake TestRuntimeProvider cannot have dlls with mulitple target framework, because real testhost process can also run just a single target framework."); - fakeTestDlls.ForEach(FileHelper.AddFile); + fakeTestDlls.ForEach(FileHelper.AddFakeFile); fakeProcessHelper.AddFakeProcess(fakeTestHostProcess); TestHostProcess.ExitCallback = p => diff --git a/test/vstest.ProgrammerTests/Fakes/Fixture.cs b/test/vstest.ProgrammerTests/Fakes/Fixture.cs index ac2125d235..a8cf51d09a 100644 --- a/test/vstest.ProgrammerTests/Fakes/Fixture.cs +++ b/test/vstest.ProgrammerTests/Fakes/Fixture.cs @@ -42,6 +42,8 @@ public Fixture() #if DEBUG // We need to use static class to find the communication endpoint, this clears all the registrations of previous tests. TestServiceLocator.Clear(); +#else + throw new InvalidOperationException("Tests cannot run in Release mode, because TestServiceLocator is compiled only for Debug, and so the tests will fail to setup channel and will hang."); #endif CurrentProcess = new FakeProcess(ErrorAggregator, @"X:\fake\vstest.console.exe", string.Empty, null, null, null, null, null); diff --git a/test/vstest.ProgrammerTests/Fakes/Id.cs b/test/vstest.ProgrammerTests/Fakes/SequentialId.cs similarity index 100% rename from test/vstest.ProgrammerTests/Fakes/Id.cs rename to test/vstest.ProgrammerTests/Fakes/SequentialId.cs diff --git a/test/vstest.ProgrammerTests/UnitTest1.cs b/test/vstest.ProgrammerTests/UnitTest1.cs index bf2d9a7997..9b640e76a7 100644 --- a/test/vstest.ProgrammerTests/UnitTest1.cs +++ b/test/vstest.ProgrammerTests/UnitTest1.cs @@ -9,6 +9,8 @@ namespace vstest.ProgrammerTests; using FluentAssertions; using FluentAssertions.Extensions; +using Intent; + using Microsoft.VisualStudio.TestPlatform.Client; using Microsoft.VisualStudio.TestPlatform.CommandLine; using Microsoft.VisualStudio.TestPlatform.CommandLine.Publisher; @@ -76,6 +78,8 @@ public async Task GivenAnMSTestAssemblyWith108Tests_WhenTestsAreRun_Then108Tests #if DEBUG TestServiceLocator.Clear(); TestServiceLocator.Register(fakeCommunicationEndpoint.TestHostConnectionInfo.Endpoint, fakeCommunicationEndpoint); +#else + throw new InvalidOperationException("Tests cannot run in Release mode, because TestServiceLocator is compiled only for Debug, and so the tests will fail to setup channel and will hang."); #endif var fakeTestHostProcess = new FakeProcess(fakeErrorAggregator, @"C:\temp\testhost.exe"); var fakeTestRuntimeProvider = new FakeTestRuntimeProvider(fakeProcessHelper, fakeTestHostProcess, fakeFileHelper, mstest1Dll.AsList(), fakeCommunicationEndpoint, fakeErrorAggregator); From 1ec47bfff94d66bb298b348efb5627325f7e9d1f Mon Sep 17 00:00:00 2001 From: nohwnd Date: Mon, 21 Feb 2022 12:31:25 +0100 Subject: [PATCH 31/32] Remove empty line --- test/Intent/ConsoleLogger.cs | 1 - 1 file changed, 1 deletion(-) diff --git a/test/Intent/ConsoleLogger.cs b/test/Intent/ConsoleLogger.cs index 45d9dd5805..b41c484532 100644 --- a/test/Intent/ConsoleLogger.cs +++ b/test/Intent/ConsoleLogger.cs @@ -54,5 +54,4 @@ private static string FormatMethodName(string methodName) return newLines.ToLowerInvariant(); } - } From 30752dcc9b9a63ba663577b1370d97193b28cc44 Mon Sep 17 00:00:00 2001 From: nohwnd Date: Mon, 21 Feb 2022 12:47:57 +0100 Subject: [PATCH 32/32] Fix errors in release --- test/vstest.ProgrammerTests/Fakes/Fixture.cs | 9 +++++++-- test/vstest.ProgrammerTests/UnitTest1.cs | 14 ++++++++++---- 2 files changed, 17 insertions(+), 6 deletions(-) diff --git a/test/vstest.ProgrammerTests/Fakes/Fixture.cs b/test/vstest.ProgrammerTests/Fakes/Fixture.cs index a8cf51d09a..e8b66c1c4f 100644 --- a/test/vstest.ProgrammerTests/Fakes/Fixture.cs +++ b/test/vstest.ProgrammerTests/Fakes/Fixture.cs @@ -38,12 +38,17 @@ internal class Fixture : IDisposable public Fixture() { -// This type is compiled only in DEBUG, and won't exist otherwise. + // This type is compiled only in DEBUG, and won't exist otherwise. #if DEBUG // We need to use static class to find the communication endpoint, this clears all the registrations of previous tests. TestServiceLocator.Clear(); #else - throw new InvalidOperationException("Tests cannot run in Release mode, because TestServiceLocator is compiled only for Debug, and so the tests will fail to setup channel and will hang."); + // This fools compiler into not being able to tell that the the rest of the code is unreachable. + var a = true; + if (a) + { + throw new InvalidOperationException("Tests cannot run in Release mode, because TestServiceLocator is compiled only for Debug, and so the tests will fail to setup channel and will hang."); + } #endif CurrentProcess = new FakeProcess(ErrorAggregator, @"X:\fake\vstest.console.exe", string.Empty, null, null, null, null, null); diff --git a/test/vstest.ProgrammerTests/UnitTest1.cs b/test/vstest.ProgrammerTests/UnitTest1.cs index 9b640e76a7..61ca59cd84 100644 --- a/test/vstest.ProgrammerTests/UnitTest1.cs +++ b/test/vstest.ProgrammerTests/UnitTest1.cs @@ -9,20 +9,21 @@ namespace vstest.ProgrammerTests; using FluentAssertions; using FluentAssertions.Extensions; -using Intent; - using Microsoft.VisualStudio.TestPlatform.Client; using Microsoft.VisualStudio.TestPlatform.CommandLine; using Microsoft.VisualStudio.TestPlatform.CommandLine.Publisher; using Microsoft.VisualStudio.TestPlatform.CommandLine.TestPlatformHelpers; using Microsoft.VisualStudio.TestPlatform.CommandLineUtilities; -using Microsoft.VisualStudio.TestPlatform.CommunicationUtilities.Interfaces; using Microsoft.VisualStudio.TestPlatform.CommunicationUtilities.ObjectModel; using Microsoft.VisualStudio.TestPlatform.CrossPlatEngine; using Microsoft.VisualStudio.TestPlatform.CrossPlatEngine.TestRunAttachmentsProcessing; using Microsoft.VisualStudio.TestPlatform.ObjectModel; using Microsoft.VisualStudio.TestPlatform.ObjectModel.Client; +# if DEBUG +using Microsoft.VisualStudio.TestPlatform.CommunicationUtilities.Interfaces; +#endif + using vstest.ProgrammerTests.Fakes; // Tests are run by Intent library that is executed from our Program.Main. To debug press F5 in VS, and maybe mark just a single test with [Only]. @@ -79,7 +80,12 @@ public async Task GivenAnMSTestAssemblyWith108Tests_WhenTestsAreRun_Then108Tests TestServiceLocator.Clear(); TestServiceLocator.Register(fakeCommunicationEndpoint.TestHostConnectionInfo.Endpoint, fakeCommunicationEndpoint); #else - throw new InvalidOperationException("Tests cannot run in Release mode, because TestServiceLocator is compiled only for Debug, and so the tests will fail to setup channel and will hang."); + // This fools compiler into not being able to tell that the the rest of the code is unreachable. + var a = true; + if (a) + { + throw new InvalidOperationException("Tests cannot run in Release mode, because TestServiceLocator is compiled only for Debug, and so the tests will fail to setup channel and will hang."); + } #endif var fakeTestHostProcess = new FakeProcess(fakeErrorAggregator, @"C:\temp\testhost.exe"); var fakeTestRuntimeProvider = new FakeTestRuntimeProvider(fakeProcessHelper, fakeTestHostProcess, fakeFileHelper, mstest1Dll.AsList(), fakeCommunicationEndpoint, fakeErrorAggregator);