Use RunSettingsFilePath from project file when using dotnet test

TestPlatform.sln

@@ -179,7 +179,9 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.TestPlatform.Exte
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.TestPlatform.Extensions.HtmlLogger.UnitTests", "test\Microsoft.TestPlatform.Extensions.HtmlLogger.UnitTests\Microsoft.TestPlatform.Extensions.HtmlLogger.UnitTests.csproj", "{41248B96-6E15-4E5E-A78F-859897676814}"
 EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "coverlet.collector", "test\TestAssets\coverlet.collector\coverlet.collector.csproj", "{F1D8630D-97D5-4CD7-BC18-A5E1779FA6E3}"
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "coverlet.collector", "test\TestAssets\coverlet.collector\coverlet.collector.csproj", "{F1D8630D-97D5-4CD7-BC18-A5E1779FA6E3}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ProjectFileRunSettingsTestProject", "test\TestAssets\ProjectFileRunSettingsTestProject\ProjectFileRunSettingsTestProject.csproj", "{8E87F6E4-E884-4404-B2E5-CBFB28038AE5}"
 EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
@@ -911,6 +913,18 @@ Global
 		{F1D8630D-97D5-4CD7-BC18-A5E1779FA6E3}.Release|x64.Build.0 = Release|Any CPU
 		{F1D8630D-97D5-4CD7-BC18-A5E1779FA6E3}.Release|x86.ActiveCfg = Release|Any CPU
 		{F1D8630D-97D5-4CD7-BC18-A5E1779FA6E3}.Release|x86.Build.0 = Release|Any CPU
+		{8E87F6E4-E884-4404-B2E5-CBFB28038AE5}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{8E87F6E4-E884-4404-B2E5-CBFB28038AE5}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{8E87F6E4-E884-4404-B2E5-CBFB28038AE5}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{8E87F6E4-E884-4404-B2E5-CBFB28038AE5}.Debug|x64.Build.0 = Debug|Any CPU
+		{8E87F6E4-E884-4404-B2E5-CBFB28038AE5}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{8E87F6E4-E884-4404-B2E5-CBFB28038AE5}.Debug|x86.Build.0 = Debug|Any CPU
+		{8E87F6E4-E884-4404-B2E5-CBFB28038AE5}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{8E87F6E4-E884-4404-B2E5-CBFB28038AE5}.Release|Any CPU.Build.0 = Release|Any CPU
+		{8E87F6E4-E884-4404-B2E5-CBFB28038AE5}.Release|x64.ActiveCfg = Release|Any CPU
+		{8E87F6E4-E884-4404-B2E5-CBFB28038AE5}.Release|x64.Build.0 = Release|Any CPU
+		{8E87F6E4-E884-4404-B2E5-CBFB28038AE5}.Release|x86.ActiveCfg = Release|Any CPU
+		{8E87F6E4-E884-4404-B2E5-CBFB28038AE5}.Release|x86.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
@@ -988,6 +1002,7 @@ Global
 		{236A71E3-01DA-4679-9DFF-16A8E079ACFF} = {5E7F18A8-F843-4C8A-AB02-4C7D9205C6CF}
 		{41248B96-6E15-4E5E-A78F-859897676814} = {020E15EA-731F-4667-95AF-226671E0C3AE}
 		{F1D8630D-97D5-4CD7-BC18-A5E1779FA6E3} = {8DA7CBD9-F17E-41B6-90C4-CFF55848A25A}
+		{8E87F6E4-E884-4404-B2E5-CBFB28038AE5} = {8DA7CBD9-F17E-41B6-90C4-CFF55848A25A}
 	EndGlobalSection
 	GlobalSection(ExtensibilityGlobals) = postSolution
 		SolutionGuid = {0541B30C-FF51-4E28-B172-83F5F3934BCD}

scripts/build.ps1

@@ -213,6 +213,19 @@ function Invoke-Build
     Write-Log "Invoke-Build: Complete. {$(Get-ElapsedTime($timer))}"
 }
 
+function Publish-PatchedDotnet { 
+    Write-Log "Publish-PatchedDotnet: Copy local dotnet installation to testArtifacts"
+    $dotnetPath = "$env:TP_TOOLS_DIR\dotnet\"
+
+    $dotnetTestArtifactsPath = "$env:TP_TESTARTIFACTS\dotnet\" 
+    $dotnetTestArtifactsSdkPath = "$env:TP_TESTARTIFACTS\dotnet\sdk\$env:DOTNET_CLI_VERSION\"
+    Copy-Item $dotnetPath $dotnetTestArtifactsPath -Force -Recurse
+
+    Write-Log "Publish-PatchedDotnet: Copy VSTest task artifacts to local dotnet installation to allow `dotnet test` to run with it"
+    $buildArtifactsPath = "$env:TP_ROOT_DIR\src\Microsoft.TestPlatform.Build\bin\$TPB_Configuration\$TPB_TargetFrameworkNS2_0\*"
+    Copy-Item $buildArtifactsPath $dotnetTestArtifactsSdkPath -Force
+}
+
 function Publish-Package
 {
     $timer = Start-Timer
@@ -901,9 +914,11 @@ Restore-Package
 Update-LocalizedResources
 Invoke-Build
 Publish-Package
+Publish-PatchedDotnet
 Publish-Tests
 Create-VsixPackage
 Create-NugetPackages
 Generate-Manifest
 Write-Log "Build complete. {$(Get-ElapsedTime($timer))}"
 if ($Script:ScriptFailed) { Exit 1 } else { Exit 0 }
+

src/Microsoft.TestPlatform.Build/Microsoft.TestPlatform.targets

@@ -31,7 +31,7 @@ Copyright (c) .NET Foundation. All rights reserved.
 
     <Microsoft.TestPlatform.Build.Tasks.VSTestTask
       TestFileFullPath="$(TargetPath)"
-      VSTestSetting="$(VSTestSetting)"
+      VSTestSetting="$([MSBuild]::ValueOrDefault($(VSTestSetting), '$(RunSettingsFilePath)'))"
       VSTestTestAdapterPath="$(VSTestTestAdapterPath)"
       VSTestFramework="$(TargetFrameworkMoniker)"
       VSTestPlatform="$(PlatformTarget)"

test/Microsoft.TestPlatform.AcceptanceTests/RunsettingsTests.cs

@@ -478,6 +478,37 @@ public void EnvironmentVariablesSettingsShouldSetEnvironmentVariables(RunnerInfo
 
         #endregion
 
+        #region RunSettings defined in project file
+        /// <summary>
+        /// RunSettingsFilePath can be specified in .csproj and should be honored by `dotnet test`, this test 
+        /// checks that the settings were honored by translating an inconlusive test to failed "result", instead of the default "skipped".
+        /// This test depends on Microsoft.TestPlatform.Build\Microsoft.TestPlatform.targets being previously copied into the 
+        /// artifacts/testArtifacts/dotnet folder. This will allow the local copy of dotnet to pickup the VSTest msbuild task.
+        /// </summary>
+        /// <param name="runnerInfo"></param>
+        [TestMethod]
+        [NetFullTargetFrameworkDataSource]
+        [NetCoreTargetFrameworkDataSource]
+        public void RunSettingsAreLoadedFromProject(RunnerInfo runnerInfo)
+        {
+            AcceptanceTestBase.SetTestEnvironment(this.testEnvironment, runnerInfo);
+
+            var projectName = "ProjectFileRunSettingsTestProject.csproj";
+            var projectPath = this.GetProjectFullPath(projectName);
+            this.InvokeDotnetTest(projectPath);
+            this.ValidateSummaryStatus(0, 1, 0);
+
+            // make sure that we can revert the project settings back by providing a config from commandline 
+            // keeping this in the same test, because it is easier to see that we are reverting settings that 
+            // are honored by dotnet test, instead of just using the default, which would produce the same 
+            // result
+            var settingsPath = this.GetProjectAssetFullPath(projectName, "inconclusive.runsettings");
+            this.InvokeDotnetTest($"{projectPath} --settings {settingsPath}");
+            this.ValidateSummaryStatus(0, 0, 1);
+        }
+
+        #endregion
+
         private string GetRunsettingsFilePath(Dictionary<string, string> runConfigurationDictionary)
         {
             var runsettingsPath = Path.Combine(

test/Microsoft.TestPlatform.TestUtilities/IntegrationTestBase.cs

@@ -106,7 +106,18 @@ public static string PrepareArguments(string testAssembly, string testAdapterPat
         /// <param name="arguments">Arguments provided to <c>vstest.console</c>.exe</param>
         public void InvokeVsTest(string arguments)
         {
-            this.Execute(arguments, out this.standardTestOutput, out this.standardTestError, out this.runnerExitCode);
+            this.ExecuteVsTestConsole(arguments, out this.standardTestOutput, out this.standardTestError, out this.runnerExitCode);
+            this.FormatStandardOutCome();
+        }
+
+
+        /// <summary>
+        /// Invokes <c>vstest.console</c> with specified arguments.
+        /// </summary>
+        /// <param name="arguments">Arguments provided to <c>vstest.console</c>.exe</param>
+        public void InvokeDotnetTest(string arguments)
+        {
+            this.ExecutePatchedDotnet("test", arguments, out this.standardTestOutput, out this.standardTestError, out this.runnerExitCode);
             this.FormatStandardOutCome();
         }
 
@@ -353,6 +364,17 @@ protected string GetAssetFullPath(string assetName, string targetFramework)
             return this.testEnvironment.GetTestAsset(assetName, targetFramework);
         }
 
+        protected string GetProjectFullPath(string projectName)
+        {
+            return this.testEnvironment.GetTestProject(projectName);
+        }
+
+        protected string GetProjectAssetFullPath(string projectName, string assetName)
+        {
+            var projectPath = this.testEnvironment.GetTestProject(projectName);
+            return Path.Combine(Path.GetDirectoryName(projectPath), assetName);
+        } 
+
         protected string GetTestAdapterPath(UnitTestFramework testFramework = UnitTestFramework.MSTest)
         {
             string adapterRelativePath = string.Empty;
@@ -478,7 +500,7 @@ private static string GetTestMethodName(string testFullName)
             return testMethodName;
         }
 
-        private void Execute(string args, out string stdOut, out string stdError, out int exitCode)
+        private void ExecuteVsTestConsole(string args, out string stdOut, out string stdError, out int exitCode)
         {
             if (this.IsNetCoreRunner())
             {
@@ -487,46 +509,88 @@ private void Execute(string args, out string stdOut, out string stdError, out in
 
             this.arguments = args;
 
-            using (Process vstestconsole = new Process())
+            this.ExecuteApplication(this.GetConsoleRunnerPath(), args, out stdOut, out stdError, out exitCode);
+        }
+
+        /// <summary>
+        /// Executes a local copy of dotnet that has VSTest task installed and possibly other modifications. Do not use this to 
+        /// do your builds or to run general tests, unless you want your changes to be reflected.
+        /// </summary>
+        /// <param name="command"></param>
+        /// <param name="args"></param>
+        /// <param name="stdOut"></param>
+        /// <param name="stdError"></param>
+        /// <param name="exitCode"></param>
+        private void ExecutePatchedDotnet(string command, string args, out string stdOut, out string stdError, out int exitCode)
+        {
+            var environmentVariables = new Dictionary<string, string> {
+                ["DOTNET_MULTILEVEL_LOOKUP"] = "0" 
+            };
+
+            var patchedDotnetPath = Path.Combine(this.testEnvironment.TestArtifactsDirectory, @"dotnet\dotnet.exe"); ;
+            this.ExecuteApplication(patchedDotnetPath, string.Join(" ", command, args), out stdOut, out stdError, out exitCode, environmentVariables);
+        }
+
+        private void ExecuteApplication(string path, string args, out string stdOut, out string stdError, out int exitCode, Dictionary<string, string> environmentVariables = null)
+        {
+            if (string.IsNullOrWhiteSpace(path))
+            {
+                throw new ArgumentException("Executable path must not be null or whitespace.", nameof(path));
+            }
+
+            var executableName = Path.GetFileName(path);
+
+            using (Process process = new Process())
             {
-                Console.WriteLine("IntegrationTestBase.Execute: Starting vstest.console.exe");
-                vstestconsole.StartInfo.FileName = this.GetConsoleRunnerPath();
-                vstestconsole.StartInfo.Arguments = args;
-                vstestconsole.StartInfo.UseShellExecute = false;
+                Console.WriteLine($"IntegrationTestBase.Execute: Starting {executableName}");
+                process.StartInfo.FileName = path;
+                process.StartInfo.Arguments = args;
+                process.StartInfo.UseShellExecute = false;
                 //vstestconsole.StartInfo.WorkingDirectory = testEnvironment.PublishDirectory;
-                vstestconsole.StartInfo.RedirectStandardError = true;
-                vstestconsole.StartInfo.RedirectStandardOutput = true;
-                vstestconsole.StartInfo.CreateNoWindow = true;
-                vstestconsole.StartInfo.StandardOutputEncoding = Encoding.UTF8;
-                vstestconsole.StartInfo.StandardErrorEncoding = Encoding.UTF8;
+                process.StartInfo.RedirectStandardError = true;
+                process.StartInfo.RedirectStandardOutput = true;
+                process.StartInfo.CreateNoWindow = true;
+                process.StartInfo.StandardOutputEncoding = Encoding.UTF8;
+                process.StartInfo.StandardErrorEncoding = Encoding.UTF8;
+                if (environmentVariables != null) {
+                    foreach (var variable in environmentVariables) {
+                        if (process.StartInfo.EnvironmentVariables.ContainsKey(variable.Key)) {
+                            process.StartInfo.EnvironmentVariables[variable.Key] = variable.Value;
+                        } 
+                        else
+                        {
+                            process.StartInfo.EnvironmentVariables.Add(variable.Key, variable.Value);
+                        }
+                    }
+                }
 
                 var stdoutBuffer = new StringBuilder();
                 var stderrBuffer = new StringBuilder();
-                vstestconsole.OutputDataReceived += (sender, eventArgs) =>
+                process.OutputDataReceived += (sender, eventArgs) =>
                 {
                     stdoutBuffer.Append(eventArgs.Data).Append(Environment.NewLine);
                 };
 
-                vstestconsole.ErrorDataReceived += (sender, eventArgs) => stderrBuffer.Append(eventArgs.Data).Append(Environment.NewLine);
+                process.ErrorDataReceived += (sender, eventArgs) => stderrBuffer.Append(eventArgs.Data).Append(Environment.NewLine);
 
-                Console.WriteLine("IntegrationTestBase.Execute: Path = {0}", vstestconsole.StartInfo.FileName);
-                Console.WriteLine("IntegrationTestBase.Execute: Arguments = {0}", vstestconsole.StartInfo.Arguments);
+                Console.WriteLine("IntegrationTestBase.Execute: Path = {0}", process.StartInfo.FileName);
+                Console.WriteLine("IntegrationTestBase.Execute: Arguments = {0}", process.StartInfo.Arguments);
 
                 Stopwatch stopwatch = new Stopwatch();
                 stopwatch.Start();
 
-                vstestconsole.Start();
-                vstestconsole.BeginOutputReadLine();
-                vstestconsole.BeginErrorReadLine();
-                if (!vstestconsole.WaitForExit(80 * 1000))
+                process.Start();
+                process.BeginOutputReadLine();
+                process.BeginErrorReadLine();
+                if (!process.WaitForExit(80 * 1000))
                 {
-                    Console.WriteLine("IntegrationTestBase.Execute: Timed out waiting for vstest.console.exe. Terminating the process.");
-                    vstestconsole.Kill();
+                    Console.WriteLine($"IntegrationTestBase.Execute: Timed out waiting for {executableName}. Terminating the process.");
+                    process.Kill();
                 }
                 else
                 {
                     // Ensure async buffers are flushed
-                    vstestconsole.WaitForExit();
+                    process.WaitForExit();
                 }
 
                 stopwatch.Stop();
@@ -535,11 +599,11 @@ private void Execute(string args, out string stdOut, out string stdError, out in
 
                 stdError = stderrBuffer.ToString();
                 stdOut = stdoutBuffer.ToString();
-                exitCode = vstestconsole.ExitCode;
+                exitCode = process.ExitCode;
 
                 Console.WriteLine("IntegrationTestBase.Execute: stdError = {0}", stdError);
                 Console.WriteLine("IntegrationTestBase.Execute: stdOut = {0}", stdOut);
-                Console.WriteLine("IntegrationTestBase.Execute: Stopped vstest.console.exe. Exit code = {0}", exitCode);
+                Console.WriteLine($"IntegrationTestBase.Execute: Stopped {executableName}. Exit code = {0}", exitCode);
             }
         }
 

test/Microsoft.TestPlatform.TestUtilities/IntegrationTestEnvironment.cs

@@ -57,6 +57,7 @@ public IntegrationTestEnvironment()
             // Need to remove this assumption when we move to a CDP.
             this.PackageDirectory = Path.Combine(TestPlatformRootDirectory, @"packages");
             this.ToolsDirectory = Path.Combine(TestPlatformRootDirectory, @"tools");
+            this.TestArtifactsDirectory = Path.Combine(TestPlatformRootDirectory, "artifacts", "testArtifacts");
             this.RunnerFramework = "net451";
         }
 
@@ -193,6 +194,15 @@ public string ToolsDirectory
             private set;
         }
 
+        /// <summary>
+        /// Gets the test artifacts directory.
+        /// </summary>
+        public string TestArtifactsDirectory
+        {
+            get;
+            private set;
+        }
+
         /// <summary>
         /// Gets the application type.
         /// Supported values = <c>net451</c>, <c>netcoreapp1.0</c>.
@@ -293,5 +303,30 @@ private static Dictionary<string, string> GetDependencies(string testPlatformRoo
 
             return dependencyProps;
         }
+
+        /// <summary>
+        /// Gets the full path to a test asset.
+        /// </summary>
+        /// <param name="assetName">Name of the asset with extension. E.g. <c>SimpleUnitTest.csproj</c></param>
+        /// <returns>Full path to the test asset.</returns>
+        /// <remarks>
+        /// Test assets follow several conventions:
+        /// (a) They are built for supported frameworks. See <see cref="TargetFramework"/>.
+        /// (b) They are built for provided build configuration.
+        /// (c) Name of the test asset matches the parent directory name. E.g. <c>TestAssets\SimpleUnitTest\SimpleUnitTest.csproj</c> must
+        /// produce <c>TestAssets\SimpleUnitTest\SimpleUnitTest.csproj</c>
+        /// </remarks>
+        public string GetTestProject(string assetName)
+        {
+            var simpleAssetName = Path.GetFileNameWithoutExtension(assetName);
+            var assetPath = Path.Combine(
+                this.TestAssetsPath,
+                simpleAssetName,
+                assetName);
+
+            Assert.IsTrue(File.Exists(assetPath), "GetTestAsset: Path not found: {0}.", assetPath);
+
+            return assetPath;
+        }
     }
 }

test/TestAssets/ProjectFileRunSettingsTestProject/ProjectFileRunSettingsTestProject.csproj

@@ -0,0 +1,31 @@
+<?xml version="1.0" encoding="utf-8"?>
+<Project Sdk="Microsoft.NET.Sdk">
+  <!-- Imports Common TestAssets props. -->
+  <Import Project="..\..\..\scripts\build\TestAssets.props" />
+  <Import Project="..\..\..\scripts\build\TestPlatform.Dependencies.props" />
+  <!-- Package dependency versions -->
+
+  <PropertyGroup>
+    <TargetFrameworks>netcoreapp2.1;net451</TargetFrameworks>
+    <PlatformTarget>x64</PlatformTarget>
+    <RunSettingsFilePath>fail.runsettings</RunSettingsFilePath>
+  </PropertyGroup>
+  <ItemGroup>
+    <PackageReference Include="MSTest.TestFramework">
+      <Version>$(MSTestFrameworkVersion)</Version>
+    </PackageReference>
+    <PackageReference Include="MSTest.TestAdapter">
+      <Version>$(MSTestAdapterVersion)</Version>
+    </PackageReference>
+    <PackageReference Include="Microsoft.NET.Test.Sdk">
+      <Version>$(NETTestSdkPreviousVersion)</Version>
+    </PackageReference>
+  </ItemGroup>
+  <ItemGroup Condition=" '$(TargetFramework)' == 'net451' ">
+    <Reference Include="System.Runtime" />
+    <Reference Include="System.Windows.Forms" />
+  </ItemGroup>
+  <ItemGroup>
+    <Service Include="{82a7f48d-3b50-4b1e-b82e-3ada8210c358}" />
+  </ItemGroup>
+</Project>

test/TestAssets/ProjectFileRunSettingsTestProject/UnitTest1.cs

@@ -0,0 +1,20 @@
+using Microsoft.VisualStudio.TestTools.UnitTesting;
+
+namespace ProjectFileRunSettingsTestProject
+{
+    [TestClass]
+    public class UnitTest1
+    {
+        [TestMethod]
+        public void TestMethod1()
+        {
+            // this project specifies runsettings in it's proj file
+            // that runsettings say that inconclusive should translate to 
+            // failed. 
+            // we can then easily figure out if the settings were applied 
+            // correctly if we set the test as failed, or did not apply if the 
+            // test is shown as skipped
+            Assert.Inconclusive();
+        }
+    }
+}