Add xunit perf test project. (microsoft#496)

* Add xunit perf test project. Extract common dependencies to a props. * Remove props from gitignore. Add Dependencies file. * Add package id for OM to prefer project reference. * Add two flags LocalizeResources and LocalizedBuild. Don't include all resx files if LocalizedBuild is false to speed up build. * Pick up MSTestAdapterPath from dependency props file in test.ps1. * Add chutzpah as a dependency. Read dependencies from props file in integration tests. Publish testhost to package directory for net46 since it is included in the runner package.
navin22 · Feb 27, 2017 · 4169206 · 4169206
1 parent e8f32e2
commit 4169206
Show file tree

Hide file tree

Showing 22 changed files with 60,585 additions and 467 deletions.
diff --git a/TestPlatform.sln b/TestPlatform.sln
diff --git a/scripts/build.ps1 b/scripts/build.ps1
@@ -168,7 +168,7 @@ function Invoke-Build
 
     Write-Log ".. .. Build: Source: $TPB_Solution"
     Write-Verbose "$dotnetExe build $TPB_Solution --configuration $TPB_Configuration --version-suffix $TPB_VersionSuffix -v:minimal -p:Version=$TPB_Version"
-    & $dotnetExe build $TPB_Solution --configuration $TPB_Configuration --version-suffix $TPB_VersionSuffix -v:minimal -p:Version=$TPB_Version -p:CIBuild=$TPB_CIBuild
+    & $dotnetExe build $TPB_Solution --configuration $TPB_Configuration --version-suffix $TPB_VersionSuffix -v:minimal -p:Version=$TPB_Version -p:CIBuild=$TPB_CIBuild -p:LocalizedBuild=$TPB_LocalizedBuild
     Write-Log ".. .. Build: Complete."
 
     if ($lastExitCode -ne 0) {
@@ -266,7 +266,7 @@ function Publish-Package
 function Publish-PackageInternal($packagename, $framework, $output)
 {
     Write-Verbose "$dotnetExe publish $packagename --configuration $TPB_Configuration --framework $framework --output $output -v:minimal"
-    & $dotnetExe publish $packagename --configuration $TPB_Configuration --framework $framework --output $output -v:minimal
+    & $dotnetExe publish $packagename --configuration $TPB_Configuration --framework $framework --output $output -v:minimal -p:LocalizedBuild=$TPB_LocalizedBuild
 }
 
 function Create-VsixPackage
@@ -373,8 +373,8 @@ function Update-LocalizedResources
     }
 
     $localizationProject = Join-Path $env:TP_PACKAGE_PROJ_DIR "Localize\Localize.proj"
-    Write-Verbose "& $dotnetExe msbuild $localizationProject -m -nologo -v:minimal -t:Localize"
-    & $dotnetExe msbuild $localizationProject -m -nologo -v:minimal -t:Localize
+    Write-Verbose "& $dotnetExe msbuild $localizationProject -m -nologo -v:minimal -t:Localize -p:LocalizeResources=true"
+    & $dotnetExe msbuild $localizationProject -m -nologo -v:minimal -t:Localize -p:LocalizeResources=true
 
     if ($lastExitCode -ne 0) {
         Set-ScriptFailed

diff --git a/scripts/build/TestPlatform.Dependencies.props b/scripts/build/TestPlatform.Dependencies.props
@@ -0,0 +1,18 @@
+<?xml version="1.0" encoding="utf-8"?>
+<Project ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+  <PropertyGroup>
+    <!-- Name of the elements must be in sync with test\Microsoft.TestPlatform.TestUtilities\IntegrationTestBase.cs -->
+    <NETTestSdkPreviousVersion>15.0.0-preview-20170106-08</NETTestSdkPreviousVersion>
+
+    <MSTestFrameworkVersion>1.0.8-rc2</MSTestFrameworkVersion>
+    <MSTestAdapterVersion>1.1.10-rc2</MSTestAdapterVersion>
+
+    <XUnitFrameworkVersion>2.2.0</XUnitFrameworkVersion>
+    <XUnitAdapterVersion>2.2.0</XUnitAdapterVersion>
+
+    <NUnitFrameworkVersion>2.6.4</NUnitFrameworkVersion>
+    <NUnitAdapterVersion>2.0.0</NUnitAdapterVersion>
+
+    <ChutzpahAdapterVersion>4.2.4</ChutzpahAdapterVersion>
+  </PropertyGroup>
+</Project>
diff --git a/scripts/build/TestPlatform.Localization.targets b/scripts/build/TestPlatform.Localization.targets
@@ -5,17 +5,32 @@
     <TestPlatformPackageDir>$(TestPlatformRoot)packages\</TestPlatformPackageDir>
   </PropertyGroup>
 
+  <PropertyGroup>
+    <ResourceDirectory>$(ProjectDir)Resources</ResourceDirectory>
+    <ProductProject Condition="'$(TestProject)' == '' and '$(TestProject)' != 'true'">true</ProductProject>
+
+    <!-- LocalizeResources is set only when xlf files are synced or resx are generated -->
+    <LocalizeResources Condition="'$(LocalizeResources)' == ''">false</LocalizeResources>
+
+    <!-- LocalizedBuild is set when non english resources are included in build process -->
+    <LocalizedBuild Condition="'$(LocalizedBuild)' == ''">false</LocalizedBuild>
+  </PropertyGroup>
+
   <ItemGroup>
     <ResxLang Include="cs;de;es;fr;it;ja;ko;pl;pt-BR;ru;tr;zh-Hans;zh-Hant" />
   </ItemGroup>
 
-  <PropertyGroup>
-    <ResourceDirectory>$(ProjectDir)Resources</ResourceDirectory>
-  </PropertyGroup>
+  <!-- In case of LocalizedBuild, EmbeddedResource contains just english resources to allow sync and
+       generation. Otherwise, let EmbeddedResource contain all resx files (regular build) -->
+  <ItemGroup Condition="'$(ProductProject)' == 'true' and '$(LocalizedBuild)' == 'true'">
+    <!-- Union of already specified EmbeddedResource and Resources\*resx (localized resx) -->
+    <_CopyOfResources Include="@(EmbeddedResource)" />
+    <EmbeddedResource Include="$(ResourceDirectory)\*.resx" Exclude="@(_CopyOfResources)" />
+  </ItemGroup>
 
-  <Target Name="Localize" DependsOnTargets="_CreateLocalizeXLF;_CreateLocalizeResx" />
+  <Target Name="Localize" Condition="'$(ProductProject)' == 'true' and @(EmbeddedResource) != ''" DependsOnTargets="_CreateLocalizeXLF;_CreateLocalizeResx" />
 
-  <Target Name="_CreateLocalizeXLF" Condition="'$(TestProject)' =='' and '$(TestProject)' != 'true' and @(EmbeddedResource) != ''">
+  <Target Name="_CreateLocalizeXLF">
     <Message Importance="High" Text="Create localized xlf for '$(AssemblyName)'" />
     <CreateItem Include="@(EmbeddedResource)" AdditionalMetadata="Language=%(ResxLang.Identity)">
       <Output ItemName="LocResourceFile" TaskParameter="Include"/>
@@ -28,7 +43,7 @@
     <Exec Command="$(TestPlatformPackageDir)fmdev.xlftool\0.1.3\tools\xlftool.exe update -Resx %(EmbeddedResource.Identity) -Xlf $(ResourceDirectory)\xlf\%(EmbeddedResource.Filename).xlf" />
   </Target>
 
-  <Target Name="_CreateLocalizeResx" Condition="'$(TestProject)' =='' and '$(TestProject)' != 'true' and @(EmbeddedResource) != ''">
+  <Target Name="_CreateLocalizeResx">
     <Message Importance="High" Text="Create localized resx for '$(AssemblyName)'" />
     <CreateItem Include="@(EmbeddedResource)" AdditionalMetadata="Language=%(ResxLang.Identity)">
       <Output ItemName="LocResourceFile" TaskParameter="Include"/>

diff --git a/scripts/build/TestPlatform.Settings.targets b/scripts/build/TestPlatform.Settings.targets
@@ -31,6 +31,9 @@
     <DefineConstants>$(DefineConstants);CODE_ANALYSIS</DefineConstants>
   </PropertyGroup> 
 
+  <!-- Package dependency versions -->
+  <Import Project="$(MSBuildThisFileDirectory)TestPlatform.Dependencies.props" />
+
   <!-- Common dependencies for all projects -->
   <ItemGroup>
     <!--<PackageReference Include="StyleCop.Analyzers">-->
@@ -45,6 +48,7 @@
     </AdditionalFiles>
   </ItemGroup>
 
+  <!-- Test project settings -->
   <Choose>
     <When Condition="$(TestProject) == 'true'"> 
       <PropertyGroup>
@@ -76,10 +80,10 @@
         </ProjectReference>
 
         <PackageReference Include="MSTest.TestFramework">
-          <Version>1.0.7-preview</Version>
+          <Version>$(MSTestFrameworkVersion)</Version>
         </PackageReference>
         <PackageReference Include="MSTest.TestAdapter">
-          <Version>1.1.6-preview</Version>
+          <Version>$(MSTestAdapterVersion)</Version>
         </PackageReference>
         <PackageReference Include="Moq">
           <Version>4.6.38-*</Version>

diff --git a/scripts/test.ps1 b/scripts/test.ps1
@@ -139,7 +139,7 @@ function Invoke-Test
             Write-Log ".. Start run ($fx)"
 
             # Tests are only built for x86 at the moment, though we don't have architecture requirement
-            $testAdapterPath = "$env:TP_PACKAGES_DIR\MSTest.TestAdapter\1.1.6-preview\build\_common"
+            $testAdapterPath = Get-TestAdapterPath
             $testArchitecture = ($Script:TPT_TargetRuntime).Split("-")[-1]
 
             if($fx -eq $TPT_TargetFrameworkCore)
@@ -254,6 +254,13 @@ function Get-PackageDirectory($framework, $targetRuntime)
     return $(Join-Path $env:TP_OUT_DIR "$($Script:TPT_Configuration)\$($framework)\$($targetRuntime)")
 }
 
+function Get-TestAdapterPath
+{
+    [xml]$dependencyProps = Get-Content $env:TP_ROOT_DIR\scripts\build\TestPlatform.Dependencies.props
+
+    return "$env:TP_PACKAGES_DIR\MSTest.TestAdapter\$($dependencyProps.Project.PropertyGroup.MSTestAdapterVersion)\build\_common"
+}
+
 function Start-Timer
 {
     return [System.Diagnostics.Stopwatch]::StartNew()

diff --git a/src/Microsoft.TestPlatform.ObjectModel/Microsoft.TestPlatform.ObjectModel.csproj b/src/Microsoft.TestPlatform.ObjectModel/Microsoft.TestPlatform.ObjectModel.csproj
diff --git a/src/package/package/package.csproj b/src/package/package/package.csproj
@@ -48,6 +48,10 @@
   <ItemGroup Condition=" '$(TargetFramework)' == 'net46' ">
     <Reference Include="System" />
     <Reference Include="Microsoft.CSharp" />
+
+    <!-- For net46, testhost is packaged along with runner, include it in publish -->
+    <ProjectReference Include="$(TestPlatformRoot)src\testhost\testhost.csproj" />
+    <ProjectReference Include="$(TestPlatformRoot)src\testhost.x86\testhost.x86.csproj" />
   </ItemGroup>
   <ItemGroup Condition=" '$(TargetFramework)' == 'netcoreapp1.0' ">
     <PackageReference Include="Microsoft.NETCore.App">
@@ -60,4 +64,4 @@
   <Import Project="$(TestPlatformRoot)scripts\build\TestPlatform.targets" />
   <Target Name="CoreCompile" />
   <!-- Prevent Csc from being called -->
-</Project>
+</Project>
diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/Microsoft.TestPlatform.AcceptanceTests.csproj b/test/Microsoft.TestPlatform.AcceptanceTests/Microsoft.TestPlatform.AcceptanceTests.csproj
@@ -20,7 +20,7 @@
       <FromP2P>true</FromP2P>
     </ProjectReference>
     <PackageReference Include="Chutzpah">
-      <Version>4.2.4</Version>
+      <Version>$(ChutzpahAdapterVersion)</Version>
     </PackageReference>
     <PackageReference Include="Microsoft.TestPlatform.TestAsset.NativeCPP">
       <Version>1.0.0</Version>

diff --git a/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestBase.cs b/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestBase.cs
@@ -28,10 +28,10 @@ public class IntegrationTestBase
 
         protected readonly IntegrationTestEnvironment testEnvironment;
 
-        private const string TestAdapterRelativePath = @"MSTest.TestAdapter\1.1.6-preview\build\_common";
-        private const string NUnitTestAdapterRelativePath = @"nunittestadapter\1.2.0\lib";
-        private const string XUnitTestAdapterRelativePath = @"xunit.runner.visualstudio\2.2.0-beta4-build1188\build\_common";
-        private const string ChutzpahTestAdapterRelativePath = @"chutzpah\4.2.4\tools";
+        private const string TestAdapterRelativePath = @"MSTest.TestAdapter\{0}\build\_common";
+        private const string NUnitTestAdapterRelativePath = @"nunittestadapter\{0}\lib";
+        private const string XUnitTestAdapterRelativePath = @"xunit.runner.visualstudio\{0}\build\_common";
+        private const string ChutzpahTestAdapterRelativePath = @"chutzpah\{0}\tools";
 
         public enum UnitTestFramework
         {
@@ -251,19 +251,19 @@ protected string GetTestAdapterPath(UnitTestFramework testFramework = UnitTestFr
 
             if (testFramework == UnitTestFramework.MSTest)
             {
-                adapterRelativePath = TestAdapterRelativePath;
+                adapterRelativePath = string.Format(TestAdapterRelativePath, this.testEnvironment.DependencyVersions["MSTestAdapterVersion"]);
             }
             else if (testFramework == UnitTestFramework.NUnit)
             {
-                adapterRelativePath = NUnitTestAdapterRelativePath;
+                adapterRelativePath = string.Format(NUnitTestAdapterRelativePath, this.testEnvironment.DependencyVersions["NUnitAdapterVersion"]);
             }
             else if (testFramework == UnitTestFramework.XUnit)
             {
-                adapterRelativePath = XUnitTestAdapterRelativePath;
+                adapterRelativePath = string.Format(XUnitTestAdapterRelativePath, this.testEnvironment.DependencyVersions["XUnitAdapterVersion"]);
             }
             else if (testFramework == UnitTestFramework.Chutzpah)
             {
-                adapterRelativePath = ChutzpahTestAdapterRelativePath;
+                adapterRelativePath = string.Format(ChutzpahTestAdapterRelativePath, this.testEnvironment.DependencyVersions["ChutzpahAdapterVersion"]);
             }
 
             return this.testEnvironment.GetNugetPackage(adapterRelativePath);

diff --git a/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestEnvironment.cs b/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestEnvironment.cs
@@ -4,7 +4,9 @@
 namespace Microsoft.TestPlatform.TestUtilities
 {
     using System;
+    using System.Collections.Generic;
     using System.IO;
+    using System.Xml;
 
     using Microsoft.VisualStudio.TestTools.UnitTesting;
 
@@ -15,8 +17,9 @@ namespace Microsoft.TestPlatform.TestUtilities
     /// </summary>
     public class IntegrationTestEnvironment
     {
-        private readonly string testPlatformRootDirectory;
+        private static Dictionary<string, string> dependencyVersions;
 
+        private readonly string testPlatformRootDirectory;
         private readonly bool runningInCli;
 
         public IntegrationTestEnvironment()
@@ -76,6 +79,19 @@ public string BuildConfiguration
             }
         }
 
+        public Dictionary<string, string> DependencyVersions
+        {
+            get
+            {
+                if (dependencyVersions == null)
+                {
+                    dependencyVersions = GetDependencies(this.testPlatformRootDirectory);
+                }
+
+                return dependencyVersions;
+            }
+        }
+
         /// <summary>
         /// Gets the nuget packages directory for enlistment.
         /// </summary>
@@ -106,9 +122,10 @@ public string PublishDirectory
                 {
                     value = Path.Combine(
                     this.testPlatformRootDirectory,
-                    @"src\Microsoft.TestPlatform.VSIXCreator\bin",
+                    @"src\package\package\bin",
                     this.BuildConfiguration,
-                    "net46");
+                    this.RunnerFramework,
+                    this.TargetRuntime);
                 }
 
                 return value;
@@ -226,5 +243,35 @@ public string GetConsoleRunnerPath()
             Assert.IsTrue(File.Exists(consoleRunnerPath), "GetConsoleRunnerPath: Path not found: {0}", consoleRunnerPath);
             return consoleRunnerPath;
         }
+
+        private static Dictionary<string, string> GetDependencies(string testPlatformRoot)
+        {
+            var dependencyPropsFile = Path.Combine(testPlatformRoot, @"scripts\build\TestPlatform.Dependencies.props");
+            var dependencyProps = new Dictionary<string, string>();
+            if (!File.Exists(dependencyPropsFile))
+            {
+                throw new FileNotFoundException("Dependency props file not found: " + dependencyPropsFile);
+            }
+
+            using (var reader = XmlReader.Create(dependencyPropsFile))
+            {
+                reader.ReadToFollowing("PropertyGroup");
+                using (var props = reader.ReadSubtree())
+                {
+                    props.MoveToContent();
+                    props.Read();   // Read thru the PropertyGroup node
+                    while (!props.EOF)
+                    {
+                        if (props.IsStartElement() && !string.IsNullOrEmpty(props.Name))
+                        {
+                            dependencyProps.Add(props.Name, props.ReadElementContentAsString());
+                        }
+                        props.Read();
+                    }
+                }
+            }
+
+            return dependencyProps;
+        }
     }
 }
diff --git a/test/Microsoft.TestPlatform.TestUtilities/Microsoft.TestPlatform.TestUtilities.csproj b/test/Microsoft.TestPlatform.TestUtilities/Microsoft.TestPlatform.TestUtilities.csproj
@@ -3,7 +3,7 @@
   <PropertyGroup>
     <TestPlatformRoot Condition="$(TestPlatformRoot) == ''">..\..\</TestPlatformRoot>
   </PropertyGroup>
-  <Import Project="$(TestPlatformRoot)scripts/build/TestPlatform.Settings.targets" />
+  <Import Project="$(TestPlatformRoot)scripts\build\TestPlatform.Settings.targets" />
   <PropertyGroup>
     <AssemblyName>Microsoft.TestPlatform.TestUtilities</AssemblyName>
     <TargetFrameworks>netcoreapp1.0;net46</TargetFrameworks>
@@ -17,7 +17,7 @@
   </ItemGroup>
   <ItemGroup>
     <PackageReference Include="MSTest.TestFramework">
-      <Version>1.0.4-preview</Version>
+      <Version>$(MSTestFrameworkVersion)</Version>
     </PackageReference>
   </ItemGroup>
   <ItemGroup Condition=" '$(TargetFramework)' == 'netcoreapp1.0' ">

diff --git a/test/TestAssets/NUTestProject/NUTestProject.csproj b/test/TestAssets/NUTestProject/NUTestProject.csproj
@@ -1,25 +1,19 @@
 <?xml version="1.0" encoding="utf-8"?>
 <Project Sdk="Microsoft.NET.Sdk">
+  <!-- Package dependency versions -->
+  <Import Project="..\..\..\scripts\build\TestPlatform.Dependencies.props" />
+
   <PropertyGroup>
     <TargetFramework>net46</TargetFramework>
     <AssemblyName>NUTestProject</AssemblyName>
   </PropertyGroup>
 
   <ItemGroup>
     <PackageReference Include="NUnit">
-      <Version>2.6.4</Version>
+      <Version>$(NUnitFrameworkVersion)</Version>
     </PackageReference>
     <PackageReference Include="NUnitTestAdapter">
-      <Version>1.2.0</Version>
+      <Version>$(NUnitAdapterVersion)</Version>
     </PackageReference>
   </ItemGroup>
-  <ItemGroup Condition=" '$(TargetFramework)' == 'net46' ">
-    <Reference Include="System.Runtime" />
-    <Reference Include="System" />
-    <Reference Include="Microsoft.CSharp" />
-  </ItemGroup>
-
-  <PropertyGroup Condition=" '$(Configuration)' == 'Release' ">
-    <DefineConstants>$(DefineConstants);RELEASE</DefineConstants>
-  </PropertyGroup>  
 </Project>
diff --git a/test/TestAssets/PerfTestProject/PerfTestProject.csproj b/test/TestAssets/PerfTestProject/PerfTestProject.csproj
diff --git a/test/TestAssets/SimpleDataCollector/SimpleDataCollector.csproj b/test/TestAssets/SimpleDataCollector/SimpleDataCollector.csproj
@@ -1,31 +1,25 @@
 <?xml version="1.0" encoding="utf-16"?>
 <Project Sdk="Microsoft.NET.Sdk">
+  <!-- Package dependency versions -->
+  <Import Project="..\..\..\scripts\build\TestPlatform.Dependencies.props" />
+
   <PropertyGroup>
     <TargetFrameworks>netcoreapp1.0;net46</TargetFrameworks>
     <WarningsAsErrors>true</WarningsAsErrors>
     <AssemblyName>SimpleDataCollector</AssemblyName>
     <AssemblyOriginatorKeyFile>TITestDllKey.snk</AssemblyOriginatorKeyFile>
     <SignAssembly>true</SignAssembly>
     <PublicSign Condition=" '$(OS)' != 'Windows_NT' ">true</PublicSign>
-    <PackageTargetFallback Condition=" '$(TargetFramework)' == 'netcoreapp1.0' ">$(PackageTargetFallback);dnxcore50;portable-net45+win8</PackageTargetFallback>
   </PropertyGroup>
   <ItemGroup>
     <PackageReference Include="MSTest.TestFramework">
-      <Version>1.0.4-preview</Version>
+      <Version>$(MSTestFrameworkVersion)</Version>
     </PackageReference>
     <PackageReference Include="Microsoft.TestPlatform.ObjectModel">
-      <Version>15.0.0-*</Version>
+      <Version>$(NETTestSdkPreviousVersion)</Version>
     </PackageReference>
     <PackageReference Include="Microsoft.NET.Test.Sdk">
-      <Version>15.0.0-preview-20170106-08</Version>
+      <Version>$(NETTestSdkPreviousVersion)</Version>
     </PackageReference>
   </ItemGroup>
-  <ItemGroup Condition=" '$(TargetFramework)' == 'net46' ">
-    <Reference Include="System.Runtime" />
-    <Reference Include="System" />
-    <Reference Include="Microsoft.CSharp" />
-  </ItemGroup>
-  <PropertyGroup Condition=" '$(Configuration)' == 'Release' ">
-    <DefineConstants>$(DefineConstants);RELEASE</DefineConstants>
-  </PropertyGroup>
 </Project>