From 7677f7dc71fafad1f35639803b86d05b0bd7df72 Mon Sep 17 00:00:00 2001 From: Andy Gocke Date: Wed, 31 Mar 2021 11:34:05 -0700 Subject: [PATCH] Add option for building a test exe as single file (#42972) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Add option for building a test exe as single file * Remove left over test * Add target to exclude references from single-file * First attempt at adding a CI job * Opt-in specific libraries for single-file testing support Start with System.Collections as all tests pass. * Config testing using single-file in build.cmd * Change yml suffix name to SingleFile * Windows_NT_x64 -> windows_x64 * Fix for helix queueing * Respond to host rename * Change TargetOS to check for windows * chmod test exe on linux * Direct singlefilehost to the locally built copy * Adjust singlefilehost copy+call * Add .exe suffix on Windows * Move libraries after hosts build to allow for libs.test to depend on hosts build * Split up host and libs packaging and tests * Move host packaging * Move pretest up * Move packages up as well * Reorder libs pretest and libs.packages * Add isSingleFile build parameter to limit Linux Helix jobs * Typo * Change conditional check * Fix yml * Fix yml * Fix neq * Fix subsets * Typo * Fix * Adjust assert * Include code from Michal to skip failing test * Remove empty ItemGroup * Update src/libraries/Common/tests/SingleFileTestRunner/SingleFileTestRunner.cs Co-authored-by: Michal Strehovský * Update eng/testing/tests.singlefile.targets Co-authored-by: Michal Strehovský * Apply suggestions from code review Co-authored-by: Michal Strehovský * Use ProjectExclusions * Update eng/testing/tests.singlefile.targets Co-authored-by: Michal Strehovský * Revert changes * Remove host build from tests Co-authored-by: Michal Strehovský --- .../libraries/helix-queues-setup.yml | 6 +- eng/pipelines/runtime.yml | 22 ++++++ eng/testing/tests.singlefile.targets | 62 +++++++++++++++ eng/testing/tests.targets | 7 +- eng/testing/xunit/xunit.console.targets | 4 +- eng/testing/xunit/xunit.props | 2 +- .../SingleFileTestRunner.cs | 77 +++++++++++++++++++ .../tests/System.Reflection.Tests.csproj | 4 + src/libraries/tests.proj | 20 +++-- 9 files changed, 188 insertions(+), 16 deletions(-) create mode 100644 eng/testing/tests.singlefile.targets create mode 100644 src/libraries/Common/tests/SingleFileTestRunner/SingleFileTestRunner.cs diff --git a/eng/pipelines/libraries/helix-queues-setup.yml b/eng/pipelines/libraries/helix-queues-setup.yml index 8b0f58894cd4ed..90bf48d7cae1ad 100644 --- a/eng/pipelines/libraries/helix-queues-setup.yml +++ b/eng/pipelines/libraries/helix-queues-setup.yml @@ -52,7 +52,7 @@ jobs: # Linux x64 - ${{ if eq(parameters.platform, 'Linux_x64') }}: - - ${{ if eq(parameters.jobParameters.interpreter, '') }}: + - ${{ if and(eq(parameters.jobParameters.interpreter, ''), ne(parameters.jobParameters.isSingleFile, true)) }}: - ${{ if and(eq(parameters.jobParameters.testScope, 'outerloop'), eq(parameters.jobParameters.runtimeFlavor, 'mono')) }}: - (Centos.8.Amd64.Open)Ubuntu.1604.Amd64.Open@mcr.microsoft.com/dotnet-buildtools/prereqs:centos-8-helix-20201229003624-c1bf759 - RedHat.7.Amd64.Open @@ -81,7 +81,7 @@ jobs: - Ubuntu.1804.Amd64.Open - SLES.15.Amd64.Open - (Fedora.30.Amd64.Open)ubuntu.1604.amd64.open@mcr.microsoft.com/dotnet-buildtools/prereqs:fedora-30-helix-20200512010621-4f8cef7 - - ${{ if eq(parameters.jobParameters.interpreter, 'true') }}: + - ${{ if or(eq(parameters.jobParameters.interpreter, 'true'), eq(parameters.jobParameters.isSingleFile, true)) }}: # Limiting interp runs as we don't need as much coverage. - Debian.9.Amd64.Open @@ -135,7 +135,7 @@ jobs: # .NETFramework - ${{ if eq(parameters.jobParameters.framework, 'net48') }}: - Windows.10.Amd64.Client19H1.Open - + # AllConfigurations - ${{ if eq(parameters.jobParameters.framework, 'allConfigurations') }}: - Windows.10.Amd64.Server19H1.Open diff --git a/eng/pipelines/runtime.yml b/eng/pipelines/runtime.yml index df3f7ca8184b61..476150aad686ab 100644 --- a/eng/pipelines/runtime.yml +++ b/eng/pipelines/runtime.yml @@ -302,6 +302,28 @@ jobs: eq(variables['monoContainsChange'], true), eq(variables['isFullMatrix'], true)) +# Build and test libraries under single-file publishing +- template: /eng/pipelines/common/platform-matrix.yml + parameters: + jobTemplate: /eng/pipelines/common/global-build-job.yml + helixQueuesTemplate: /eng/pipelines/libraries/helix-queues-setup.yml + buildConfig: Release + platforms: + - windows_x64 + - Linux_x64 + jobParameters: + testGroup: innerloop + isFullMatrix: ${{ variables.isFullMatrix }} + isSingleFile: true + nameSuffix: SingleFile + buildArgs: -s clr+libs+libs.tests -c $(_BuildConfig) /p:TestSingleFile=true /p:ArchiveTests=true + timeoutInMinutes: 120 + # extra steps, run tests + extraStepsTemplate: /eng/pipelines/libraries/helix.yml + extraStepsParameters: + creator: dotnet-bot + testRunNamePrefixSuffix: SingleFile_$(_BuildConfig) + # # Build the whole product using Mono and run runtime tests # diff --git a/eng/testing/tests.singlefile.targets b/eng/testing/tests.singlefile.targets new file mode 100644 index 00000000000000..86cbc7ee5109a1 --- /dev/null +++ b/eng/testing/tests.singlefile.targets @@ -0,0 +1,62 @@ + + + Exe + + $([MSBuild]::NormalizeDirectory('$(OutDir)', 'publish')) + $([MSBuild]::NormalizePath('$(BundleDir)', '$(RunScriptOutputName)')) + $(PackageRID) + + $(AssemblyName).exe + chmod +rwx $(AssemblyName) && ./$(AssemblyName) + + + + true + true + true + $([MSBuild]::NormalizeDirectory('$(ArtifactsBinDir)', 'coreclr', '$(TargetOS).$(TargetArchitecture).$(Configuration)', 'corehost'))/singlefilehost + $(SingleFileHostSourcePath).exe + + + + + + + + + + + + + <__Identity>%(ResolvedFileToPublish.Identity) + <__FileName>%(ResolvedFileToPublish.Filename)%(ResolvedFileToPublish.Extension) + + + + <__NewResolvedFiles Include="@(ResolvedFileToPublish)"> + true + + + + + + + + + + + + + + diff --git a/eng/testing/tests.targets b/eng/testing/tests.targets index b7b642b3527ec3..9fc08ddd6bcc70 100644 --- a/eng/testing/tests.targets +++ b/eng/testing/tests.targets @@ -25,7 +25,7 @@ - + @@ -37,7 +37,7 @@ <_ZipSourceDirectory>$(OutDir) - <_ZipSourceDirectory Condition="'$(TargetOS)' == 'Browser'">$(BundleDir) + <_ZipSourceDirectory Condition="'$(TargetOS)' == 'Browser' or '$(TestSingleFile)' == 'true'">$(BundleDir) @@ -118,10 +118,11 @@ + - + diff --git a/eng/testing/xunit/xunit.console.targets b/eng/testing/xunit/xunit.console.targets index a88b08f6139813..7710a05b5a37db 100644 --- a/eng/testing/xunit/xunit.console.targets +++ b/eng/testing/xunit/xunit.console.targets @@ -5,7 +5,7 @@ true - + <_depsFileArgument Condition="'$(GenerateDependencyFile)' == 'true'">--depsfile $(AssemblyName).deps.json "$(RunScriptHost)" exec --runtimeconfig $(AssemblyName).runtimeconfig.json $(_depsFileArgument) xunit.console.dll xunit.console.exe @@ -36,7 +36,7 @@ $(_withoutCategories.Replace(';', '%0dcategory=')) - + diff --git a/eng/testing/xunit/xunit.props b/eng/testing/xunit/xunit.props index f63b3906fccb61..3f9c4b67141a2f 100644 --- a/eng/testing/xunit/xunit.props +++ b/eng/testing/xunit/xunit.props @@ -13,7 +13,7 @@ - + diff --git a/src/libraries/Common/tests/SingleFileTestRunner/SingleFileTestRunner.cs b/src/libraries/Common/tests/SingleFileTestRunner/SingleFileTestRunner.cs new file mode 100644 index 00000000000000..7b67a1338e5a6c --- /dev/null +++ b/src/libraries/Common/tests/SingleFileTestRunner/SingleFileTestRunner.cs @@ -0,0 +1,77 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +#nullable disable + +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Reflection; +using System.Runtime.Loader; +using System.Threading.Tasks; +using System.Xml.Linq; +using Xunit; +using Xunit.Abstractions; +using Xunit.Sdk; + +public class SingleFileTestRunner : XunitTestFramework +{ + private SingleFileTestRunner(IMessageSink messageSink) + : base(messageSink) { } + + public static int Main(string[] args) + { + var asm = typeof(SingleFileTestRunner).Assembly; + Console.WriteLine("Running assembly:" + asm.FullName); + + var diagnosticSink = new ConsoleDiagnosticMessageSink(); + var testsFinished = new TaskCompletionSource(); + var testSink = new TestMessageSink(); + var summarySink = new DelegatingExecutionSummarySink(testSink, + () => false, + (completed, summary) => Console.WriteLine($"Tests run: {summary.Total}, Errors: {summary.Errors}, Failures: {summary.Failed}, Skipped: {summary.Skipped}. Time: {TimeSpan.FromSeconds((double)summary.Time).TotalSeconds}s")); + var resultsXmlAssembly = new XElement("assembly"); + var resultsSink = new DelegatingXmlCreationSink(summarySink, resultsXmlAssembly); + + testSink.Execution.TestSkippedEvent += args => { Console.WriteLine($"[SKIP] {args.Message.Test.DisplayName}"); }; + testSink.Execution.TestFailedEvent += args => { Console.WriteLine($"[FAIL] {args.Message.Test.DisplayName}{Environment.NewLine}{Xunit.ExceptionUtility.CombineMessages(args.Message)}{Environment.NewLine}{Xunit.ExceptionUtility.CombineStackTraces(args.Message)}"); }; + + testSink.Execution.TestAssemblyFinishedEvent += args => + { + Console.WriteLine($"Finished {args.Message.TestAssembly.Assembly}{Environment.NewLine}"); + testsFinished.SetResult(); + }; + + var xunitTestFx = new SingleFileTestRunner(diagnosticSink); + var asmInfo = Reflector.Wrap(asm); + var asmName = asm.GetName(); + + var discoverySink = new TestDiscoverySink(); + var discoverer = xunitTestFx.CreateDiscoverer(asmInfo); + discoverer.Find(false, discoverySink, TestFrameworkOptions.ForDiscovery()); + discoverySink.Finished.WaitOne(); + XunitFilters filters = new XunitFilters(); + filters.ExcludedTraits.Add("category", new List { "failing" }); + var filteredTestCases = discoverySink.TestCases.Where(filters.Filter).ToList(); + var executor = xunitTestFx.CreateExecutor(asmName); + executor.RunTests(filteredTestCases, resultsSink, TestFrameworkOptions.ForExecution()); + + resultsSink.Finished.WaitOne(); + + var failed = resultsSink.ExecutionSummary.Failed > 0 || resultsSink.ExecutionSummary.Errors > 0; + return failed ? 1 : 0; + } +} + +internal class ConsoleDiagnosticMessageSink : IMessageSink +{ + public bool OnMessage(IMessageSinkMessage message) + { + if (message is IDiagnosticMessage diagnosticMessage) + { + return true; + } + return false; + } +} diff --git a/src/libraries/System.Reflection/tests/System.Reflection.Tests.csproj b/src/libraries/System.Reflection/tests/System.Reflection.Tests.csproj index 5b4ff278583951..deceb65e37f3e0 100644 --- a/src/libraries/System.Reflection/tests/System.Reflection.Tests.csproj +++ b/src/libraries/System.Reflection/tests/System.Reflection.Tests.csproj @@ -71,4 +71,8 @@ + + + <__ExcludeFromBundle Include="TestAssembly.dll" /> + diff --git a/src/libraries/tests.proj b/src/libraries/tests.proj index 55fdbc13f9f1fb..1442c9d3020fca 100644 --- a/src/libraries/tests.proj +++ b/src/libraries/tests.proj @@ -12,7 +12,7 @@ false false - + @@ -107,17 +107,17 @@ - + - + - + - + @@ -320,6 +320,12 @@ + + + + + + - + @@ -357,7 +363,7 @@ Exclude="$(RepoRoot)\src\tests\FunctionalTests\Android\Device_Emulator\AOT\Android.Device_Emulator.Aot.Test.csproj" BuildInParallel="false" /> - +