From 769fd62b966f20a44d0800cdfeecb7d86e10f348 Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" <42748379+dotnet-maestro[bot]@users.noreply.github.com> Date: Tue, 18 May 2021 08:13:25 +0000 Subject: [PATCH] Update dependencies from https://github.com/dotnet/arcade build 20210514.2 (#603) [main] Update dependencies from dotnet/arcade --- eng/Version.Details.xml | 8 +- .../post-build/sourcelink-validation.ps1 | 78 ++++++++++++------- eng/common/post-build/symbols-validation.ps1 | 21 ++--- global.json | 4 +- 4 files changed, 70 insertions(+), 41 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index dcd162449..759e133da 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -3,13 +3,13 @@ - + https://github.com/dotnet/arcade - 6b9758661f4483a70654bcaf6f8d7c6a79ee5660 + 42de78a825b575a1ddeb73020a01fb8cd9311d09 - + https://github.com/dotnet/arcade - 6b9758661f4483a70654bcaf6f8d7c6a79ee5660 + 42de78a825b575a1ddeb73020a01fb8cd9311d09 diff --git a/eng/common/post-build/sourcelink-validation.ps1 b/eng/common/post-build/sourcelink-validation.ps1 index 8c554729b..85c898617 100644 --- a/eng/common/post-build/sourcelink-validation.ps1 +++ b/eng/common/post-build/sourcelink-validation.ps1 @@ -16,6 +16,8 @@ $global:RepoFiles = @{} # Maximum number of jobs to run in parallel $MaxParallelJobs = 16 +$MaxRetries = 5 + # Wait time between check for system load $SecondsBetweenLoadChecks = 10 @@ -29,7 +31,10 @@ $ValidatePackage = { # Ensure input file exist if (!(Test-Path $PackagePath)) { Write-Host "Input file does not exist: $PackagePath" - return 1 + return [pscustomobject]@{ + result = 1 + packagePath = $PackagePath + } } # Extensions for which we'll look for SourceLink information @@ -59,7 +64,10 @@ $ValidatePackage = { # We ignore resource DLLs if ($FileName.EndsWith('.resources.dll')) { - return + return [pscustomobject]@{ + result = 0 + packagePath = $PackagePath + } } [System.IO.Compression.ZipFileExtensions]::ExtractToFile($_, $TargetFile, $true) @@ -91,36 +99,49 @@ $ValidatePackage = { $Status = 200 $Cache = $using:RepoFiles - if ( !($Cache.ContainsKey($FilePath)) ) { - try { - $Uri = $Link -as [System.URI] - - # Only GitHub links are valid - if ($Uri.AbsoluteURI -ne $null -and ($Uri.Host -match 'github' -or $Uri.Host -match 'githubusercontent')) { - $Status = (Invoke-WebRequest -Uri $Link -UseBasicParsing -Method HEAD -TimeoutSec 5).StatusCode + $totalRetries = 0 + + while ($totalRetries -lt $using:MaxRetries) { + if ( !($Cache.ContainsKey($FilePath)) ) { + try { + $Uri = $Link -as [System.URI] + + # Only GitHub links are valid + if ($Uri.AbsoluteURI -ne $null -and ($Uri.Host -match 'github' -or $Uri.Host -match 'githubusercontent')) { + $Status = (Invoke-WebRequest -Uri $Link -UseBasicParsing -Method HEAD -TimeoutSec 5).StatusCode + } + else { + # If it's not a github link, we want to break out of the loop and not retry. + $Status = 0 + $totalRetries = $using:MaxRetries + } } - else { + catch { + Write-Host $_ $Status = 0 } } - catch { - write-host $_ - $Status = 0 - } - } - if ($Status -ne 200) { - if ($NumFailedLinks -eq 0) { - if ($FailedFiles.Value -eq 0) { - Write-Host + if ($Status -ne 200) { + $totalRetries++ + + if ($totalRetries -ge $using:MaxRetries) { + if ($NumFailedLinks -eq 0) { + if ($FailedFiles.Value -eq 0) { + Write-Host + } + + Write-Host "`tFile $RealPath has broken links:" + } + + Write-Host "`t`tFailed to retrieve $Link" + + $NumFailedLinks++ } - - Write-Host "`tFile $RealPath has broken links:" } - - Write-Host "`t`tFailed to retrieve $Link" - - $NumFailedLinks++ + else { + break + } } } } @@ -136,7 +157,7 @@ $ValidatePackage = { } } catch { - + Write-Host $_ } finally { $zip.Dispose() @@ -220,6 +241,7 @@ function ValidateSourceLinkLinks { # Process each NuGet package in parallel Get-ChildItem "$InputPath\*.symbols.nupkg" | ForEach-Object { + Write-Host "Starting $($_.FullName)" Start-Job -ScriptBlock $ValidatePackage -ArgumentList $_.FullName | Out-Null $NumJobs = @(Get-Job -State 'Running').Count @@ -267,6 +289,10 @@ function InstallSourcelinkCli { try { InstallSourcelinkCli + foreach ($Job in @(Get-Job)) { + Remove-Job -Id $Job.Id + } + ValidateSourceLinkLinks } catch { diff --git a/eng/common/post-build/symbols-validation.ps1 b/eng/common/post-build/symbols-validation.ps1 index f26f6de81..a5af041ba 100644 --- a/eng/common/post-build/symbols-validation.ps1 +++ b/eng/common/post-build/symbols-validation.ps1 @@ -133,27 +133,27 @@ $CountMissingSymbols = { elseif (Test-Path $SymbolPath) { return 'Module' } - elseif ($output.Contains("503 Service Unavailable")) { - # If we got a 503 error, we should retry. + else + { $totalRetries++ } - else { - return $null - } } return $null } + $FileGuid = New-Guid + $ExpandedSymbolsPath = Join-Path -Path $SymbolsPath -ChildPath $FileGuid + $SymbolsOnMSDL = & $FirstMatchingSymbolDescriptionOrDefault ` -FullPath $FileName ` -TargetServerParam '--microsoft-symbol-server' ` - -SymbolsPath $SymbolsPath ` + -SymbolsPath "$ExpandedSymbolsPath-msdl" ` -WindowsPdbVerificationParam $WindowsPdbVerificationParam $SymbolsOnSymWeb = & $FirstMatchingSymbolDescriptionOrDefault ` -FullPath $FileName ` -TargetServerParam '--internal-server' ` - -SymbolsPath $SymbolsPath ` + -SymbolsPath "$ExpandedSymbolsPath-symweb" ` -WindowsPdbVerificationParam $WindowsPdbVerificationParam Write-Host -NoNewLine "`t Checking file " $FileName "... " @@ -217,6 +217,7 @@ function CheckSymbolsAvailable { Remove-Item $ExtractPath -Force -Recurse -ErrorAction SilentlyContinue } + $TotalPackages = 0 $TotalFailures = 0 $DupedSymbols = 0 @@ -239,6 +240,8 @@ function CheckSymbolsAvailable { return } + $TotalPackages++ + Start-Job -ScriptBlock $CountMissingSymbols -ArgumentList @($FullName,$WindowsPdbVerificationParam) | Out-Null $NumJobs = @(Get-Job -State 'Running').Count @@ -264,11 +267,11 @@ function CheckSymbolsAvailable { if ($TotalFailures -gt 0 -or $DupedSymbols -gt 0) { if ($TotalFailures -gt 0) { - Write-PipelineTelemetryError -Category 'CheckSymbols' -Message "Symbols missing for $TotalFailures packages" + Write-PipelineTelemetryError -Category 'CheckSymbols' -Message "Symbols missing for $TotalFailures/$TotalPackages packages" } if ($DupedSymbols -gt 0) { - Write-PipelineTelemetryError -Category 'CheckSymbols' -Message "$DupedSymbols packages had duplicated symbol files" + Write-PipelineTelemetryError -Category 'CheckSymbols' -Message "$DupedSymbols/$TotalPackages packages had duplicated symbol files and could not be extracted" } ExitWithExitCode 1 diff --git a/global.json b/global.json index a3937e414..4af823ee2 100644 --- a/global.json +++ b/global.json @@ -3,7 +3,7 @@ "dotnet": "6.0.100-preview.3.21202.5" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "6.0.0-beta.21263.1", - "Microsoft.DotNet.Helix.Sdk": "6.0.0-beta.21263.1" + "Microsoft.DotNet.Arcade.Sdk": "6.0.0-beta.21264.2", + "Microsoft.DotNet.Helix.Sdk": "6.0.0-beta.21264.2" } }