Organize published artifacts (#14832)

* Begin rearranging published packages

* Add step to arrange artifacts

* Remove steps for testing

* Swtich output directory

* Remove analyze and test pipeline

* Remove artifact staging from release stages

* Fix Doc Upload

* Bump Template version, remove test and analyze job

* Uncomment deployment steps

* ReAdd release stage

* Remove stage artifacts file

* Bump up template version

* Update archetype-net-release.yml and copy-docs-to-blobstorage.ps1

* Make use of VerifyPackages to retrieve pkg properties

* Update template version to test release

* Bump Azure Template

Directory.Build.props

@@ -32,7 +32,7 @@
     <IntermediateOutputPath Condition="'$(PlatformName)' == 'AnyCPU'">$(BaseIntermediateOutputPath)$(Configuration)\</IntermediateOutputPath>
     <IntermediateOutputPath Condition="'$(PlatformName)' != 'AnyCPU'">$(BaseIntermediateOutputPath)$(PlatformName)\$(Configuration)\</IntermediateOutputPath>
 
-    <PackageOutputPath>$(ArtifactsPackagesDir)</PackageOutputPath>
+    <PackageOutputPath>$(ArtifactsPackagesDir)/$(MSBuildProjectName)</PackageOutputPath>
   </PropertyGroup>
 
   <Import Project="$(RepoEngPath)\mgmt\Directory.Build.Mgmt.props" Condition="'$(IsDataPlaneProject)' != 'true'" />

doc/ApiDocGeneration/Generate-Api-Docs.ps1

@@ -45,7 +45,8 @@ Param (
     $RepoRoot = "${PSScriptRoot}/../..",
     [Parameter(Mandatory = $True)]
     $BinDirectory,
-    $DocGenDir = "${PSScriptRoot}"
+    $DocGenDir = "${PSScriptRoot}",
+    $ArtifactStagingDirectory
 )
 
 Write-Verbose "Name Reccuring paths with variable names"
@@ -139,5 +140,5 @@ Write-Verbose "Build Doc Content"
 Write-Verbose "Copy over site Logo"
 Copy-Item "${DocGenDir}/assets/logo.svg" -Destination "${DocOutHtmlDir}" -Recurse -Force
 
-Write-Verbose "Set variable for publish pipeline step"
-echo "##vso[task.setvariable variable=PublishTargetPath]${DocOutHtmlDir}"
+Write-Verbose "Compress and copy HTML into the staging Area"
+Compress-Archive -Path "${DocOutHtmlDir}/*" -DestinationPath "${ArtifactStagingDirectory}/${ArtifactName}/${ArtifactName}.docs.zip" -CompressionLevel Fastest

eng/common/pipelines/templates/steps/publish-blobs.yml

@@ -25,6 +25,7 @@ steps:
       -BlobName "${{ parameters.BlobName }}"
       -PublicArtifactLocation "${{ parameters.ArtifactLocation }}"
       -RepoReplaceRegex "(https://github.com/${{ parameters.RepoId }}/(?:blob|tree)/)master"
+      -WorkingDirectory "$(System.DefaultWorkingDirectory)"
     pwsh: true
     workingDirectory: $(Pipeline.Workspace)
   displayName: Copy Docs to Blob

eng/common/scripts/copy-docs-to-blobstorage.ps1

@@ -9,7 +9,8 @@ param (
   $ExitOnError=1,
   $UploadLatest=1,
   $PublicArtifactLocation = "",
-  $RepoReplaceRegex = "(https://github.com/.*/(?:blob|tree)/)master"
+  $RepoReplaceRegex = "(https://github.com/.*/(?:blob|tree)/)master",
+  $WorkingDirectory
 )
 . (Join-Path $PSScriptRoot artifact-metadata-parsing.ps1)
 
@@ -257,30 +258,31 @@ if ($Language -eq "javascript")
 
 if ($Language -eq "dotnet")
 {
-    $PublishedPkgs = Get-ChildItem "$($DocLocation)/packages" | Where-Object -FilterScript {$_.Name.EndsWith(".nupkg") -and -not $_.Name.EndsWith(".symbols.nupkg")}
-    $PublishedDocs = Get-ChildItem "$($DocLocation)" | Where-Object -FilterScript {$_.Name.StartsWith("Docs.")}
+    $PublishedPkgs = Get-ChildItem "$($DocLocation)" | Where-Object -FilterScript {$_.Name.EndsWith(".nupkg") -and -not $_.Name.EndsWith(".symbols.nupkg")}
+    $PublishedDocs = Get-ChildItem "$($DocLocation)" | Where-Object -FilterScript {$_.Name.EndsWith("docs.zip")}
 
-    foreach ($Item in $PublishedDocs) {
-        $PkgName = $Item.Name.Remove(0, 5)
-        $PkgFullName = $PublishedPkgs | Where-Object -FilterScript {$_.Name -match "$($PkgName).\d"}
-
-        if (($PkgFullName | Measure-Object).count -eq 1)
-        {
-            $DocVersion = $PkgFullName[0].BaseName.Remove(0, $PkgName.Length + 1)
-
-            Write-Host "Start Upload for $($PkgName)/$($DocVersion)"
-            Write-Host "DocDir $($Item)"
-            Write-Host "PkgName $($PkgName)"
-            Write-Host "DocVersion $($DocVersion)"
-            $releaseTag = RetrieveReleaseTag "Nuget" $PublicArtifactLocation 
-            Upload-Blobs -DocDir "$($Item)" -PkgName $PkgName -DocVersion $DocVersion -ReleaseTag $releaseTag
-        }
-        else
-        {
-            Write-Host "Package with the same name Exists. Upload Skipped"
-            continue
-        }
+    if (($PublishedPkgs.Count -gt 1) -or ($PublishedDoc.Count -gt 1))
+    {
+        Write-Host "$($DocLocation) should contain only one (1) published package and docs"
+        Write-Host "No of Packages $($PublishedPkgs.Count)"
+        Write-Host "No of Docs $($PublishedDoc.Count)"
+        exit 1
     }
+
+    $DocsStagingDir = "$WorkingDirectory/docstaging"
+    $TempDir = "$WorkingDirectory/temp"
+
+    New-Item -ItemType directory -Path $DocsStagingDir
+    New-Item -ItemType directory -Path $TempDir
+
+    Expand-Archive -LiteralPath $PublishedDocs[0].FullName -DestinationPath $DocsStagingDir
+    $pkgProperties = ParseNugetPackage -pkg $PublishedPkgs[0].FullName -workingDirectory $TempDir
+
+    Write-Host "Start Upload for $($pkgProperties.Tag)"
+    Write-Host "DocDir $($DocsStagingDir)"
+    Write-Host "PkgName $($pkgProperties.PackageId)"
+    Write-Host "DocVersion $($pkgProperties.PackageVersion)"
+    Upload-Blobs -DocDir "$($DocsStagingDir)" -PkgName $pkgProperties.PackageId -DocVersion $pkgProperties.PackageVersion -ReleaseTag $pkgProperties.Tag
 }
 
 if ($Language -eq "python")

eng/pipelines/templates/jobs/archetype-sdk-client.yml

@@ -31,24 +31,28 @@ jobs:
           BuildSHA: $(Build.SourceVersion)
           RepoId: 'Azure/azure-sdk-for-net'
       - script: >-
-          dotnet pack eng/service.proj -o $(Build.ArtifactStagingDirectory) -warnaserror /p:ServiceDirectory=${{parameters.ServiceToBuild}}
-          /p:PublicSign=false $(VersioningProperties) /p:Configuration=$(BuildConfiguration) /p:CommitSHA=$(Build.SourceVersion)
+          dotnet pack eng/service.proj -warnaserror
+          /p:ServiceDirectory=${{parameters.ServiceToBuild}}
+          /p:PublicSign=false $(VersioningProperties)
+          /p:Configuration=$(BuildConfiguration)
+          /p:CommitSHA=$(Build.SourceVersion)
+          /p:ArtifactsPackagesDir=$(Build.ArtifactStagingDirectory)
         displayName: "Build and Package"
         env:
           DOTNET_SKIP_FIRST_TIME_EXPERIENCE: 1
           DOTNET_CLI_TELEMETRY_OPTOUT: 1
           DOTNET_MULTILEVEL_LOOKUP: 0
-      - task: PublishPipelineArtifact@1
-        condition: succeeded()
-        inputs:
-          artifactName: packages
-          path: $(Build.ArtifactStagingDirectory)
       - template: ../steps/archetype-sdk-docs.yml
         parameters:
           ServiceDirectory: ${{parameters.ServiceDirectory}}
           Artifacts: ${{parameters.Artifacts}}
           DocGenerationDir: "$(Build.SourcesDirectory)/doc/ApiDocGeneration"
           LibType: 'client'
+      - task: PublishPipelineArtifact@1
+        condition: succeeded()
+        inputs:
+          artifactName: packages
+          path: $(Build.ArtifactStagingDirectory)
   - job: "Analyze"
     condition: ne(variables['Skip.Analyze'], true)
     variables:
@@ -156,4 +160,4 @@ jobs:
         displayName: 'Publish code coverage report'
         inputs:
           codeCoverageTool: 'Cobertura'
-          summaryFileLocation: 'Cobertura.xml'
+          summaryFileLocation: 'Cobertura.xml'

eng/pipelines/templates/stages/archetype-net-release.yml

@@ -23,21 +23,14 @@ stages:
                   artifact: ${{parameters.ArtifactName}}
                   timeoutInMinutes: 5
 
-                - ${{ each artifact in parameters.Artifacts }}:
-                  - pwsh: |
-                      New-Item -Type Directory -Name staging -Path $(Pipeline.Workspace) -Force
-                      Copy-Item $(Pipeline.Workspace)/${{parameters.ArtifactName}}/${{artifact.name}}.[0-9]*.[0-9]*.[0-9]* $(Pipeline.Workspace)/staging
-                      Get-ChildItem $(Pipeline.Workspace)/staging
-                    displayName: Copying ${{artifact.name}} to staging directory
-
                 - template: tools/clone-buildtools/clone-buildtools.yml@azure-sdk-build-tools
 
                 - template: pipelines/steps/net-signing.yml@azure-sdk-build-tools
                   parameters:
-                    PackagesPath: $(Pipeline.Workspace)/staging
+                    PackagesPath:  $(Pipeline.Workspace)/${{parameters.ArtifactName}}
                     BuildToolsPath: $(AzureSDKBuildToolsPath)
 
-                - publish: $(Pipeline.Workspace)/staging
+                - publish: $(Pipeline.Workspace)/${{parameters.ArtifactName}}
                   artifact: ${{parameters.ArtifactName}}-signed
                   displayName: 'Store signed packages in ${{parameters.ArtifactName}}-signed artifact'
 
@@ -66,14 +59,9 @@ stages:
                         PackageName: ${{artifact.name}}
                         ServiceName: ${{parameters.ServiceDirectory}}
                         ForRelease: true
-                    - template: /eng/pipelines/templates/steps/stage-artifacts.yml
-                      parameters:
-                        SourceFolder: ${{parameters.ArtifactName}}-signed
-                        TargetFolder: ${{artifact.safeName}}
-                        FileFilter: ${{artifact.name}}.[0-9]*.[0-9]*.[0-9]*
                     - template: /eng/common/pipelines/templates/steps/create-tags-and-git-release.yml
                       parameters:
-                        ArtifactLocation: $(Pipeline.Workspace)/${{artifact.safeName}}
+                        ArtifactLocation: $(Pipeline.Workspace)/${{parameters.ArtifactName}}-signed/${{artifact.name}}
                         PackageRepository: Nuget
                         ReleaseSha: $(Build.SourceVersion)
                         RepoId: Azure/azure-sdk-for-net
@@ -93,27 +81,18 @@ stages:
                   deploy:
                     steps:
                       - checkout: none
-                      - template: /eng/pipelines/templates/steps/stage-artifacts.yml
-                        parameters:
-                          SourceFolder: ${{parameters.ArtifactName}}-signed
-                          TargetFolder: staging
-                          FileFilter: ${{artifact.name}}.[0-9]*.[0-9]*.[0-9]*
-                      - pwsh: |
-                          Get-ChildItem -Recurse $(Pipeline.Workspace)/staging
-                        workingDirectory: $(Pipeline.Workspace)
-                        displayName: Output Visible Artifacts
                       - task: NuGetCommand@2
                         displayName: 'Publish ${{artifact.name}} package to NuGet.org'
                         inputs:
                           command: push
-                          packagesToPush: '$(Pipeline.Workspace)/staging/**/*.nupkg;!$(Pipeline.Workspace)/staging/**/*.symbols.nupkg'
+                          packagesToPush: '$(Pipeline.Workspace)/${{parameters.ArtifactName}}-signed/${{artifact.name}}/*.nupkg;!$(Pipeline.Workspace)//${{parameters.ArtifactName}}-signed/${{artifact.name}}/*.symbols.nupkg'
                           nuGetFeedType: external
                           publishFeedCredentials: Nuget.org
                       - task: NuGetCommand@2
                         displayName: 'Publish to DevOps Feed'
                         inputs:
                           command: push
-                          packagesToPush: '$(Pipeline.Workspace)/staging/**/*.nupkg;!$(Pipeline.Workspace)/staging/**/*.symbols.nupkg'
+                          packagesToPush: '$(Pipeline.Workspace)/${{parameters.ArtifactName}}-signed/${{artifact.name}}/*.nupkg;!$(Pipeline.Workspace)/${{parameters.ArtifactName}}-signed/${{artifact.name}}/*.symbols.nupkg'
                           publishVstsFeed: ${{ parameters.DevOpsFeedID }}
 
           - ${{if ne(artifact.skipSymbolsUpload, 'true')}}:
@@ -132,20 +111,15 @@ stages:
                     steps:
                       - checkout: none
                       - template: tools/clone-buildtools/clone-buildtools.yml@azure-sdk-build-tools
-                      - template: /eng/pipelines/templates/steps/stage-artifacts.yml
-                        parameters:
-                          SourceFolder: ${{parameters.ArtifactName}}-signed
-                          TargetFolder: staging
-                          FileFilter: ${{artifact.name}}.[0-9]*.[0-9]*.[0-9]*
-                      - pwsh: |
-                          Get-ChildItem -Recurse $(Pipeline.Workspace)/staging
-                        workingDirectory: $(Pipeline.Workspace)
-                        displayName: Output Visible Artifacts
                       - task: MSBuild@1
                         displayName: 'Upload Symbols for ${{artifact.name}}'
                         inputs:
                           solution: '$(AzureSDKBuildToolsPath)/tools/symboltool/SymbolUploader.proj'
-                          msbuildArguments: '/p:PackagesPath=$(Pipeline.Workspace)/staging /p:MSPublicSymbolsPAT=$(azuresdk-microsoftpublicsymbols-devops-pat) /p:MSSymbolsPAT=$(azuresdk-microsoft-devops-pat) /p:AzureSDKSymbolsPAT=$(azuresdk-azure-sdk-devops-pat)'
+                          msbuildArguments: >-
+                            /p:PackagesPath=$(Pipeline.Workspace)/${{parameters.ArtifactName}}-signed/${{artifact.name}}
+                            /p:MSPublicSymbolsPAT=$(azuresdk-microsoftpublicsymbols-devops-pat)
+                            /p:MSSymbolsPAT=$(azuresdk-microsoft-devops-pat)
+                            /p:AzureSDKSymbolsPAT=$(azuresdk-azure-sdk-devops-pat)
 
           - ${{if ne(artifact.skipPublishDocMs, 'true')}}:
             - deployment: PublicDocsMS
@@ -165,22 +139,13 @@ stages:
                   deploy:
                     steps:
                       - checkout: self
-                      - template: /eng/pipelines/templates/steps/stage-artifacts.yml
-                        parameters:
-                          SourceFolder: ${{parameters.ArtifactName}}-signed
-                          TargetFolder: ${{artifact.safeName}}
-                          FileFilter: ${{artifact.name}}.[0-9]*.[0-9]*.[0-9]*
-                      - pwsh: |
-                          Get-ChildItem -Recurse $(Pipeline.Workspace)/${{artifact.safeName}}
-                        workingDirectory: $(Pipeline.Workspace)
-                        displayName: Output Visible Artifacts
                       - template: /eng/common/pipelines/templates/steps/get-pr-owners.yml
                         parameters:
                           TargetVariable: "OwningGHUser"
                           ServiceDirectory: ${{parameters.ServiceDirectory}}
                       - template: /eng/common/pipelines/templates/steps/docs-metadata-release.yml
                         parameters:
-                          ArtifactLocation: $(Pipeline.Workspace)/${{artifact.safeName}}
+                          ArtifactLocation: $(Pipeline.Workspace)/${{parameters.ArtifactName}}-signed/${{artifact.name}}
                           PackageRepository: Nuget
                           ReleaseSha: $(Build.SourceVersion)
                           RepoId: Azure/azure-sdk-for-net
@@ -210,28 +175,13 @@ stages:
                   deploy:
                     steps:
                       - checkout: self
-                      - template: /eng/pipelines/templates/steps/stage-artifacts.yml
-                        parameters:
-                          SourceFolder: ${{parameters.ArtifactName}}-signed
-                          TargetFolder: ${{artifact.safeName}}/packages
-                          FileFilter: "${{artifact.name}}.[0-9]*.[0-9]*.[0-9]*"
-                          Exclude: "*.symbols.nupkg"
-                      - template: /eng/pipelines/templates/steps/stage-artifacts.yml
-                        parameters:
-                          SourceFolder: Docs.${{artifact.name}}
-                          TargetFolder: ${{artifact.safeName}}/Docs.${{artifact.name}}
-                          FileFilter: '*'
-                      - pwsh: |
-                          Get-ChildItem -Recurse $(Pipeline.Workspace)/${{artifact.safeName}}
-                        workingDirectory: $(Pipeline.Workspace)
-                        displayName: Output Visible Artifacts
                       - template: /eng/common/pipelines/templates/steps/publish-blobs.yml
                         parameters:
-                          FolderForUpload: '$(Pipeline.Workspace)/${{artifact.safeName}}'
+                          FolderForUpload: '$(Pipeline.Workspace)/${{parameters.ArtifactName}}/${{artifact.name}}'
                           BlobSASKey: '$(azure-sdk-docs-prod-sas)'
                           BlobName: '$(azure-sdk-docs-prod-blob-name)'
                           TargetLanguage: 'dotnet'
-                          ArtifactLocation: '$(Pipeline.Workspace)/${{artifact.safeName}}/packages'
+                          ArtifactLocation: '$(Pipeline.Workspace)/${{parameters.ArtifactName}}/${{artifact.name}}'
                           # we override the regular script path because we have cloned the build tools repo as a separate artifact.
                           ScriptPath: 'eng/common/scripts/copy-docs-to-blobstorage.ps1'
 

eng/pipelines/templates/steps/archetype-sdk-docs.yml

@@ -15,13 +15,18 @@ steps:
     displayName: Download and Extract Required Software
 
   - ${{ each artifact in parameters.Artifacts }}:
-    - pwsh: |
-        $(Build.SourcesDirectory)/doc/ApiDocGeneration/Generate-Api-Docs.ps1 -ArtifactName '${{artifact.name}}' -ServiceDirectory '${{parameters.ServiceDirectory}}' `
-        -ArtifactsDirectoryName '${{artifact.directoryName}}' `
-        -LibType '${{parameters.LibType}}' -RepoRoot $(Build.SourcesDirectory) -BinDirectory $(Build.BinariesDirectory) -DocGenDir ${{parameters.DocGenerationDir}} -verbose
+    - task: PowerShell@2
       displayName: Build DocFx Documenttion Site for ${{artifact.name}}
-    - task: PublishPipelineArtifact@0
-      condition: succeeded()
       inputs:
-        artifactName: Docs.${{artifact.name}}
-        targetPath: $(PublishTargetPath)
+        pwsh: true
+        filePath: $(Build.SourcesDirectory)/doc/ApiDocGeneration/Generate-Api-Docs.ps1
+        arguments: >
+          -ArtifactName '${{artifact.name}}'
+          -ServiceDirectory '${{parameters.ServiceDirectory}}'
+          -ArtifactsDirectoryName '${{artifact.directoryName}}'
+          -LibType '${{parameters.LibType}}'
+          -RepoRoot $(Build.SourcesDirectory)
+          -BinDirectory $(Build.BinariesDirectory)
+          -DocGenDir ${{parameters.DocGenerationDir}}
+          -ArtifactStagingDirectory '$(Build.ArtifactStagingDirectory)'
+          -verbose

sdk/template/Azure.Template/CHANGELOG.md

@@ -1,6 +1,24 @@
 # Release History
-# 1.0.3-beta.11 (Unreleased)
+## 1.0.3-beta.17 (2020-09-16)
+- Package Artifacts directory restructure, attempt 7
 
+## 1.0.3-beta.16 (2020-09-15)
+- Package Artifacts directory restructure, attempt 6
+
+## 1.0.3-beta.15 (2020-09-15)
+- Package Artifacts directory restructure, attempt 5
+
+## 1.0.3-beta.14 (2020-09-14)
+- Package Artifacts directory restructure, attempt 4
+
+## 1.0.3-beta.13 (2020-09-14)
+- Package Artifacts directory restructure, attempt 3
+
+## 1.0.3-beta.12 (2020-09-11)
+- Package Artifacts directory restructure, attempt 2
+
+## 1.0.3-beta.11 (2020-09-10)
+- Package Artifacts directory restructure
 
 ## 1.0.3-beta.10 (2020-09-09)
 - Enable the master link replacement feature, attempt 3
@@ -50,6 +68,9 @@
 ## 1.0.3-beta.1 (2020-08-27)
 - Test new alpha beta versioning
 
+## 1.0.2-preview.17 (2020-07-24)
+- Test release pipeline
+
 ## 1.0.2-preview.15 (2020-06-29)
 - Test release pipeline
 

sdk/template/Azure.Template/src/Azure.Template.csproj

@@ -2,7 +2,7 @@
   <PropertyGroup>
     <Description>This is a template project to demonstrate how to create a package that uses code generation as well as use for testing our build and release pipelines</Description>
     <AssemblyTitle>Azure SDK Template</AssemblyTitle>
-    <Version>1.0.3-beta.11</Version>
+    <Version>1.0.3-beta.17</Version>
     <PackageTags>Azure Template</PackageTags>
     <TargetFrameworks>$(RequiredTargetFrameworks)</TargetFrameworks>
   </PropertyGroup>