Vcpkg manifest and binary caching (#3329)

* binary cache and manifest

* update packages

* udpate

* add cache to private pipeline

* next try

* aver

* other

* again

* aver quick

* more

* this way

* again

* one more

* print

* test

* use depend

* more deps

* dep

* Apply suggestions from code review

Co-authored-by: Daniel Jurek <djurek@microsoft.com>

* updated to use group variable

* update identity

* end line

* Update vcpkg.json

* Apply suggestions from code review

* cspell

* remove comment

* updates

* make cache mode depend on SAS env var

* map env var only for internal pipelines

* other approach

* what about this

* and this

* try

* amd

* another

* extra step

* typo

* override for internal

* use default succeded

* azure core update manifest

* a

* run cmake-generate nightly as well

* check for SAS

* check cache

* no secret

* fix is secret

* pass explicit

* use secret all the time

* char

* One more

* export

* echos

* last

* array

* remove question

* ok

* weird

* use account key

* substring

* VCPKG_BINARY_SOURCES

* Add module installation

* task:

* Correct pathing for module

* update source gen

* format

* update spelling

* IsWindoows

* Use pwsh

* Cannot clobber with PSModule-Helpers. Attempt plain install

* Attempt plain install

* Revert unnecessary change to Update-DocsMsToc.ps1

* template ready

* curl is required on Windows as well for some CI gates

* attestation

* fix format

Co-authored-by: Daniel Jurek <djurek@microsoft.com>

.vscode/cspell.json

@@ -143,7 +143,8 @@
       "filename": "**/eng/**",
       "words": [
         "TEAMPROJECTID",
-        "issecret"
+        "issecret",
+        "cppvcpkgcache"
       ]
     },
     {

eng/pipelines/templates/jobs/archetype-sdk-client.yml

@@ -61,7 +61,6 @@ jobs:
         name: azsdk-pool-mms-win-2019-general
         vmImage: MMS2019
       variables:
-        VcpkgDependencies: curl[winssl] libxml2 openssl
         VCPKG_DEFAULT_TRIPLET: 'x64-windows-static'
         Package.EnableSBOMSigning: true
       steps:
@@ -98,8 +97,6 @@ jobs:
         # TODO: We should be able to generate docs without having to install these
         # prerequisites:
         - template: /eng/pipelines/templates/steps/vcpkg.yml
-          parameters:
-            DependenciesVariableName: VcpkgDependencies
 
         - template: /eng/pipelines/templates/steps/cmake-build.yml
           parameters:

eng/pipelines/templates/jobs/archetype-sdk-cmake-generate.yml

@@ -14,9 +14,6 @@ parameters:
   - name: CMakeSourceTestOptions
     type: object
     default: []
-  - name: CMakeTestDependencies
-    type: string
-    default: ''
 
 jobs:
 - job: CMakeGenerate
@@ -28,19 +25,16 @@ jobs:
         OSVmImage: MMSUbuntu18.04
         CmakeEnvArg: ''
         AptDependencies: 'libcurl4-openssl-dev'
-        VcpkgInstall: ${{ parameters.CMakeTestDependencies }}
         VCPKG_DEFAULT_TRIPLET: 'x64-linux'
       Windows:
         Pool: azsdk-pool-mms-win-2019-general
         OSVmImage: MMS2019
         CmakeEnvArg: ''
-        VcpkgInstall: ${{ parameters.CMakeTestDependencies }}
         VCPKG_DEFAULT_TRIPLET: 'x64-windows'
       Mac:
         Pool: Azure Pipelines
         OSVmImage: macOS-10.15
         CmakeEnvArg: ''
-        VcpkgInstall: 'openssl ${{ parameters.CMakeTestDependencies }}'
         VCPKG_DEFAULT_TRIPLET: 'x64-osx'
   pool:
     name: $(Pool)
@@ -63,8 +57,6 @@ jobs:
     displayName: Install dependencies from apt
 
   - template: /eng/pipelines/templates/steps/vcpkg.yml
-    parameters:
-      DependenciesVariableName: VcpkgInstall
 
   - script: cmake --version
     displayName: cmake --version
@@ -122,6 +114,8 @@ jobs:
       brew install $(BrewDependencies) 
     condition: and(succeeded(), ne(variables['BrewDependencies'], ''))
     displayName: Install dependencies from brew
+  
+  - template: /eng/pipelines/templates/steps/vcpkg.yml
 
   - script: cmake --version
     displayName: cmake --version

eng/pipelines/templates/jobs/archetype-sdk-tests.yml

@@ -35,7 +35,6 @@ jobs:
       Linux_x64_gcc5_with_unit_test:
         Pool: azsdk-pool-mms-ubuntu-1804-general
         OSVmImage: MMSUbuntu18.04
-        VcpkgInstall: 'curl[ssl] libxml2 openssl'
         VCPKG_DEFAULT_TRIPLET: 'x64-linux'
         # Can't set CC and CXX to env because it would affect VCPKG building which requires g++ > 7
         # So, this conf will set the CXX for cmake inline.
@@ -45,7 +44,6 @@ jobs:
       Linux_x64_with_unit_test:
         Pool: azsdk-pool-mms-ubuntu-1804-general
         OSVmImage: MMSUbuntu18.04
-        VcpkgInstall: 'curl[ssl] libxml2 openssl'
         VCPKG_DEFAULT_TRIPLET: 'x64-linux'
         CmakeArgs: ' -DBUILD_TESTING=ON -DBUILD_PERFORMANCE_TESTS=ON -DRUN_LONG_UNIT_TESTS=ON -DCMAKE_BUILD_TYPE=Debug -DBUILD_CODE_COVERAGE=ON'
         AptDependencies: 'gcovr lcov'
@@ -57,15 +55,13 @@ jobs:
       Ubuntu20_x64_with_unit_test:
         Pool: azsdk-pool-mms-ubuntu-1804-general
         OSVmImage: MMSUbuntu18.04
-        VcpkgInstall: 'curl[ssl] libxml2 openssl'
         VCPKG_DEFAULT_TRIPLET: 'x64-linux'
         CmakeArgs: ' -DBUILD_TESTING=ON -DBUILD_PERFORMANCE_TESTS=ON -DRUN_LONG_UNIT_TESTS=ON -DCMAKE_BUILD_TYPE=Release -DBUILD_SAMPLES=ON'
         AZURE_CORE_ENABLE_JSON_TESTS: 1
         BuildArgs: '-j 4'
       Ubuntu20_samples:
         Pool: azsdk-pool-mms-ubuntu-1804-general
         OSVmImage: MMSUbuntu18.04
-        VcpkgInstall: 'curl[ssl] libxml2 openssl'
         VCPKG_DEFAULT_TRIPLET: 'x64-linux'
         # Builds samples and run them against the deployed resources. Samples requires Azure Account, so only works on live tests.
         CmakeArgs: ' -DBUILD_TESTING=ON -DBUILD_SAMPLES=ON -DCMAKE_BUILD_TYPE=Release '
@@ -74,7 +70,6 @@ jobs:
       Ubuntu20_x64_no_rtti:
         Pool: azsdk-pool-mms-ubuntu-1804-general
         OSVmImage: MMSUbuntu18.04
-        VcpkgInstall: 'curl[ssl] libxml2 openssl'
         VCPKG_DEFAULT_TRIPLET: 'x64-linux'
         CmakeArgs: ' -DBUILD_RTTI=OFF -DCMAKE_BUILD_TYPE=Release '
         BuildArgs: '-j 4'
@@ -126,7 +121,6 @@ jobs:
       Win_x86_with_unit_test_libcurl:
         Pool: azsdk-pool-mms-win-2019-general
         OSVmImage: MMS2019
-        VcpkgInstall: 'curl[winssl] openssl'
         VCPKG_DEFAULT_TRIPLET: 'x86-windows-static'
         CMAKE_GENERATOR: 'Visual Studio 16 2019'
         CMAKE_GENERATOR_PLATFORM: Win32
@@ -136,7 +130,6 @@ jobs:
       Win_x64_with_unit_test_libcurl:
         Pool: azsdk-pool-mms-win-2019-general
         OSVmImage: MMS2019
-        VcpkgInstall: 'curl[winssl] openssl'
         VCPKG_DEFAULT_TRIPLET: 'x64-windows-static'
         CMAKE_GENERATOR: 'Visual Studio 16 2019'
         CMAKE_GENERATOR_PLATFORM: x64
@@ -158,7 +151,6 @@ jobs:
       MacOS_x64_with_unit_test:
         Pool: Azure Pipelines
         OSVmImage: 'macOS-10.15'
-        VcpkgInstall: 'curl[ssl] libxml2 openssl'
         VCPKG_DEFAULT_TRIPLET: 'x64-osx'
         CmakeArgs: ' -DBUILD_TESTING=ON -DRUN_LONG_UNIT_TESTS=ON -DBUILD_PERFORMANCE_TESTS=ON -DBUILD_SAMPLES=ON'
         AZURE_CORE_ENABLE_JSON_TESTS: 1
@@ -215,8 +207,6 @@ jobs:
     displayName: Install dependencies from apt
 
   - template: /eng/pipelines/templates/steps/vcpkg.yml
-    parameters:
-      DependenciesVariableName: VcpkgInstall
 
   - script: |
       dotnet tool install -g dotnet-reportgenerator-globaltool

eng/pipelines/templates/jobs/ci.tests.yml

@@ -95,8 +95,6 @@ jobs:
         displayName: Set Xcode version
 
       - template: /eng/pipelines/templates/steps/vcpkg.yml
-        parameters:
-          DependenciesVariableName: VcpkgInstall
 
       # Validate all the files are formatted correctly according to the
       # .clang-format file. This step runs on linux only only and assumes that

eng/pipelines/templates/stages/archetype-sdk-client.yml

@@ -47,22 +47,17 @@ parameters:
 - name: CMakeSourceTestOptions
   type: object
   default: []
-- name: CMakeTestDependencies
-  type: string
-  default: ''
 
 stages:
-  - ${{ if eq(variables['System.TeamProject'], 'public') }}:
-    - stage: CMakeGeneration
-      jobs:
-        - template: /eng/pipelines/templates/jobs/archetype-sdk-cmake-generate.yml
-          parameters:
-            ServiceDirectory: ${{ parameters.ServiceDirectory }}
-            Artifacts: ${{ parameters.Artifacts }}
-            ArtifactsSource: ${{ parameters.ArtifactsSource }}
-            CMakeTestOptions: ${{ parameters.CMakeTestOptions }}
-            CMakeSourceTestOptions: ${{ parameters.CMakeSourceTestOptions }}
-            CMakeTestDependencies: ${{ parameters.CMakeTestDependencies }}
+  - stage: CMakeGeneration
+    jobs:
+      - template: /eng/pipelines/templates/jobs/archetype-sdk-cmake-generate.yml
+        parameters:
+          ServiceDirectory: ${{ parameters.ServiceDirectory }}
+          Artifacts: ${{ parameters.Artifacts }}
+          ArtifactsSource: ${{ parameters.ArtifactsSource }}
+          CMakeTestOptions: ${{ parameters.CMakeTestOptions }}
+          CMakeSourceTestOptions: ${{ parameters.CMakeSourceTestOptions }}
 
   - stage: Build
     dependsOn: []

eng/pipelines/templates/stages/platform-matrix.json

@@ -16,7 +16,6 @@
             "StaticConfigs": {
                 "_": {
                     "Pool": "Azure Pipelines",
-                    "VcpkgInstall": "curl[ssl] libxml2 openssl",
                     "BuildArgs": "-j 10",
                     "VCPKG_DEFAULT_TRIPLET": "x64-osx",
                     "CmakeArgs": " -DBUILD_TESTING=ON -DBUILD_PERFORMANCE_TESTS=ON -DRUN_LONG_UNIT_TESTS=ON -DBUILD_TRANSPORT_CURL=ON",
@@ -42,18 +41,14 @@
             },
             "TargetPlatform": {
                 "Win32Api_curl": {
-                    "VcpkgInstall": "curl[winssl] openssl",
-                    "CmakeArgs": " -DBUILD_TRANSPORT_CURL=ON",
-                    "BuildArgs": "--parallel 8"
+                    "CmakeArgs": " -DBUILD_TRANSPORT_CURL=ON"
                 },
                 "Win32Api_release_curl": {
-                    "VcpkgInstall": "curl[winssl] openssl",
                     "CmakeArgs": " -DBUILD_TESTING=ON -DBUILD_PERFORMANCE_TESTS=ON -DRUN_LONG_UNIT_TESTS=ON -DBUILD_TRANSPORT_CURL=ON",
                     "BuildArgs": "--parallel 8 --config Release",
                     "PublishMapFiles": "true"
                 },
                 "Win32Api_debug_tests": {
-                    "VcpkgInstall": "curl[winssl] openssl",
                     "CmakeArgs": " -DBUILD_TESTING=ON -DBUILD_PERFORMANCE_TESTS=ON -DRUN_LONG_UNIT_TESTS=ON -DBUILD_TRANSPORT_CURL=ON -DBUILD_TRANSPORT_WINHTTP=ON",
                     "BuildArgs": "--parallel 8 --config Debug",
                     "PublishMapFiles": "true"
@@ -109,7 +104,6 @@
                 "Ubuntu18": {
                     "OSVmImage": "MMSUbuntu18.04",
                     "Pool": "azsdk-pool-mms-ubuntu-1804-general",
-                    "VcpkgInstall": "curl[ssl] libxml2 openssl",
                     "VCPKG_DEFAULT_TRIPLET": "x64-linux",
                     "BuildArgs": "-j 10"
                 }
@@ -136,8 +130,7 @@
                     "CODE_COVERAGE": "enabled",
                     "PublishMapFiles": "true"
                 },
-                "included": {
-                },
+                "included": {},
                 "included_release": {
                     "CMAKE_BUILD_TYPE": "Release",
                     "CmakeArgs": " -DBUILD_TESTING=ON -DBUILD_PERFORMANCE_TESTS=ON -DRUN_LONG_UNIT_TESTS=ON",
@@ -154,7 +147,6 @@
                 "Ubuntu20": {
                     "OSVmImage": "MMSUbuntu20.04",
                     "Pool": "azsdk-pool-mms-ubuntu-2004-general",
-                    "VcpkgInstall": "curl[ssl] libxml2 openssl",
                     "VCPKG_DEFAULT_TRIPLET": "x64-linux",
                     "BuildArgs": "-j 10",
                     "CC": "/usr/bin/clang-11",

eng/pipelines/templates/steps/cmake-build.yml

@@ -14,9 +14,12 @@ steps:
     workingDirectory: build
     displayName: cmake --version
 
-  - script: ${{ parameters.Env }} cmake ${{ parameters.GenerateArgs }} ..
+  - script: |
+      ${{ parameters.Env }} cmake ${{ parameters.GenerateArgs }} ..
     workingDirectory: build
     displayName: cmake generate
+    env:
+      VCPKG_BINARY_SOURCES: $(VCPKG_BINARY_SOURCES_SECRET)
 
 # Core should build all cmake tagets
   - ${{ if and(eq(parameters.Build, true), eq(parameters.ServiceDirectory, 'core')) }}:

eng/pipelines/templates/steps/cmake-generate.yml

@@ -16,9 +16,12 @@ steps:
     workingDirectory: ${{ parameters.CmakeGeneratePath }}/build
     displayName: Show current path
 
-  - script: ${{ parameters.Env }} cmake ${{ parameters.GenerateArgs }} ..
+  - script: |
+      ${{ parameters.Env }} cmake ${{ parameters.GenerateArgs }} ..
     workingDirectory: ${{ parameters.CmakeGeneratePath }}/build
     displayName: cmake generate
+    env:
+      VCPKG_BINARY_SOURCES: $(VCPKG_BINARY_SOURCES_SECRET)
 
   - script: rm -rf build
     workingDirectory: ${{ parameters.CmakeGeneratePath }}

eng/pipelines/templates/steps/vcpkg.yml

@@ -1,9 +1,3 @@
-parameters:
-  # Use the variable name itself (no $() or other wrapping syntax). This is
-  # because we use runtime and macro expressions for conditions and script
-  # invocations
-  DependenciesVariableName: vcpkg.deps
-
 steps:
   - pwsh: |
       $TargetPath = "$(Agent.TempDirectory)/vcpkg"
@@ -17,46 +11,23 @@ steps:
       Write-Host "##vso[task.prependpath]$TargetPath"
       Write-Host "##vso[task.setvariable variable=VCPKG_INSTALLATION_ROOT]$TargetPath"
       Write-Host "##vso[task.setvariable variable=VcpkgCommit]$VcpkgCommit"
+      Write-Host "##vso[task.setvariable variable=VCPKG_BINARY_SOURCES_SECRET;issecret=true;]clear;x-azblob,https://cppvcpkgcache.blob.core.windows.net/public-vcpkg-container,,read"
     displayName: Set Vcpkg Variables
-    condition: >-
-      and(
-        succeeded(),
-        not(eq(variables['${{ parameters.DependenciesVariableName }}'], ''))
-      )
 
-  # This task times out after 10 minutes. An analysis of a subset of Vcpkg Cache
-  # runs showed that after 10 minutes the cache step is very unlikely to
-  # succeed.
-  - task: Cache@2
+  - task: PowerShell@2
     inputs:
-      key: >-
-        $(Agent.JobName)
-        | "$(VcpkgCommit)"
-        | $(Agent.Os)
-        | $(${{ parameters.DependenciesVariableName }})
-      path: $(VCPKG_INSTALLATION_ROOT)
-      cacheHitVar: VcpkgRestoredFromCache
-    displayName: Vcpkg Cache
-    timeoutInMinutes: 10
-    condition: >-
-      and(
-        succeeded(),
-        not(eq(variables['${{ parameters.DependenciesVariableName }}'], '')),
-        not(eq(variables['Skip.VcpkgCache'], 'true'))
-      )
+      pwsh: true
+      targetType: filePath
+      filePath: eng/scripts/Set-VcpkgWriteModeCache.ps1
+      arguments: -StorageAccountKey '$(cpp-vcpkg-cache-storage-key)'
+    displayName: Set Vcpkg Write-mode Cache
+    condition: eq(variables['System.TeamProject'], 'internal')
 
   - task: PowerShell@2
     inputs:
       targetType: filePath
       filePath: eng/scripts/vcpkg.ps1
       arguments: >-
         -Ref $(VcpkgCommit)
-        -Dependencies "$(${{ parameters.DependenciesVariableName }})"
         -VcpkgPath $(VCPKG_INSTALLATION_ROOT)
       pwsh: true
-    condition: >-
-      and(
-        succeeded(),
-        not(eq(variables['${{ parameters.DependenciesVariableName }}'], '')),
-        ne(variables['VcpkgRestoredFromCache'], true)
-      )

eng/scripts/Set-VcpkgWriteModeCache.ps1

@@ -0,0 +1,17 @@
+param(
+    [string] $StorageAccountKey
+)
+Install-Module "Az.Storage" -AllowClobber -Force
+
+$ctx = New-AzStorageContext `
+    -StorageAccountName 'cppvcpkgcache' `
+    -StorageAccountKey $StorageAccountKey
+$token = New-AzStorageAccountSASToken `
+    -Service Blob `
+    -ResourceType Object `
+    -Permission "rwc" `
+    -Context $ctx
+$vcpkgBinarySourceSas = $token.Substring(1)
+
+Write-Host "Setting vcpkg binary cache to read and write"
+Write-Host "##vso[task.setvariable variable=VCPKG_BINARY_SOURCES_SECRET;issecret=true;]clear;x-azblob,https://cppvcpkgcache.blob.core.windows.net/public-vcpkg-container,$vcpkgBinarySourceSas,readwrite"

eng/scripts/vcpkg.ps1

@@ -4,10 +4,6 @@ Param (
     [ValidateNotNullOrEmpty()]
     [string] $Ref = (Get-Content "$PSScriptRoot/../vcpkg-commit.txt"),
 
-    [Parameter(Mandatory = $true)]
-    [ValidateNotNullOrEmpty()]
-    [string] $Dependencies,
-
     [Parameter()]
     [ValidateNotNullOrEmpty()]
     [string] $VcpkgPath = "$PSScriptRoot/../../vcpkg"
@@ -23,11 +19,9 @@ try {
 
     if ($IsWindows) {
         .\bootstrap-vcpkg.bat
-        .\vcpkg.exe install $Dependencies.Split(' ')
     } else {
         ./bootstrap-vcpkg.sh
-        ./vcpkg install $Dependencies.Split(' ')
     }
 } finally {
     Set-Location $initialDirectory
-}
+}

sdk/attestation/azure-security-attestation/vcpkg.json

@@ -0,0 +1,18 @@
+{
+  "name": "azure-security-attestation-cpp",
+  "version-semver": "1.0.0-beta.1",
+  "dependencies": [
+    {
+      "name": "azure-core-cpp"
+    },
+    {
+      "name": "vcpkg-cmake"
+    },
+    {
+      "name": "vcpkg-cmake-config"
+    },
+    {
+      "name": "openssl"
+    }
+  ]
+}