Using Dynamic Matrix Config

eng/common/pipelines/templates/jobs/generate-job-matrix.yml

@@ -42,6 +42,11 @@ parameters:
 - name: PreGenerationSteps
   type: stepList
   default: []
+- name: EnablePRGeneration
+  type: boolean
+  default: false
+- name: PRMatrix
+  type: object
 # Mappings to OS name required at template compile time by 1es pipeline templates
 - name: Pools
   type: object
@@ -84,57 +89,87 @@ jobs:
 
     - ${{ parameters.PreGenerationSteps }}
 
-    - ${{ each config in parameters.MatrixConfigs }}:
-      - ${{ each pool in parameters.Pools }}:
-        - ${{ if eq(config.GenerateVMJobs, 'true') }}:
+    - ${{ if eq(parameters.EnablePRGeneration, false) }}:
+      - ${{ each config in parameters.MatrixConfigs }}:
+        - ${{ each pool in parameters.Pools }}:
+          - ${{ if eq(config.GenerateVMJobs, 'true') }}:
+            - task: Powershell@2
+              inputs:
+                pwsh: true
+                filePath: eng/common/scripts/job-matrix/Create-JobMatrix.ps1
+                arguments: >
+                  -ConfigPath ${{ config.Path }}
+                  -Selection ${{ config.Selection }}
+                  -DisplayNameFilter '$(displayNameFilter)'
+                  -Filters '${{ join(''',''', parameters.MatrixFilters) }}', 'container=^$', 'SupportedClouds=^$|${{ parameters.CloudConfig.Cloud }}', 'Pool=${{ pool.filter }}'
+                  -Replace '${{ join(''',''', parameters.MatrixReplace) }}'
+                  -NonSparseParameters '${{ join(''',''', config.NonSparseParameters) }}'
+              displayName: Create ${{ pool.name }} Matrix ${{ config.Name }}
+              name: vm_job_matrix_${{ config.Name }}_${{ pool.name }}
+          - ${{ if eq(config.GenerateContainerJobs, 'true') }}:
+            - task: Powershell@2
+              inputs:
+                pwsh: true
+                filePath: eng/common/scripts/job-matrix/Create-JobMatrix.ps1
+                arguments: >
+                  -ConfigPath ${{ config.Path }}
+                  -Selection ${{ config.Selection }}
+                  -DisplayNameFilter '$(displayNameFilter)'
+                  -Filters '${{ join(''',''', parameters.MatrixFilters) }}', 'container=^$', 'SupportedClouds=^$|${{ parameters.CloudConfig.Cloud }}', 'Pool=${{ pool.filter }}'
+                  -NonSparseParameters '${{ join(''',''', config.NonSparseParameters) }}'
+              displayName: Create ${{ pool.name }} Container Matrix ${{ config.Name }}
+              name: container_job_matrix_${{ config.Name }}_${{ pool.name }}
+
+      # This else being set also currently assumes that the $(Build.ArtifactStagingDirectory)/PackageInfo folder is populated by PreGenerationSteps.
+      # Not currently not hardcoded, so not doing the needful and populating this folder before we hit this step will result in generation errors.
+      - ${{ else }}:
+        - ${{ each pool in parameters.Pools }}:
           - task: Powershell@2
             inputs:
               pwsh: true
-              filePath: eng/common/scripts/job-matrix/Create-JobMatrix.ps1
+              filePath: eng/common/scripts/job-matrix/Create-PrJobMatrix.ps1
               arguments: >
-                -ConfigPath ${{ config.Path }}
-                -Selection ${{ config.Selection }}
+                -PackagePropertiesFolder $(Build.ArtifactStagingDirectory)/PackageInfo
+                -PRMatrixFile ${{ parameters.PRMatrix.Path }}
+                -PRMatrixSetting ${{ parameters.PRMatrix.Setting }}
                 -DisplayNameFilter '$(displayNameFilter)'
                 -Filters '${{ join(''',''', parameters.MatrixFilters) }}', 'container=^$', 'SupportedClouds=^$|${{ parameters.CloudConfig.Cloud }}', 'Pool=${{ pool.filter }}'
                 -Replace '${{ join(''',''', parameters.MatrixReplace) }}'
-                -NonSparseParameters '${{ join(''',''', config.NonSparseParameters) }}'
-            displayName: Create ${{ pool.name }} Matrix ${{ config.Name }}
-            name: vm_job_matrix_${{ config.Name }}_${{ pool.name }}
+            displayName: Create ${{ pool.name }} PR Matrix
+            name: vm_job_matrix_pr_${{ pool.name }}
 
-        - ${{ if eq(config.GenerateContainerJobs, 'true') }}:
-          - task: Powershell@2
-            inputs:
-              pwsh: true
-              filePath: eng/common/scripts/job-matrix/Create-JobMatrix.ps1
-              arguments: >
-                -ConfigPath ${{ config.Path }}
-                -Selection ${{ config.Selection }}
-                -DisplayNameFilter '$(displayNameFilter)'
-                -Filters '${{ join(''',''', parameters.MatrixFilters) }}', 'container=^$', 'SupportedClouds=^$|${{ parameters.CloudConfig.Cloud }}', 'Pool=${{ pool.filter }}'
-                -NonSparseParameters '${{ join(''',''', config.NonSparseParameters) }}'
-            displayName: Create ${{ pool.name }} Container Matrix ${{ config.Name }}
-            name: container_job_matrix_${{ config.Name }}_${{ pool.name }}
+- ${{ if eq(parameters.EnablePRGeneration, false) }}:
+  - ${{ each config in parameters.MatrixConfigs }}:
+    - ${{ each pool in parameters.Pools }}:
+      - ${{ if eq(config.GenerateVMJobs, 'true') }}:
+        - template: ${{ parameters.JobTemplatePath }}
+          parameters:
+            UsePlatformContainer: false
+            OSName: ${{ pool.os }}
+            Matrix: dependencies.${{ parameters.GenerateJobName }}.outputs['vm_job_matrix_${{ config.Name }}_${{ pool.name }}.matrix']
+            DependsOn: ${{ parameters.GenerateJobName }}
+            CloudConfig: ${{ parameters.CloudConfig }}
+            ${{ each param in parameters.AdditionalParameters }}:
+              ${{ param.key }}: ${{ param.value }}
 
-- ${{ each config in parameters.MatrixConfigs }}:
+      - ${{ if eq(config.GenerateContainerJobs, 'true') }}:
+        - template: ${{ parameters.JobTemplatePath }}
+          parameters:
+            UsePlatformContainer: true
+            OSName: ${{ pool.os }}
+            Matrix: dependencies.${{ parameters.GenerateJobName }}.outputs['vm_job_matrix_${{ config.Name }}_${{ pool.name }}.matrix']
+            DependsOn: ${{ parameters.GenerateJobName }}
+            CloudConfig: ${{ parameters.CloudConfig }}
+            ${{ each param in parameters.AdditionalParameters }}:
+              ${{ param.key }}: ${{ param.value }}
+- ${{ else }}:
   - ${{ each pool in parameters.Pools }}:
-    - ${{ if eq(config.GenerateVMJobs, 'true') }}:
-      - template: ${{ parameters.JobTemplatePath }}
-        parameters:
-          UsePlatformContainer: false
-          OSName: ${{ pool.os }}
-          Matrix: dependencies.${{ parameters.GenerateJobName }}.outputs['vm_job_matrix_${{ config.Name }}_${{ pool.name }}.matrix']
-          DependsOn: ${{ parameters.GenerateJobName }}
-          CloudConfig: ${{ parameters.CloudConfig }}
-          ${{ each param in parameters.AdditionalParameters }}:
-            ${{ param.key }}: ${{ param.value }}
-
-    - ${{ if eq(config.GenerateContainerJobs, 'true') }}:
-      - template: ${{ parameters.JobTemplatePath }}
-        parameters:
-          UsePlatformContainer: true
-          OSName: ${{ pool.os }}
-          Matrix: dependencies.${{ parameters.GenerateJobName }}.outputs['vm_job_matrix_${{ config.Name }}_${{ pool.name }}.matrix']
-          DependsOn: ${{ parameters.GenerateJobName }}
-          CloudConfig: ${{ parameters.CloudConfig }}
-          ${{ each param in parameters.AdditionalParameters }}:
-            ${{ param.key }}: ${{ param.value }}
+    - template: ${{ parameters.JobTemplatePath }}
+      parameters:
+        UsePlatformContainer: false
+        OSName: ${{ pool.os }}
+        Matrix: dependencies.${{ parameters.GenerateJobName }}.outputs['vm_job_matrix_pr_${{ pool.name }}.matrix']
+        DependsOn: ${{ parameters.GenerateJobName }}
+        CloudConfig: ${{ parameters.CloudConfig }}
+        ${{ each param in parameters.AdditionalParameters }}:
+          ${{ param.key }}: ${{ param.value }}

eng/common/pipelines/templates/steps/save-package-properties.yml

@@ -22,7 +22,7 @@ steps:
 
   # The other public CI builds will pass a real service directory, which will not activate the PR diff logic and as such will operate
   # as before this change.
-  - ${{ if and(eq(variables['Build.Reason'], 'PullRequest'), eq(parameters.ServiceDirectory, 'auto')) }}:
+  - ${{ if eq(parameters.ServiceDirectory, 'auto') }}:
       - task: Powershell@2
         displayName: Generate PR Diff
         inputs:

eng/common/scripts/Helpers/Package-Helpers.ps1

@@ -176,4 +176,24 @@ function GetValueSafelyFrom-Yaml {
   }
 
   return [object]$current
-}
+}
+
+function Split-ArrayIntoBatches {
+    param (
+        [Parameter(Mandatory=$true)]
+        [string[]]$InputArray,
+
+        [Parameter(Mandatory=$true)]
+        [int]$BatchSize
+    )
+
+    $batches = @()
+
+    for ($i = 0; $i -lt $InputArray.Count; $i += $BatchSize) {
+        $batch = $InputArray[$i..[math]::Min($i + $BatchSize - 1, $InputArray.Count - 1)]
+
+        $batches += ,$batch
+    }
+
+    return ,$batches
+}

eng/common/scripts/Package-Properties.ps1

@@ -21,6 +21,7 @@ class PackageProps
     # additional packages required for validation of this one
     [string[]]$AdditionalValidationPackages
     [HashTable]$ArtifactDetails
+    [HashTable[]]$CIMatrixConfigs
 
     PackageProps([string]$name, [string]$version, [string]$directoryPath, [string]$serviceDirectory)
     {
@@ -84,8 +85,7 @@ class PackageProps
         $this.Group = $group
     }
 
-    hidden [HashTable]ParseYmlForArtifact([string]$ymlPath) {
-
+    hidden [PSCustomObject]ParseYmlForArtifact([string]$ymlPath) {
         $content = LoadFrom-Yaml $ymlPath
         if ($content) {
             $artifacts = GetValueSafelyFrom-Yaml $content @("extends", "parameters", "Artifacts")
@@ -95,8 +95,21 @@ class PackageProps
                 $artifactForCurrentPackage = $artifacts | Where-Object { $_["name"] -eq $this.ArtifactName -or $_["name"] -eq $this.Name }
             }
 
+            # if we found an artifact for the current package, we should count this ci file as the source of the matrix for this package
             if ($artifactForCurrentPackage) {
-                return [HashTable]$artifactForCurrentPackage
+                $result = [PSCustomObject]@{
+                    ArtifactConfig = [HashTable]$artifactForCurrentPackage
+                    MatrixConfigs = @()
+                }
+
+                # if we know this is the matrix for our file, we should now see if there is a custom matrix config for the package
+                $matrixConfigList = GetValueSafelyFrom-Yaml $content @("extends", "parameters", "MatrixConfigs")
+
+                if ($matrixConfigList) {
+                    $result.MatrixConfigs = matrixConfigList
+                }
+
+                return $result
             }
         }
         return $null
@@ -112,7 +125,10 @@ class PackageProps
             foreach($ciFile in $ciFiles) {
                 $ciArtifactResult = $this.ParseYmlForArtifact($ciFile.FullName)
                 if ($ciArtifactResult) {
-                    $this.ArtifactDetails = [Hashtable]$ciArtifactResult
+                    $this.ArtifactDetails = [Hashtable]$ciArtifactResult.ArtifactConfig
+                    $this.CIMatrixConfigs = $ciArtifactResult.MatrixConfigs
+                    # if this package appeared in this ci file, then we should
+                    # treat this CI file as the source of the Matrix for this package
                     break
                 }
             }

eng/common/scripts/job-matrix/Create-PrJobMatrix.ps1

@@ -0,0 +1,109 @@
+<#
+.SYNOPSIS
+Generates a combined PR job matrix from a package properties folder. It is effectively a combination of
+Create-JobMatrix and distribute-packages-to-matrix
+
+This script is intended to be used within an Azure DevOps pipeline to generate a job matrix for a PR.
+
+.EXAMPLE
+./eng/common/scripts/Create-PRJobMatrix $(Build.ArtifactStagingDirectory)/PackageProperties
+#>
+
+[CmdletBinding()]
+param (
+    [Parameter(Mandatory=$true)][string] $PackagePropertiesFolder,
+    [Parameter(Mandatory=$true)][string] $PRMatrixFile,
+    [Parameter(Mandatory=$true)][string] $PRMatrixSetting,
+    [Parameter(Mandatory=$False)][string] $DisplayNameFilter,
+    [Parameter(Mandatory=$False)][array] $Filters,
+    [Parameter(Mandatory=$False)][array] $Replace,
+    [Parameter()][switch] $CI = ($null -ne $env:SYSTEM_TEAMPROJECTID)
+)
+
+. $PSScriptRoot/job-matrix-functions.ps1
+. $PSScriptRoot/../Helpers/Package-Helpers.ps1
+
+if (!(Test-Path $PackagePropertiesFolder)) {
+    Write-Error "Package Properties folder doesn't exist"
+    exit 1
+}
+
+function GenerateMatrixForConfig {
+    param (
+        [Parameter(Mandatory=$true)][string] $ConfigPath,
+        [Parameter(Mandatory=$True)][string] $Selection,
+        [Parameter(Mandatory=$false)][string] $DisplayNameFilter,
+        [Parameter(Mandatory=$false)][array] $Filters,
+        [Parameter(Mandatory=$false)][array] $Replace
+    )
+
+    $config = GetMatrixConfigFromFile (Get-Content $ConfigPath -Raw)
+    # Strip empty string filters in order to be able to use azure pipelines yaml join()
+    $Filters = $Filters | Where-Object { $_ }
+
+    [array]$matrix = GenerateMatrix `
+        -config $config `
+        -selectFromMatrixType $Selection `
+        -displayNameFilter $DisplayNameFilter `
+        -filters $Filters `
+        -replace $Replace
+
+    return ,$matrix
+}
+
+# calculate general targeting information and create our batches prior to generating any matrix
+# this prototype doesn't handle direct and indirect, it just batches for simplicity of the proto
+$packageProperties = Get-ChildItem -Recurse "$PackagePropertiesFolder" *.json `
+    | % { Get-Content -Path $_.FullName | ConvertFrom-Json }
+
+# in the full proto, we will compress the CIMatrixConfig value into a single string, then use it as a key
+# to group the packages
+# then we will simply iterate over the groups and generate the matrix for each group
+# at the very end, will distribute the packages to the matrix based on the group key (batched over course)
+$configs = @(
+  [PsCustomObject]@{
+    ConfigPath = $PRMatrixFile
+    Selection = "sparse"
+  }
+)
+
+$OverallResult = @()
+foreach($matrixConfig in $configs) {
+  Write-Host "Generating config for $($matrixConfig.ConfigPath)"
+  $generatedMatrix = GenerateMatrixForConfig -ConfigPath $matrixConfig.ConfigPath -Selection $matrixConfig.Selection -DisplayNameFilter $DisplayNameFilter -Filters $Filters -Replace $Replace
+
+  # also note here that we should be using a filtered set of package properties
+  # (remember we figured out which matrix config is associated with each package)
+  # $batches = Split-ArrayIntoBatches -InputArray $packageProperties -BatchSize $BATCHSIZE
+  $batches = @()
+  $everything = ($packageProperties | ForEach-Object { $_.ArtifactName })
+  $batches += ,$everything
+
+  foreach($batch in $batches) {
+    $ModifiedMatrix = @()
+    # to understand this iteration, one must understand that the matrix is a list of hashtables, each with a couple keys:
+    # [
+    #  { "name": "jobname", "parameters": { matrixSetting1: matrixValue1, ...} },
+    # ]
+    if($batches.Length -gt 1) {
+      throw "This script is not prepared to handle more than one batch. We will need to duplicate the input objects."
+    }
+    else {
+      foreach($config in $generatedMatrix) {
+          $namesForBatch = $batches[0] -join ","
+          # we just need to iterate across them, grab the parameters hashtable, and add the new key
+          # if there is more than one batch, we will need to add a suffix including the batch name to the job name
+          $config["parameters"]["$PRMatrixSetting"] = $namesForBatch
+          $OverallResult += $config
+      }
+    }
+  }
+}
+
+$serialized = SerializePipelineMatrix $OverallResult
+
+Write-Output $serialized.pretty
+
+if ($CI) {
+    Write-Output "##vso[task.setVariable variable=matrix;isOutput=true]$($serialized.compressed)"
+}

eng/pipelines/templates/jobs/ci.yml

@@ -75,20 +75,19 @@ jobs:
           MatrixFilters: ${{ parameters.MatrixFilters }}
           MatrixReplace: ${{ parameters.MatrixReplace }}
           SparseCheckoutPaths: [ "sdk/", ".vscode"]
-          PreGenerationSteps:
-            - template: /eng/common/pipelines/templates/steps/save-package-properties.yml
-              parameters:
-                ServiceDirectory: ${{parameters.ServiceDirectory}}
+          ${{ if eq(parameters.ServiceDirectory, 'auto') }}:
+            EnablePRGeneration: true
+
+            PRMatrix:
+              Name: pr_test_base
+              Path: eng/pipelines/templates/stages/platform-matrix.json
+              Setting: ArtifactPackageNames
+
+            PreGenerationSteps:
+              - template: /eng/common/pipelines/templates/steps/save-package-properties.yml
+                parameters:
+                  ServiceDirectory: ${{parameters.ServiceDirectory}}
 
-            - task: Powershell@2
-              inputs:
-                pwsh: true
-                filePath: eng/scripts/distribute-packages-to-matrix.ps1
-                arguments: >-
-                  -PackageInfoFolder "$(Build.ArtifactStagingDirectory)/PackageInfo"
-                  -PlatformMatrix "${{ parameters.MatrixConfigs[0].Path }}"
-              displayName: 'Distribute Packages to Matrix'
-              condition: and(eq(variables['Build.Reason'], 'PullRequest'), eq('${{ parameters.ServiceDirectory }}','auto'))
           CloudConfig:
             Cloud: Public
           AdditionalParameters: