Using Dynamic Matrix Config

eng/common/pipelines/templates/jobs/generate-job-matrix.yml

@@ -42,6 +42,12 @@ parameters:
 - name: PreGenerationSteps
   type: stepList
   default: []
+- name: EnablePRGeneration
+  type: boolean
+  default: false
+- name: PRMatrixSetting
+  type: string
+  default: 'ArtifactPackageNames'
 # Mappings to OS name required at template compile time by 1es pipeline templates
 - name: Pools
   type: object
@@ -84,57 +90,87 @@ jobs:
 
     - ${{ parameters.PreGenerationSteps }}
 
-    - ${{ each config in parameters.MatrixConfigs }}:
+    - ${{ if eq(parameters.EnablePRGeneration, false) }}:
+      - ${{ each config in parameters.MatrixConfigs }}:
+        - ${{ each pool in parameters.Pools }}:
+          - ${{ if eq(config.GenerateVMJobs, 'true') }}:
+            - task: Powershell@2
+              inputs:
+                pwsh: true
+                filePath: eng/common/scripts/job-matrix/Create-JobMatrix.ps1
+                arguments: >
+                  -ConfigPath ${{ config.Path }}
+                  -Selection ${{ config.Selection }}
+                  -DisplayNameFilter '$(displayNameFilter)'
+                  -Filters '${{ join(''',''', parameters.MatrixFilters) }}', 'container=^$', 'SupportedClouds=^$|${{ parameters.CloudConfig.Cloud }}', 'Pool=${{ pool.filter }}'
+                  -Replace '${{ join(''',''', parameters.MatrixReplace) }}'
+                  -NonSparseParameters '${{ join(''',''', config.NonSparseParameters) }}'
+              displayName: Create ${{ pool.name }} Matrix ${{ config.Name }}
+              name: vm_job_matrix_${{ config.Name }}_${{ pool.name }}
+          - ${{ if eq(config.GenerateContainerJobs, 'true') }}:
+            - task: Powershell@2
+              inputs:
+                pwsh: true
+                filePath: eng/common/scripts/job-matrix/Create-JobMatrix.ps1
+                arguments: >
+                  -ConfigPath ${{ config.Path }}
+                  -Selection ${{ config.Selection }}
+                  -DisplayNameFilter '$(displayNameFilter)'
+                  -Filters '${{ join(''',''', parameters.MatrixFilters) }}', 'container=^$', 'SupportedClouds=^$|${{ parameters.CloudConfig.Cloud }}', 'Pool=${{ pool.filter }}'
+                  -NonSparseParameters '${{ join(''',''', config.NonSparseParameters) }}'
+              displayName: Create ${{ pool.name }} Container Matrix ${{ config.Name }}
+              name: container_job_matrix_${{ config.Name }}_${{ pool.name }}
+
+      # This else being set also currently assumes that the $(Build.ArtifactStagingDirectory)/PackageInfo folder is populated by PreGenerationSteps.
+      # Not currently not hardcoded, so not doing the needful and populating this folder before we hit this step will result in generation errors.
+    - ${{ else }}:
       - ${{ each pool in parameters.Pools }}:
-        - ${{ if eq(config.GenerateVMJobs, 'true') }}:
-          - task: Powershell@2
-            inputs:
-              pwsh: true
-              filePath: eng/common/scripts/job-matrix/Create-JobMatrix.ps1
-              arguments: >
-                -ConfigPath ${{ config.Path }}
-                -Selection ${{ config.Selection }}
-                -DisplayNameFilter '$(displayNameFilter)'
-                -Filters '${{ join(''',''', parameters.MatrixFilters) }}', 'container=^$', 'SupportedClouds=^$|${{ parameters.CloudConfig.Cloud }}', 'Pool=${{ pool.filter }}'
-                -Replace '${{ join(''',''', parameters.MatrixReplace) }}'
-                -NonSparseParameters '${{ join(''',''', config.NonSparseParameters) }}'
-            displayName: Create ${{ pool.name }} Matrix ${{ config.Name }}
-            name: vm_job_matrix_${{ config.Name }}_${{ pool.name }}
+        - pwsh: |
+            # dump the conglomerated CI matrix
+            '${{ convertToJson(parameters.MatrixConfigs) }}' | Set-Content matrix.json
 
-        - ${{ if eq(config.GenerateContainerJobs, 'true') }}:
-          - task: Powershell@2
-            inputs:
-              pwsh: true
-              filePath: eng/common/scripts/job-matrix/Create-JobMatrix.ps1
-              arguments: >
-                -ConfigPath ${{ config.Path }}
-                -Selection ${{ config.Selection }}
-                -DisplayNameFilter '$(displayNameFilter)'
-                -Filters '${{ join(''',''', parameters.MatrixFilters) }}', 'container=^$', 'SupportedClouds=^$|${{ parameters.CloudConfig.Cloud }}', 'Pool=${{ pool.filter }}'
-                -NonSparseParameters '${{ join(''',''', config.NonSparseParameters) }}'
-            displayName: Create ${{ pool.name }} Container Matrix ${{ config.Name }}
-            name: container_job_matrix_${{ config.Name }}_${{ pool.name }}
+            ./eng/common/scripts/job-matrix/Create-PrJobMatrix.ps1 `
+              -PackagePropertiesFolder $(Build.ArtifactStagingDirectory)/PackageInfo `
+              -PRMatrixFile matrix.json `
+              -PRMatrixSetting ${{ parameters.PRMatrixSetting }} `
+              -DisplayNameFilter '$(displayNameFilter)' `
+              -Filters '${{ join(''',''', parameters.MatrixFilters) }}', 'container=^$', 'SupportedClouds=^$|${{ parameters.CloudConfig.Cloud }}', 'Pool=${{ pool.filter }}' `
+              -Replace '${{ join(''',''', parameters.MatrixReplace) }}'
+          displayName: Create ${{ pool.name }} PR Matrix
+          name: vm_job_matrix_pr_${{ pool.name }}
 
-- ${{ each config in parameters.MatrixConfigs }}:
-  - ${{ each pool in parameters.Pools }}:
-    - ${{ if eq(config.GenerateVMJobs, 'true') }}:
-      - template: ${{ parameters.JobTemplatePath }}
-        parameters:
-          UsePlatformContainer: false
-          OSName: ${{ pool.os }}
-          Matrix: dependencies.${{ parameters.GenerateJobName }}.outputs['vm_job_matrix_${{ config.Name }}_${{ pool.name }}.matrix']
-          DependsOn: ${{ parameters.GenerateJobName }}
-          CloudConfig: ${{ parameters.CloudConfig }}
-          ${{ each param in parameters.AdditionalParameters }}:
-            ${{ param.key }}: ${{ param.value }}
+- ${{ if eq(parameters.EnablePRGeneration, false) }}:
+  - ${{ each config in parameters.MatrixConfigs }}:
+    - ${{ each pool in parameters.Pools }}:
+      - ${{ if eq(config.GenerateVMJobs, 'true') }}:
+        - template: ${{ parameters.JobTemplatePath }}
+          parameters:
+            UsePlatformContainer: false
+            OSName: ${{ pool.os }}
+            Matrix: dependencies.${{ parameters.GenerateJobName }}.outputs['vm_job_matrix_${{ config.Name }}_${{ pool.name }}.matrix']
+            DependsOn: ${{ parameters.GenerateJobName }}
+            CloudConfig: ${{ parameters.CloudConfig }}
+            ${{ each param in parameters.AdditionalParameters }}:
+              ${{ param.key }}: ${{ param.value }}
 
-    - ${{ if eq(config.GenerateContainerJobs, 'true') }}:
-      - template: ${{ parameters.JobTemplatePath }}
-        parameters:
-          UsePlatformContainer: true
-          OSName: ${{ pool.os }}
-          Matrix: dependencies.${{ parameters.GenerateJobName }}.outputs['vm_job_matrix_${{ config.Name }}_${{ pool.name }}.matrix']
-          DependsOn: ${{ parameters.GenerateJobName }}
-          CloudConfig: ${{ parameters.CloudConfig }}
-          ${{ each param in parameters.AdditionalParameters }}:
-            ${{ param.key }}: ${{ param.value }}
+      - ${{ if eq(config.GenerateContainerJobs, 'true') }}:
+        - template: ${{ parameters.JobTemplatePath }}
+          parameters:
+            UsePlatformContainer: true
+            OSName: ${{ pool.os }}
+            Matrix: dependencies.${{ parameters.GenerateJobName }}.outputs['vm_job_matrix_${{ config.Name }}_${{ pool.name }}.matrix']
+            DependsOn: ${{ parameters.GenerateJobName }}
+            CloudConfig: ${{ parameters.CloudConfig }}
+            ${{ each param in parameters.AdditionalParameters }}:
+              ${{ param.key }}: ${{ param.value }}
+- ${{ else }}:
+  - ${{ each pool in parameters.Pools }}:
+    - template: ${{ parameters.JobTemplatePath }}
+      parameters:
+        UsePlatformContainer: false
+        OSName: ${{ pool.os }}
+        Matrix: dependencies.${{ parameters.GenerateJobName }}.outputs['vm_job_matrix_pr_${{ pool.name }}.matrix']
+        DependsOn: ${{ parameters.GenerateJobName }}
+        CloudConfig: ${{ parameters.CloudConfig }}
+        ${{ each param in parameters.AdditionalParameters }}:
+          ${{ param.key }}: ${{ param.value }}

eng/common/pipelines/templates/steps/save-package-properties.yml

@@ -22,7 +22,7 @@ steps:
 
   # The other public CI builds will pass a real service directory, which will not activate the PR diff logic and as such will operate
   # as before this change.
-  - ${{ if and(eq(variables['Build.Reason'], 'PullRequest'), eq(parameters.ServiceDirectory, 'auto')) }}:
+  - ${{ if eq(parameters.ServiceDirectory, 'auto') }}:
       - task: Powershell@2
         displayName: Generate PR Diff
         inputs:

eng/common/scripts/Helpers/Package-Helpers.ps1

@@ -68,7 +68,7 @@ Get-Content -Raw path/to/file.yml | CompatibleConvertFrom-Yaml
 #>
 function CompatibleConvertFrom-Yaml {
   param(
-    [Parameter(Mandatory=$true, ValueFromPipeline=$true)]
+    [Parameter(Mandatory = $true, ValueFromPipeline = $true)]
     [string]$Content
   )
 
@@ -85,10 +85,10 @@ function CompatibleConvertFrom-Yaml {
 
   # Process the content (for example, you could convert from YAML here)
   if ($yqPresent) {
-      return ($content | yq -o=json | ConvertFrom-Json -AsHashTable)
+    return ($content | yq -o=json | ConvertFrom-Json -AsHashTable)
   }
   else {
-      return ConvertFrom-Yaml $content
+    return ConvertFrom-Yaml $content
   }
 }
 
@@ -114,7 +114,7 @@ LoadFrom-Yaml -YmlFile path/to/file.yml
 #>
 function LoadFrom-Yaml {
   param(
-    [Parameter(Mandatory=$true)]
+    [Parameter(Mandatory = $true)]
     [string]$YmlFile
   )
   if (Test-Path -Path $YmlFile) {
@@ -159,21 +159,99 @@ GetValueSafelyFrom-Yaml -YamlContentAsHashtable $YmlFileContent -Keys @("extends
 #>
 function GetValueSafelyFrom-Yaml {
   param(
-    [Parameter(Mandatory=$true)]
+    [Parameter(Mandatory = $true)]
     $YamlContentAsHashtable,
-    [Parameter(Mandatory=$true)]
+    [Parameter(Mandatory = $true)]
     [string[]]$Keys
   )
   $current = $YamlContentAsHashtable
   foreach ($key in $Keys) {
-      if ($current.ContainsKey($key) -or $current[$key]) {
-        $current = $current[$key]
-      }
-      else {
-        Write-Host "The '$key' part of the path $($Keys -join "/") doesn't exist or is null."
-        return $null
-      }
+    if ($current.ContainsKey($key) -or $current[$key]) {
+      $current = $current[$key]
+    }
+    else {
+      return $null
+    }
   }
 
   return [object]$current
-}
+}
+
+function Get-ObjectKey {
+  param (
+    [Parameter(Mandatory = $true)]
+    [object]$Object
+  )
+
+  if (-not $Object) {
+    return "unset"
+  }
+
+  if ($Object -is [hashtable] -or $Object -is [System.Collections.Specialized.OrderedDictionary]) {
+    $sortedEntries = $Object.GetEnumerator() | Sort-Object Name
+    $hashString = ($sortedEntries | ForEach-Object { "$($_.Key)=$($_.Value)" }) -join ";"
+    return $hashString.GetHashCode()
+  }
+
+  elseif ($Object -is [PSCustomObject]) {
+    $sortedProperties = $Object.PSObject.Properties | Sort-Object Name
+    $propertyString = ($sortedProperties | ForEach-Object { "$($_.Name)=$($_.Value)" }) -join ";"
+    return $propertyString.GetHashCode()
+  }
+
+  elseif ($Object -is [array]) {
+    $arrayString = ($Object | ForEach-Object { Get-ObjectKey $_ }) -join ","
+    return $arrayString.GetHashCode()
+  }
+
+  else {
+    return $Object.ToString()
+  }
+}
+
+
+function Group-ByObjectKey {
+  param (
+    [Parameter(Mandatory)]
+    [array]$Items,
+
+    [Parameter(Mandatory)]
+    [string]$GroupByProperty
+  )
+
+  $groupedDictionary = @{}
+
+  foreach ($item in $Items) {
+    $key = Get-ObjectKey $item."$GroupByProperty"
+
+    if (-not $groupedDictionary.ContainsKey($key)) {
+      $groupedDictionary[$key] = @()
+    }
+
+    # Add the current item to the array for this key
+    $groupedDictionary[$key] += $item
+  }
+
+  return $groupedDictionary
+}
+
+
+function Split-ArrayIntoBatches {
+  param (
+    [Parameter(Mandatory = $true)]
+    [Object[]]$InputArray,
+
+    [Parameter(Mandatory = $true)]
+    [int]$BatchSize
+  )
+
+  $batches = @()
+
+  for ($i = 0; $i -lt $InputArray.Count; $i += $BatchSize) {
+    $batch = $InputArray[$i..[math]::Min($i + $BatchSize - 1, $InputArray.Count - 1)]
+
+    $batches += , $batch
+  }
+
+  return , $batches
+}

eng/common/scripts/Package-Properties.ps1

@@ -21,6 +21,7 @@ class PackageProps
     # additional packages required for validation of this one
     [string[]]$AdditionalValidationPackages
     [HashTable]$ArtifactDetails
+    [HashTable[]]$CIMatrixConfigs
 
     PackageProps([string]$name, [string]$version, [string]$directoryPath, [string]$serviceDirectory)
     {
@@ -84,8 +85,7 @@ class PackageProps
         $this.Group = $group
     }
 
-    hidden [HashTable]ParseYmlForArtifact([string]$ymlPath) {
-
+    hidden [PSCustomObject]ParseYmlForArtifact([string]$ymlPath) {
         $content = LoadFrom-Yaml $ymlPath
         if ($content) {
             $artifacts = GetValueSafelyFrom-Yaml $content @("extends", "parameters", "Artifacts")
@@ -95,8 +95,21 @@ class PackageProps
                 $artifactForCurrentPackage = $artifacts | Where-Object { $_["name"] -eq $this.ArtifactName -or $_["name"] -eq $this.Name }
             }
 
+            # if we found an artifact for the current package, we should count this ci file as the source of the matrix for this package
             if ($artifactForCurrentPackage) {
-                return [HashTable]$artifactForCurrentPackage
+                $result = [PSCustomObject]@{
+                    ArtifactConfig = [HashTable]$artifactForCurrentPackage
+                    MatrixConfigs = @()
+                }
+
+                # if we know this is the matrix for our file, we should now see if there is a custom matrix config for the package
+                $matrixConfigList = GetValueSafelyFrom-Yaml $content @("extends", "parameters", "MatrixConfigs")
+
+                if ($matrixConfigList) {
+                    $result.MatrixConfigs = matrixConfigList
+                }
+
+                return $result
             }
         }
         return $null
@@ -112,7 +125,10 @@ class PackageProps
             foreach($ciFile in $ciFiles) {
                 $ciArtifactResult = $this.ParseYmlForArtifact($ciFile.FullName)
                 if ($ciArtifactResult) {
-                    $this.ArtifactDetails = [Hashtable]$ciArtifactResult
+                    $this.ArtifactDetails = [Hashtable]$ciArtifactResult.ArtifactConfig
+                    $this.CIMatrixConfigs = $ciArtifactResult.MatrixConfigs
+                    # if this package appeared in this ci file, then we should
+                    # treat this CI file as the source of the Matrix for this package
                     break
                 }
             }