Sync eng/common directory with azure-sdk-tools for PR 1521 (#17597)

* Fix issue where 2d index arrays with length 1 get flattened by powershell * Pass NonSparseParameters to imported matrix generation Co-authored-by: Ben Broderick Phillips <bebroder@microsoft.com>
Azure · Mar 26, 2021 · 7284d8d · 7284d8d
1 parent 68b8d77
commit 7284d8d
Show file tree

Hide file tree

Showing 3 changed files with 86 additions and 5 deletions.
diff --git a/eng/common/scripts/job-matrix/README.md b/eng/common/scripts/job-matrix/README.md
@@ -171,6 +171,8 @@ Importing can be useful, for example, in cases where there is a shared base matr
 once for each instance of a language version. Importing does not support overriding duplicate parameters. To achieve
 this, use the [Replace](#replace-values) argument instead.
 
+The `Selection` and `NonSparseParameters` parameters are respected when generating an imported matrix.
+
 The processing order is as follows:
 
 Given a matrix and import matrix like below:
@@ -494,6 +496,8 @@ Given a matrix like below with `JavaTestVersion` marked as a non-sparse paramete
 A matrix with 6 entries will be generated: A sparse matrix of Agent, AZURE_TEST_HTTP_CLIENTS and ArmTemplateParameters
 (3 total entries) will be multipled by the two `JavaTestVersion` parameters `1.8` and `1.11`.
 
+NOTE: NonSparseParameters are also applied when generating an imported matrix.
+
 #### Under the hood
 
 The script generates an N-dimensional matrix with dimensions equal to the parameter array lengths. For example,

diff --git a/eng/common/scripts/job-matrix/job-matrix-functions.ps1 b/eng/common/scripts/job-matrix/job-matrix-functions.ps1
@@ -94,7 +94,8 @@ function GenerateMatrix(
     [Array]$replace = @(),
     [Array]$nonSparseParameters = @()
 ) {
-    $matrixParameters, $importedMatrix, $combinedDisplayNameLookup = ProcessImport $config.matrixParameters $selectFromMatrixType $config.displayNamesLookup
+    $matrixParameters, $importedMatrix, $combinedDisplayNameLookup = `
+            ProcessImport $config.matrixParameters $selectFromMatrixType $nonSparseParameters $config.displayNamesLookup
     if ($selectFromMatrixType -eq "sparse") {
         $matrix = GenerateSparseMatrix $matrixParameters $config.displayNamesLookup $nonSparseParameters
     } elseif ($selectFromMatrixType -eq "all") {
@@ -340,7 +341,7 @@ function ProcessReplace
     return $replaceMatrix
 }
 
-function ProcessImport([MatrixParameter[]]$matrix, [String]$selection, [Hashtable]$displayNamesLookup)
+function ProcessImport([MatrixParameter[]]$matrix, [String]$selection, [Array]$nonSparseParameters, [Hashtable]$displayNamesLookup)
 {
     $importPath = ""
     $matrix = $matrix | ForEach-Object {
@@ -355,7 +356,10 @@ function ProcessImport([MatrixParameter[]]$matrix, [String]$selection, [Hashtabl
     }
 
     $importedMatrixConfig = GetMatrixConfigFromJson (Get-Content $importPath)
-    $importedMatrix = GenerateMatrix $importedMatrixConfig $selection
+    $importedMatrix = GenerateMatrix `
+                        -config $importedMatrixConfig `
+                        -selectFromMatrixType $selection `
+                        -nonSparseParameters $nonSparseParameters
 
     $combinedDisplayNameLookup = $importedMatrixConfig.displayNamesLookup
     foreach ($lookup in $displayNamesLookup.GetEnumerator()) {
@@ -437,7 +441,7 @@ function GenerateSparseMatrix(
     $matrix = GenerateFullMatrix $parameters $displayNamesLookup
 
     $sparseMatrix = @()
-    $indexes = GetSparseMatrixIndexes $dimensions
+    [array]$indexes = GetSparseMatrixIndexes $dimensions
     foreach ($idx in $indexes) {
         $sparseMatrix += GetNdMatrixElement $idx $matrix $dimensions
     }
@@ -469,7 +473,7 @@ function GetSparseMatrixIndexes([Array]$dimensions)
         $indexes += ,$idx
     }
 
-    return $indexes
+    return ,$indexes
 }
 
 function GenerateFullMatrix(

diff --git a/eng/common/scripts/job-matrix/tests/job-matrix-functions.modification.tests.ps1 b/eng/common/scripts/job-matrix/tests/job-matrix-functions.modification.tests.ps1
@@ -83,6 +83,46 @@ Describe "Platform Matrix nonSparse" -Tag "nonsparse" {
         $matrix = GenerateMatrix $config "sparse" -nonSparseParameters "testField3","testField4"
         $matrix.Length | Should -Be 8
     }
+
+    It "Should apply nonSparseParameters to an imported matrix" {
+        $matrixJson = @'
+{
+    "matrix": {
+        "$IMPORT": "./test-import-matrix.json",
+        "TestField1": "test1"
+    },
+    "exclude": [ { "Baz": "importedBaz" } ]
+}
+'@
+
+        $expectedMatrix = @'
+[
+  {
+    "parameters": { "TestField1": "test1", "Foo": "foo1", "Bar": "bar1" },
+    "name": "test1_foo1_bar1"
+  },
+  {
+    "parameters": { "TestField1": "test1", "Foo": "foo1", "Bar": "bar2" },
+    "name": "test1_foo1_bar2"
+  },
+  {
+    "parameters": { "TestField1": "test1", "Foo": "foo2", "Bar": "bar1" },
+    "name": "test1_foo2_bar1"
+  },
+  {
+    "parameters": { "TestField1": "test1", "Foo": "foo2", "Bar": "bar2" },
+    "name": "test1_foo2_bar2"
+  }
+]
+'@
+
+        $importConfig = GetMatrixConfigFromJson $matrixJson
+        $matrix = GenerateMatrix $importConfig "sparse" -nonSparseParameters "Foo"
+        $expected = $expectedMatrix | ConvertFrom-Json -AsHashtable
+
+        $matrix.Length | Should -Be 4
+        CompareMatrices $matrix $expected
+    }
 }
 
 Describe "Platform Matrix Import" -Tag "import" {
@@ -129,6 +169,39 @@ Describe "Platform Matrix Import" -Tag "import" {
         CompareMatrices $matrix $expected
     }
 
+    It "Should import a matrix and combine with length=1 vectors" {
+        $matrixJson = @'
+{
+    "matrix": {
+        "$IMPORT": "./test-import-matrix.json",
+        "TestField1": "test1",
+        "TestField2": "test2"
+    },
+    "exclude": [ { "Baz": "importedBaz" } ]
+}
+'@
+
+        $expectedMatrix = @'
+[
+  {
+    "parameters": { "TestField1": "test1", "TestField2": "test2", "Foo": "foo1", "Bar": "bar1" },
+    "name": "test1_test2_foo1_bar1"
+  },
+  {
+    "parameters": { "TestField1": "test1", "TestField2": "test2", "Foo": "foo2", "Bar": "bar2" },
+    "name": "test1_test2_foo2_bar2"
+  }
+]
+'@
+
+        $importConfig = GetMatrixConfigFromJson $matrixJson
+        $matrix = GenerateMatrix $importConfig "sparse"
+        $expected = $expectedMatrix | ConvertFrom-Json -AsHashtable
+
+        $matrix.Length | Should -Be 2
+        CompareMatrices $matrix $expected
+    }
+
     It "Should generate a matrix with nonSparseParameters and an imported sparse matrix" {
         $matrixJson = @'
 {