From 7284d8df01508933a75ac3b2341c0b0f96989353 Mon Sep 17 00:00:00 2001 From: Azure SDK Bot <53356347+azure-sdk@users.noreply.github.com> Date: Fri, 26 Mar 2021 12:52:39 -0700 Subject: [PATCH] Sync eng/common directory with azure-sdk-tools for PR 1521 (#17597) * Fix issue where 2d index arrays with length 1 get flattened by powershell * Pass NonSparseParameters to imported matrix generation Co-authored-by: Ben Broderick Phillips --- eng/common/scripts/job-matrix/README.md | 4 + .../job-matrix/job-matrix-functions.ps1 | 14 ++-- ...ob-matrix-functions.modification.tests.ps1 | 73 +++++++++++++++++++ 3 files changed, 86 insertions(+), 5 deletions(-) diff --git a/eng/common/scripts/job-matrix/README.md b/eng/common/scripts/job-matrix/README.md index 13bf6cf43990..e4d0464a941f 100644 --- a/eng/common/scripts/job-matrix/README.md +++ b/eng/common/scripts/job-matrix/README.md @@ -171,6 +171,8 @@ Importing can be useful, for example, in cases where there is a shared base matr once for each instance of a language version. Importing does not support overriding duplicate parameters. To achieve this, use the [Replace](#replace-values) argument instead. +The `Selection` and `NonSparseParameters` parameters are respected when generating an imported matrix. + The processing order is as follows: Given a matrix and import matrix like below: @@ -494,6 +496,8 @@ Given a matrix like below with `JavaTestVersion` marked as a non-sparse paramete A matrix with 6 entries will be generated: A sparse matrix of Agent, AZURE_TEST_HTTP_CLIENTS and ArmTemplateParameters (3 total entries) will be multipled by the two `JavaTestVersion` parameters `1.8` and `1.11`. +NOTE: NonSparseParameters are also applied when generating an imported matrix. + #### Under the hood The script generates an N-dimensional matrix with dimensions equal to the parameter array lengths. For example, diff --git a/eng/common/scripts/job-matrix/job-matrix-functions.ps1 b/eng/common/scripts/job-matrix/job-matrix-functions.ps1 index adcdaf4bd963..25ff756950d3 100644 --- a/eng/common/scripts/job-matrix/job-matrix-functions.ps1 +++ b/eng/common/scripts/job-matrix/job-matrix-functions.ps1 @@ -94,7 +94,8 @@ function GenerateMatrix( [Array]$replace = @(), [Array]$nonSparseParameters = @() ) { - $matrixParameters, $importedMatrix, $combinedDisplayNameLookup = ProcessImport $config.matrixParameters $selectFromMatrixType $config.displayNamesLookup + $matrixParameters, $importedMatrix, $combinedDisplayNameLookup = ` + ProcessImport $config.matrixParameters $selectFromMatrixType $nonSparseParameters $config.displayNamesLookup if ($selectFromMatrixType -eq "sparse") { $matrix = GenerateSparseMatrix $matrixParameters $config.displayNamesLookup $nonSparseParameters } elseif ($selectFromMatrixType -eq "all") { @@ -340,7 +341,7 @@ function ProcessReplace return $replaceMatrix } -function ProcessImport([MatrixParameter[]]$matrix, [String]$selection, [Hashtable]$displayNamesLookup) +function ProcessImport([MatrixParameter[]]$matrix, [String]$selection, [Array]$nonSparseParameters, [Hashtable]$displayNamesLookup) { $importPath = "" $matrix = $matrix | ForEach-Object { @@ -355,7 +356,10 @@ function ProcessImport([MatrixParameter[]]$matrix, [String]$selection, [Hashtabl } $importedMatrixConfig = GetMatrixConfigFromJson (Get-Content $importPath) - $importedMatrix = GenerateMatrix $importedMatrixConfig $selection + $importedMatrix = GenerateMatrix ` + -config $importedMatrixConfig ` + -selectFromMatrixType $selection ` + -nonSparseParameters $nonSparseParameters $combinedDisplayNameLookup = $importedMatrixConfig.displayNamesLookup foreach ($lookup in $displayNamesLookup.GetEnumerator()) { @@ -437,7 +441,7 @@ function GenerateSparseMatrix( $matrix = GenerateFullMatrix $parameters $displayNamesLookup $sparseMatrix = @() - $indexes = GetSparseMatrixIndexes $dimensions + [array]$indexes = GetSparseMatrixIndexes $dimensions foreach ($idx in $indexes) { $sparseMatrix += GetNdMatrixElement $idx $matrix $dimensions } @@ -469,7 +473,7 @@ function GetSparseMatrixIndexes([Array]$dimensions) $indexes += ,$idx } - return $indexes + return ,$indexes } function GenerateFullMatrix( diff --git a/eng/common/scripts/job-matrix/tests/job-matrix-functions.modification.tests.ps1 b/eng/common/scripts/job-matrix/tests/job-matrix-functions.modification.tests.ps1 index 676968f9b7a6..08979caaaf54 100644 --- a/eng/common/scripts/job-matrix/tests/job-matrix-functions.modification.tests.ps1 +++ b/eng/common/scripts/job-matrix/tests/job-matrix-functions.modification.tests.ps1 @@ -83,6 +83,46 @@ Describe "Platform Matrix nonSparse" -Tag "nonsparse" { $matrix = GenerateMatrix $config "sparse" -nonSparseParameters "testField3","testField4" $matrix.Length | Should -Be 8 } + + It "Should apply nonSparseParameters to an imported matrix" { + $matrixJson = @' +{ + "matrix": { + "$IMPORT": "./test-import-matrix.json", + "TestField1": "test1" + }, + "exclude": [ { "Baz": "importedBaz" } ] +} +'@ + + $expectedMatrix = @' +[ + { + "parameters": { "TestField1": "test1", "Foo": "foo1", "Bar": "bar1" }, + "name": "test1_foo1_bar1" + }, + { + "parameters": { "TestField1": "test1", "Foo": "foo1", "Bar": "bar2" }, + "name": "test1_foo1_bar2" + }, + { + "parameters": { "TestField1": "test1", "Foo": "foo2", "Bar": "bar1" }, + "name": "test1_foo2_bar1" + }, + { + "parameters": { "TestField1": "test1", "Foo": "foo2", "Bar": "bar2" }, + "name": "test1_foo2_bar2" + } +] +'@ + + $importConfig = GetMatrixConfigFromJson $matrixJson + $matrix = GenerateMatrix $importConfig "sparse" -nonSparseParameters "Foo" + $expected = $expectedMatrix | ConvertFrom-Json -AsHashtable + + $matrix.Length | Should -Be 4 + CompareMatrices $matrix $expected + } } Describe "Platform Matrix Import" -Tag "import" { @@ -129,6 +169,39 @@ Describe "Platform Matrix Import" -Tag "import" { CompareMatrices $matrix $expected } + It "Should import a matrix and combine with length=1 vectors" { + $matrixJson = @' +{ + "matrix": { + "$IMPORT": "./test-import-matrix.json", + "TestField1": "test1", + "TestField2": "test2" + }, + "exclude": [ { "Baz": "importedBaz" } ] +} +'@ + + $expectedMatrix = @' +[ + { + "parameters": { "TestField1": "test1", "TestField2": "test2", "Foo": "foo1", "Bar": "bar1" }, + "name": "test1_test2_foo1_bar1" + }, + { + "parameters": { "TestField1": "test1", "TestField2": "test2", "Foo": "foo2", "Bar": "bar2" }, + "name": "test1_test2_foo2_bar2" + } +] +'@ + + $importConfig = GetMatrixConfigFromJson $matrixJson + $matrix = GenerateMatrix $importConfig "sparse" + $expected = $expectedMatrix | ConvertFrom-Json -AsHashtable + + $matrix.Length | Should -Be 2 + CompareMatrices $matrix $expected + } + It "Should generate a matrix with nonSparseParameters and an imported sparse matrix" { $matrixJson = @' {