Az eng add matrix for data

eng/common/scripts/job-matrix/job-matrix-functions.ps1

@@ -94,7 +94,8 @@ function GenerateMatrix(
     [Array]$replace = @(),
     [Array]$nonSparseParameters = @()
 ) {
-    $matrixParameters, $importedMatrix, $combinedDisplayNameLookup = ProcessImport $config.matrixParameters $selectFromMatrixType $config.displayNamesLookup
+    $matrixParameters, $importedMatrix, $combinedDisplayNameLookup = `
+            ProcessImport $config.matrixParameters $selectFromMatrixType $nonSparseParameters $config.displayNamesLookup
     if ($selectFromMatrixType -eq "sparse") {
         $matrix = GenerateSparseMatrix $matrixParameters $config.displayNamesLookup $nonSparseParameters
     } elseif ($selectFromMatrixType -eq "all") {
@@ -340,7 +341,7 @@ function ProcessReplace
     return $replaceMatrix
 }
 
-function ProcessImport([MatrixParameter[]]$matrix, [String]$selection, [Hashtable]$displayNamesLookup)
+function ProcessImport([MatrixParameter[]]$matrix, [String]$selection, [Array]$nonSparseParameters, [Hashtable]$displayNamesLookup)
 {
     $importPath = ""
     $matrix = $matrix | ForEach-Object {
@@ -355,7 +356,10 @@ function ProcessImport([MatrixParameter[]]$matrix, [String]$selection, [Hashtabl
     }
 
     $importedMatrixConfig = GetMatrixConfigFromJson (Get-Content $importPath)
-    $importedMatrix = GenerateMatrix $importedMatrixConfig $selection
+    $importedMatrix = GenerateMatrix `
+                        -config $importedMatrixConfig `
+                        -selectFromMatrixType $selection `
+                        -nonSparseParameters $nonSparseParameters
 
     $combinedDisplayNameLookup = $importedMatrixConfig.displayNamesLookup
     foreach ($lookup in $displayNamesLookup.GetEnumerator()) {
@@ -437,7 +441,7 @@ function GenerateSparseMatrix(
     $matrix = GenerateFullMatrix $parameters $displayNamesLookup
 
     $sparseMatrix = @()
-    $indexes = GetSparseMatrixIndexes $dimensions
+    [array]$indexes = GetSparseMatrixIndexes $dimensions
     foreach ($idx in $indexes) {
         $sparseMatrix += GetNdMatrixElement $idx $matrix $dimensions
     }
@@ -469,7 +473,7 @@ function GetSparseMatrixIndexes([Array]$dimensions)
         $indexes += ,$idx
     }
 
-    return $indexes
+    return ,$indexes
 }
 
 function GenerateFullMatrix(

eng/pipelines/templates/stages/archetype-sdk-tests.yml

@@ -87,7 +87,7 @@ parameters:
   type: object
   default:
     - Name: Java_live_test_base
-      Path: eng/pipelines/templates/stages/platform-matrix.json
+      Path: eng/pipelines/templates/stages/client-matrix.json
       Selection: sparse
       GenerateVMJobs: true
 - name: AdditionalMatrixConfigs

eng/pipelines/templates/stages/client-matrix.json

@@ -1,6 +1,10 @@
 {
+  "displayNames": {
+    "true": "TestFromSource"
+  },
   "matrix": {
-    "$IMPORT": "eng/pipelines/templates/stages/platform-matrix.json"
+    "$IMPORT": "eng/pipelines/templates/stages/platform-matrix.json",
+    "TestGoals": "surefire:test"
   },
   "exclude": [
     {
@@ -14,20 +18,16 @@
         "ubuntu-18.04": { "OSVmImage": "MMSUbuntu18.04", "Pool": "azsdk-pool-mms-ubuntu-1804-general" }
       },
       "JavaTestVersion": "1.11",
-      "AZURE_TEST_HTTP_CLIENTS": "netty",
-      "TestFromSource": true,
       "TestGoals": "surefire:test",
-      "TestOptions": ""
+      "TestFromSource": true
     },
     {
       "Agent": {
         "windows-2019": { "OSVmImage": "MMS2019", "Pool": "azsdk-pool-mms-win-2019-general" }
       },
       "JavaTestVersion": "1.11",
-      "AZURE_TEST_HTTP_CLIENTS": "netty",
-      "AdditionalOptions": "-am",
-      "TestGoals": "verify",
-      "TestOptions": ""
+      "TestOptions": "-am",
+      "TestGoals": "verify"
     }
   ]
-}
+}

eng/pipelines/templates/stages/cosmos-matrix.json

@@ -0,0 +1,31 @@
+{
+  "matrix": {
+    "$IMPORT": "eng/pipelines/templates/stages/platform-matrix.json",
+    "options": {
+      " ": {
+          "TestGoals": "surefire:test",
+          "TestOptions": "-Punit"
+      }
+    }
+  },
+  "exclude": [
+    {
+      "Pool": "azsdk-pool-mms-win-2019-general",
+      "JavaTestVersion": "1.11"
+    }
+  ],
+  "include": [
+    {
+      "Agent": {
+        "windows-2019": { "OSVmImage": "MMS2019", "Pool": "azsdk-pool-mms-win-2019-general" }
+      },
+      "JavaTestVersion": "1.11",
+      "options": {
+        "verify": {
+          "TestGoals": "verify",
+          "TestOptions": "-Punit -am"
+        }
+      }
+    }
+  ]
+}

eng/pipelines/templates/stages/cosmos-sdk-client.yml

@@ -38,15 +38,10 @@ stages:
           - template: ../steps/install-reporting-tools.yml
         MatrixConfigs:
           - Name: Cosmos_ci_test
-            Path: eng/pipelines/templates/stages/platform-matrix.json
+            Path: eng/pipelines/templates/stages/cosmos-matrix.json
             Selection: sparse
             NonSparseParameters: Agent
             GenerateVMJobs: true
-        MatrixFilters:
-          - TestFromSource=^$
-        MatrixReplace:
-          - TestOptions=.*/-Punit
-          - AZURE_TEST.*=.*/
 
   - stage: TestEmulator
     displayName: Test Emulator

eng/pipelines/templates/stages/data-matrix.json

@@ -4,4 +4,4 @@
     "TestGoals": "verify",
     "TestOptions": "-am"
   }
-}
+}

eng/pipelines/templates/stages/live-matrix.json

@@ -0,0 +1,41 @@
+{
+  "displayNames": {
+    "true": "TestFromSource"
+  },
+  "matrix": {
+    "$IMPORT": "eng/pipelines/templates/stages/platform-matrix.json",
+    "options": {
+      " ": {
+        "TestGoals": "verify",
+        "TestOptions": "-am"
+      }
+    }
+  },
+  "exclude": [
+    {
+      "Pool": "azsdk-pool-mms-win-2019-general",
+      "JavaTestVersion": "1.11"
+    }
+  ],
+  "include": [
+    {
+      "Agent": {
+        "ubuntu-18.04": { "OSVmImage": "MMSUbuntu18.04", "Pool": "azsdk-pool-mms-ubuntu-1804-general" }
+      },
+      "JavaTestVersion": "1.11",
+      "AZURE_TEST_HTTP_CLIENTS": "netty",
+      "TestFromSource": true,
+      "TestGoals": "verify"
+    },
+    {
+      "Agent": {
+        "windows-2019": { "OSVmImage": "MMS2019", "Pool": "azsdk-pool-mms-win-2019-general" }
+      },
+      "JavaTestVersion": "1.11",
+      "AZURE_TEST_HTTP_CLIENTS": "netty",
+      "AdditionalOptions": "-am",
+      "TestGoals": "verify",
+      "TestOptions": ""
+    }
+  ]
+}

eng/pipelines/templates/stages/platform-matrix.json

@@ -1,16 +1,11 @@
 {
-  "displayNames": {
-    "true": "TestFromSource"
-  },
   "matrix": {
     "Agent": {
       "ubuntu-18.04": { "OSVmImage": "MMSUbuntu18.04", "Pool": "azsdk-pool-mms-ubuntu-1804-general" },
       "windows-2019": { "OSVmImage": "MMS2019", "Pool": "azsdk-pool-mms-win-2019-general" },
       "macOS-10.15": { "OSVmImage": "macOS-10.15", "Pool": "Azure Pipelines" }
     },
     "JavaTestVersion": [ "1.8", "1.11" ],
-    "AZURE_TEST_HTTP_CLIENTS": [ "okhttp", "netty" ],
-    "TestGoals": "surefire:test",
-    "TestOptions": ""
+    "AZURE_TEST_HTTP_CLIENTS": [ "okhttp", "netty" ]
   }
 }