Update stress test template to handle Scenario name and group deploym…

…ents (#1866) * [stress test] Support scenario name injection into job/volume properties * Deploy for stress tests at resource group scope * Bump stress-test-addons helm version * Add built-in Values.scenario list handling for stress-test-addons * [stress test] Fix helm repo indexing to merge with previous index * [stress test] Automatically compile bicep files in stress test deploy script
Azure · Aug 11, 2021 · fde2de9 · fde2de9
1 parent 4d4108e
commit fde2de9
Show file tree

Hide file tree

Showing 17 changed files with 148 additions and 86 deletions.
diff --git a/eng/common/scripts/stress-testing/deploy-stress-tests.ps1 b/eng/common/scripts/stress-testing/deploy-stress-tests.ps1
@@ -116,10 +116,15 @@ function DeployStressPackage(
         exit 1
     }
 
-    if ($pushImages) {
-        Run helm dependency update $pkg.Directory
-        if ($LASTEXITCODE) { return $LASTEXITCODE }
+    Run helm dependency update $pkg.Directory
+    if ($LASTEXITCODE) { return }
+
+    if (Test-Path "$($pkg.Directory)/test-resources.bicep") {
+        Run az bicep build -f "$($pkg.Directory)/test-resources.bicep"
+        if ($LASTEXITCODE) { return }
+    }
 
+    if ($pushImages) {
         $dockerFiles = Get-ChildItem "$($pkg.Directory)/Dockerfile*"
         foreach ($dockerFile in $dockerFiles) {
             # Infer docker image name from parent directory name, if file is named `Dockerfile`
@@ -131,13 +136,13 @@ function DeployStressPackage(
             $imageTag = "${registry}.azurecr.io/$($repository.ToLower())/$($imageName):$deployId"
             Write-Host "Building and pushing stress test docker image '$imageTag'"
             Run docker build -t $imageTag -f $dockerFile.FullName $dockerFile.DirectoryName
-            if ($LASTEXITCODE) { return $LASTEXITCODE }
+            if ($LASTEXITCODE) { return }
             Run docker push $imageTag
             if ($LASTEXITCODE) {
                 if ($PSCmdlet.ParameterSetName -ne 'DoLogin') {
                     Write-Warning "If docker push is failing due to authentication issues, try calling this script with '-Login'"
                 }
-                return $LASTEXITCODE
+                return
             }
         }
     }
@@ -157,7 +162,7 @@ function DeployStressPackage(
         # can be the result of cancelled `upgrade` operations (e.g. ctrl-c).
         # See https://github.com/helm/helm/issues/4558
         Write-Warning "The issue may be fixable by first running 'helm rollback -n $($pkg.Namespace) $($pkg.ReleaseName)'"
-        return $LASTEXITCODE
+        return
     }
 
     # Helm 3 stores release information in kubernetes secrets. The only way to add extra labels around

diff --git a/tools/stress-cluster/chaos/README.md b/tools/stress-cluster/chaos/README.md
@@ -14,6 +14,7 @@ The chaos environment is an AKS cluster (Azure Kubernetes Service) with several
      * [Helm Chart Dependencies](#helm-chart-dependencies)
      * [Job Manifest](#job-manifest)
      * [Chaos Manifest](#chaos-manifest)
+     * [Scenarios and values.yaml](#scenarios-and-valuesyaml)
   * [Deploying a Stress Test](#deploying-a-stress-test)
   * [Configuring faults](#configuring-faults)
      * [Faults via Dashboard](#faults-via-dashboard)
@@ -154,7 +155,7 @@ The basic layout for a stress test is the following (see `examples/stress_deploy
 
     # Optional files/directories
 
-    values.yaml                  # Any default helm template values for this chart
+    values.yaml                  # Any default helm template values for this chart, e.g. a `scenarios` list
     <misc scripts/configs>       # Any language specific files for building/running/deploying the test
     <source directories>         # Directories containing code for stress tests
     <bicep modules>              # Any additional bicep module files/directories referenced by test-resources.bicep
@@ -306,6 +307,39 @@ For more detailed examples, see:
 - `./examples/network_stress_example/chart/templates/network_loss.yaml` for an example network loss manifest within a helm chart
 - The [Faults via Dashboard section](#faults-via-dashboard) for generating the configs from the UI
 
+### Scenarios and values.yaml
+
+For cases where the same job config must be run for multiple instances, a `scenarios` list can be specified in a file called `values.yaml`.
+
+For example, given a stress test package with multiple tests represented as separate files:
+
+```
+values.yaml
+templates/
+app/
+  scenarioLongRunning.js
+  scenarioPeekMessages.js
+  scenarioBatchReceive.js
+...
+```
+
+The pod command in the job manifest could be configured to run a variable file name:
+
+```
+command: ["node", "app/{{ .Scenario }}.js"]
+```
+
+In order to accomplish this, add the scenarios list to `values.yaml`
+
+```
+scenarios:
+  - scenarioLongRunning
+  - scenarioPeekMessages
+  - scenarioBatchReceive
+```
+
+The underlying `stress-test-addons` helm library will handle a scenarios list automatically, and deploy multiple instances of the stress test job, one for each scenario.
+
 ## Deploying a Stress Test
 
 To build and deploy the stress test, first log in to access the cluster resources if not already set up:

diff --git a/tools/stress-cluster/chaos/examples/network-stress-example/Chart.lock b/tools/stress-cluster/chaos/examples/network-stress-example/Chart.lock
@@ -1,6 +1,6 @@
 dependencies:
 - name: stress-test-addons
   repository: https://stresstestcharts.blob.core.windows.net/helm/
-  version: 0.1.2
-digest: sha256:b38f530a7f691eb3f11d48809ba7f86ea9d7b226c3ecb311d1ae47fbb0585466
-generated: "2021-07-28T22:23:31.0555163-04:00"
+  version: 0.1.4
+digest: sha256:c463bfc41e22ce2d3e6ae19279297b3727fa1d76866fbcbd93dd4744c7b768bd
+generated: "2021-08-10T13:03:46.0104659-04:00"
diff --git a/tools/stress-cluster/chaos/examples/network-stress-example/Chart.yaml b/tools/stress-cluster/chaos/examples/network-stress-example/Chart.yaml
@@ -7,9 +7,9 @@ annotations:
   stressTest: 'true'
   namespace: 'examples'
   example: 'true'
-  test: 'true'
+  hasChaos: 'true'
 
 dependencies:
 - name: stress-test-addons
-  version: 0.1.2
+  version: 0.1.4
   repository: https://stresstestcharts.blob.core.windows.net/helm/
diff --git a/tools/stress-cluster/chaos/examples/stress-deployment-example/Chart.lock b/tools/stress-cluster/chaos/examples/stress-deployment-example/Chart.lock
@@ -1,6 +1,6 @@
 dependencies:
 - name: stress-test-addons
   repository: https://stresstestcharts.blob.core.windows.net/helm/
-  version: 0.1.2
-digest: sha256:b38f530a7f691eb3f11d48809ba7f86ea9d7b226c3ecb311d1ae47fbb0585466
-generated: "2021-07-06T18:42:04.6255273-04:00"
+  version: 0.1.4
+digest: sha256:c463bfc41e22ce2d3e6ae19279297b3727fa1d76866fbcbd93dd4744c7b768bd
+generated: "2021-08-10T13:10:34.5201711-04:00"
diff --git a/tools/stress-cluster/chaos/examples/stress-deployment-example/Chart.yaml b/tools/stress-cluster/chaos/examples/stress-deployment-example/Chart.yaml
@@ -7,8 +7,9 @@ annotations:
   stressTest: 'true'
   namespace: 'examples'
   example: 'true'
+  hasDeploy: 'true'
 
 dependencies:
 - name: stress-test-addons
-  version: 0.1.2
+  version: 0.1.4
   repository: https://stresstestcharts.blob.core.windows.net/helm/
diff --git a/tools/stress-cluster/chaos/examples/stress-deployment-example/parameters.json b/tools/stress-cluster/chaos/examples/stress-deployment-example/parameters.json
@@ -1,12 +1,5 @@
 {
   "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentParameters.json#",
   "contentVersion": "1.0.0.0",
-  "parameters": {
-    "groupName": {
-      "value": "stress-test-deploy-example"
-    },
-    "location": {
-      "value": "westus2"
-    }
-  }
+  "parameters": { }
 }
diff --git a/tools/stress-cluster/chaos/examples/stress-deployment-example/test-resources.bicep b/tools/stress-cluster/chaos/examples/stress-deployment-example/test-resources.bicep
@@ -1,15 +1,13 @@
-targetScope = 'subscription'
+// Dummy parameter to handle defaults the script passes in
+param testApplicationOid string = ''
 
-param groupName string
-param location string
-param now string = utcNow('u')
-
-resource group 'Microsoft.Resources/resourceGroups@2020-10-01' = {
-    name: 'rg-${groupName}-${uniqueString(now)}'
-    location: location
-    tags: {
-        DeleteAfter: dateTimeAdd(now, 'PT8H')
-    }
+resource config 'Microsoft.AppConfiguration/configurationStores@2020-07-01-preview' = {
+  name: 'config-${resourceGroup().name}'
+  location: resourceGroup().location
+  sku: {
+    name: 'Standard'
+  }
 }
 
-output RESOURCE_GROUP string = group.name
+output RESOURCE_GROUP string = resourceGroup().name
+output AZURE_CLIENT_OID string = testApplicationOid
diff --git a/tools/stress-cluster/chaos/examples/stress-deployment-example/test-resources.json b/tools/stress-cluster/chaos/examples/stress-deployment-example/test-resources.json
@@ -1,41 +1,39 @@
 {
-  "$schema": "https://schema.management.azure.com/schemas/2018-05-01/subscriptionDeploymentTemplate.json#",
+  "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#",
   "contentVersion": "1.0.0.0",
   "metadata": {
     "_generator": {
       "name": "bicep",
       "version": "0.4.63.48766",
-      "templateHash": "658186316551815960"
+      "templateHash": "13987799099034517242"
     }
   },
   "parameters": {
-    "groupName": {
-      "type": "string"
-    },
-    "location": {
-      "type": "string"
-    },
-    "now": {
+    "testApplicationOid": {
       "type": "string",
-      "defaultValue": "[utcNow('u')]"
+      "defaultValue": ""
     }
   },
   "functions": [],
   "resources": [
     {
-      "type": "Microsoft.Resources/resourceGroups",
-      "apiVersion": "2020-10-01",
-      "name": "[format('rg-{0}-{1}', parameters('groupName'), uniqueString(parameters('now')))]",
-      "location": "[parameters('location')]",
-      "tags": {
-        "DeleteAfter": "[dateTimeAdd(parameters('now'), 'PT8H')]"
+      "type": "Microsoft.AppConfiguration/configurationStores",
+      "apiVersion": "2020-07-01-preview",
+      "name": "[format('config-{0}', resourceGroup().name)]",
+      "location": "[resourceGroup().location]",
+      "sku": {
+        "name": "Standard"
       }
     }
   ],
   "outputs": {
     "RESOURCE_GROUP": {
       "type": "string",
-      "value": "[format('rg-{0}-{1}', parameters('groupName'), uniqueString(parameters('now')))]"
+      "value": "[resourceGroup().name]"
+    },
+    "AZURE_CLIENT_OID": {
+      "type": "string",
+      "value": "[parameters('testApplicationOid')]"
     }
   }
 }
diff --git a/tools/stress-cluster/cluster/kubernetes/stress-test-addons/Chart.yaml b/tools/stress-cluster/cluster/kubernetes/stress-test-addons/Chart.yaml
@@ -2,5 +2,5 @@ apiVersion: v2
 name: stress-test-addons
 description: Baseline resources and templates for stress testing clusters
 
-version: 0.1.2
+version: 0.1.4
 appVersion: v0.1
diff --git a/tools/stress-cluster/cluster/kubernetes/stress-test-addons/deploy.sh b/tools/stress-cluster/cluster/kubernetes/stress-test-addons/deploy.sh
@@ -1,11 +1,13 @@
 export AZURE_STORAGE_ACCOUNT=stresstestcharts
-# AZURE_STORAGE_KEY must be exported too, run the below command to get the key:
-# az storage account keys list --account-name stresstestcharts -o json --query '[0].value'
+export AZURE_STORAGE_KEY=$(az storage account keys list --account-name $AZURE_STORAGE_ACCOUNT -o tsv --query '[0].value')
 
 rm *.tgz
 
 helm package  .
-helm repo index --url https://stresstestcharts.blob.core.windows.net/helm/ .
+helm repo index --url https://stresstestcharts.blob.core.windows.net/helm/ --merge index.yaml .
+
+# The index.yaml in git should be synced with the index.yaml already in blob storage
+# az storage blob download -c helm -n index.yaml -f index.yaml
 
 az storage blob upload --container-name helm --file index.yaml --name index.yaml
 az storage blob upload --container-name helm --file *.tgz --name *.tgz

diff --git a/tools/stress-cluster/cluster/kubernetes/stress-test-addons/index.yaml b/tools/stress-cluster/cluster/kubernetes/stress-test-addons/index.yaml
@@ -3,11 +3,29 @@ entries:
   stress-test-addons:
   - apiVersion: v2
     appVersion: v0.1
-    created: "2021-07-28T22:24:58.3999792-04:00"
+    created: "2021-08-10T12:25:32.7932615-04:00"
+    description: Baseline resources and templates for stress testing clusters
+    digest: 40c564cf2fdc37d4cccdd8529b4df0a4f4acabb4d81f9e86d738e7a88cd26081
+    name: stress-test-addons
+    urls:
+    - https://stresstestcharts.blob.core.windows.net/helm/stress-test-addons-0.1.4.tgz
+    version: 0.1.4
+  - apiVersion: v2
+    appVersion: v0.1
+    created: "2021-07-30T07:38:55.4331073-04:00"
+    description: Baseline resources and templates for stress testing clusters
+    digest: 8adc1ae609fda0d0c4c54937876363ac97979485ac2802884409e67e207327b0
+    name: stress-test-addons
+    urls:
+    - https://stresstestcharts.blob.core.windows.net/helm/stress-test-addons-0.1.3.tgz
+    version: 0.1.3
+  - apiVersion: v2
+    appVersion: v0.1
+    created: "2021-07-28T22:21:27.6697515-04:00"
     description: Baseline resources and templates for stress testing clusters
     digest: cce228906811f1b39db7bcd031c94192751caa0a94f7c6035e21e3de8fc5858d
     name: stress-test-addons
     urls:
     - https://stresstestcharts.blob.core.windows.net/helm/stress-test-addons-0.1.2.tgz
     version: 0.1.2
-generated: "2021-07-28T22:24:58.3988616-04:00"
+generated: "2021-08-10T12:25:32.7923949-04:00"
diff --git a/tools/stress-cluster/cluster/kubernetes/stress-test-addons/templates/_container_env.tpl b/tools/stress-cluster/cluster/kubernetes/stress-test-addons/templates/_container_env.tpl
@@ -3,6 +3,6 @@ env:
   - name: ENV_FILE
     value: /mnt/outputs/.env
 volumeMounts:
-  - name: test-env-{{ .Release.Name }}
+  - name: test-env-{{ lower .Scenario }}-{{ .Release.Name }}-{{ .Release.Revision }}
     mountPath: /mnt/outputs
 {{- end -}}
diff --git a/tools/stress-cluster/cluster/kubernetes/stress-test-addons/templates/_env_volumes.tpl b/tools/stress-cluster/cluster/kubernetes/stress-test-addons/templates/_env_volumes.tpl
@@ -1,5 +1,5 @@
 {{ define "stress-test-addons.env-volumes" }}
-- name: test-env-{{ .Release.Name }}
+- name: test-env-{{ lower .Scenario }}-{{ .Release.Name }}-{{ .Release.Revision }}
   emptyDir: {}
 - name: cluster-secrets-{{ .Release.Name }}
   csi:

diff --git a/tools/stress-cluster/cluster/kubernetes/stress-test-addons/templates/_init_deploy.tpl b/tools/stress-cluster/cluster/kubernetes/stress-test-addons/templates/_init_deploy.tpl
@@ -7,13 +7,19 @@
       source /mnt/secrets/static/* &&
       az login --service-principal -u $AZURE_CLIENT_ID -p $AZURE_CLIENT_SECRET --tenant $AZURE_TENANT_ID &&
       az account set -s $AZURE_SUBSCRIPTION_ID &&
-      az deployment sub create \
-          -n {{ .Release.Name }} \
-          -l westus2 \
+      groupName='{{ lower .Scenario }}-{{ .Release.Name }}-{{ .Release.Revision }}'
+      az group create -l westus2 -g $groupName &&
+      group=$(az group show -g $groupName -o tsv --query "id") &&
+      az tag create --resource-id $group --tags DeleteAfter="$(date -d '+192:00:00' -Iseconds -u)" &&
+      az deployment group create \
+          -g $groupName \
+          -n $groupName \
           -f /mnt/testresources/test-resources.json \
-          --parameters /mnt/testresources/parameters.json &&
-      az deployment sub show \
-          -n {{ .Release.Name }} \
+          --parameters /mnt/testresources/parameters.json \
+          --parameters testApplicationOid=$AZURE_CLIENT_OID &&
+      az deployment group show \
+          -g $groupName \
+          -n $groupName \
           -o json \
           --query properties.outputs \
           | jq -r 'keys[] as $k | "\($k | ascii_upcase)=\(.[$k].value)"' >> /mnt/outputs/.env
@@ -23,7 +29,7 @@
   volumeMounts:
     - name: "{{ .Release.Name }}-test-resources"
       mountPath: /mnt/testresources
-    - name: "test-env-{{ .Release.Name }}"
+    - name: test-env-{{ lower .Scenario }}-{{ .Release.Name }}-{{ .Release.Revision }}
       mountPath: /mnt/outputs
     - name: "static-secrets-{{ .Release.Name }}"
       mountPath: "/mnt/secrets/static"

diff --git a/tools/stress-cluster/cluster/kubernetes/stress-test-addons/templates/_init_env.tpl b/tools/stress-cluster/cluster/kubernetes/stress-test-addons/templates/_init_env.tpl
@@ -9,7 +9,7 @@
     - name: ENV_FILE
       value: /mnt/outputs/.env
   volumeMounts:
-    - name: test-env-{{ .Release.Name }}
+    - name: test-env-{{ lower .Scenario }}-{{ .Release.Name }}-{{ .Release.Revision }}
       mountPath: /mnt/outputs
     - name: static-secrets-{{ .Release.Name }}
       mountPath: "/mnt/secrets/static"