From adc98df9ef35eb60882296752c4945f783adee85 Mon Sep 17 00:00:00 2001
From: Nicolas Pepin-Perreault <nicolas.pepin-perreault@camunda.com>
Date: Fri, 8 Dec 2023 17:57:03 +0100
Subject: [PATCH] feat: add new job push cluster restart experiment

---
 .../job-push/cluster-restart.json             | 76 +++++++++++++++++++
 .../camunda-cloud/manifest.yml                |  4 +
 2 files changed, 80 insertions(+)
 create mode 100644 go-chaos/internal/chaos-experiments/camunda-cloud/job-push/cluster-restart.json

diff --git a/go-chaos/internal/chaos-experiments/camunda-cloud/job-push/cluster-restart.json b/go-chaos/internal/chaos-experiments/camunda-cloud/job-push/cluster-restart.json
new file mode 100644
index 000000000..a6cc40951
--- /dev/null
+++ b/go-chaos/internal/chaos-experiments/camunda-cloud/job-push/cluster-restart.json
@@ -0,0 +1,76 @@
+{
+  "version": "0.1.0",
+  "title": "Job push multiple leader change experiment",
+  "description": "Job workers with streaming enabled should be fault-tolerant. The worker should receive and complete jobs even if all brokers restart.",
+  "contributions": {
+      "performance": "high",
+      "reliability": "high",
+      "availability": "high"
+  },
+  "steady-state-hypothesis": {
+      "title": "Zeebe is alive",
+      "probes": [
+          {
+              "name": "All pods should be ready",
+              "type": "probe",
+              "tolerance": 0,
+              "provider": {
+                  "type": "process",
+                  "path": "zbchaos",
+                  "arguments": ["verify", "readiness"],
+                  "timeout": 900
+              }
+          },
+          {
+              "name": "Can deploy process model",
+              "type": "probe",
+              "tolerance": 0,
+              "provider": {
+                  "type": "process",
+                  "path": "zbchaos",
+                  "arguments": ["deploy", "process"],
+                  "timeout": 900
+              }
+          },
+          {
+            "name": "Can deploy workers",
+            "type": "probe",
+            "tolerance": 0,
+            "provider": {
+                "type": "process",
+                "path": "zbchaos",
+                "arguments": ["deploy", "worker", "--pollingDelay", "86400000"],
+                "timeout": 900
+            },
+            "pauses": {
+                "after": 5
+            }
+          },
+          {
+              "name": "Should be able to create process instances",
+              "type": "probe",
+              "tolerance": 0,
+              "provider": {
+                  "type": "process",
+                  "path": "zbchaos",
+                  "arguments": ["verify", "instance-creation", "--awaitResult"],
+                  "timeout": 900
+              }
+          }
+      ]
+  },
+  "method": [
+    {
+        "name": "Restart all brokers",
+        "type": "probe",
+        "tolerance": 0,
+        "provider": {
+            "type": "process",
+            "path": "zbchaos",
+            "arguments": ["restart", "broker", "--all"],
+            "timeout": 900
+        }
+    }
+  ],
+  "rollbacks": []
+}
diff --git a/go-chaos/internal/chaos-experiments/camunda-cloud/manifest.yml b/go-chaos/internal/chaos-experiments/camunda-cloud/manifest.yml
index 36030933a..a5d77ddda 100644
--- a/go-chaos/internal/chaos-experiments/camunda-cloud/manifest.yml
+++ b/go-chaos/internal/chaos-experiments/camunda-cloud/manifest.yml
@@ -47,6 +47,10 @@ experiments:
     clusterPlans:
       - production-s
     minVersion: 8.4
+  - path: job-push/cluster-restart.json
+    clusterPlans:
+      - production-s
+    minVersion: 8.4
   # only for testing
   - path: test/experiment.json
     clusterPlans: