Worker for read chaos experiments

go-chaos/cmd/worker.go

@@ -25,7 +25,8 @@ import (
 	worker "github.com/zeebe-io/zeebe-chaos/go-chaos/worker"
 )
 
-const jobType = "zbchaos"
+const jobTypeZbChaos = "zbchaos"
+const jobTypeReadExperiments = "readExperiments"
 
 func init() {
 	rootCmd.AddCommand(workerCommand)
@@ -54,7 +55,8 @@ func start_worker(cmd *cobra.Command, args []string) {
 	}
 
 	// Allow only one job at a time, otherwise job handling might interfere (e.g. override global vars)
-	jobWorker := client.NewJobWorker().JobType(jobType).Handler(handleZbChaosJob).MaxJobsActive(1).Open()
+	jobWorker := client.NewJobWorker().JobType(jobTypeZbChaos).Handler(handleZbChaosJob).MaxJobsActive(1).Open()
+	client.NewJobWorker().JobType(jobTypeReadExperiments).Handler(handleZbChaosJob).MaxJobsActive(1).Open()
 	jobWorker.AwaitClose()
 }
 

go-chaos/internal/chaos-experiments/camunda-cloud/development/leader-restart/experiment.json

@@ -0,0 +1,47 @@
+{
+    "version": "0.1.0",
+    "title": "Zeebe Leader restart gracefully experiment",
+    "description": "Zeebe should be fault-tolerant. Zeebe should recover after a partition leader was restarted gracefully.",
+    "contributions": {
+        "reliability": "high",
+        "availability": "high"
+    },
+    "steady-state-hypothesis": {
+        "title": "Zeebe is alive",
+        "probes": [
+            {
+                "name": "All pods should be ready",
+                "type": "probe",
+                "tolerance": 0,
+                "provider": {
+                    "type": "process",
+                    "path": "verify-readiness.sh",
+                    "timeout": 900
+                }
+            },
+            {
+                "name": "Should be able to create process instances on partition 1",
+                "type": "probe",
+                "tolerance": 0,
+                "provider": {
+                    "type": "process",
+                    "path": "verify-steady-state.sh",
+                    "arguments": "1",
+                    "timeout": 900
+                }
+            }
+        ]
+    },
+    "method": [
+        {
+            "type": "action",
+            "name": "Terminate leader of partition 1",
+            "provider": {
+                "type": "process",
+                "path": "shutdown-gracefully-partition.sh",
+                "arguments": [ "Leader", "1" ]
+            }
+        }
+    ],
+    "rollbacks": []
+}

go-chaos/internal/chaos-experiments/camunda-cloud/development/leader-terminate/experiment.json

@@ -0,0 +1,47 @@
+{
+    "version": "0.1.0",
+    "title": "Zeebe Leader restart non-graceful experiment",
+    "description": "Zeebe should be fault-tolerant. We expect that Zeebe can handle non-graceful leader restarts.",
+    "contributions": {
+        "reliability": "high",
+        "availability": "high"
+    },
+    "steady-state-hypothesis": {
+        "title": "Zeebe is alive",
+        "probes": [
+            {
+                "name": "All pods should be ready",
+                "type": "probe",
+                "tolerance": 0,
+                "provider": {
+                    "type": "process",
+                    "path": "verify-readiness.sh",
+                    "timeout": 900
+                }
+            },
+            {
+                "name": "Should be able to create process instances on partition 1",
+                "type": "probe",
+                "tolerance": 0,
+                "provider": {
+                    "type": "process",
+                    "path": "verify-steady-state.sh",
+                    "arguments": "1",
+                    "timeout": 900
+                }
+            }
+        ]
+    },
+    "method": [
+        {
+            "type": "action",
+            "name": "Terminate leader of partition 1 non-gracefully",
+            "provider": {
+                "type": "process",
+                "path": "terminate-partition.sh",
+                "arguments": [ "Leader", "1" ]
+            }
+        }
+    ],
+    "rollbacks": []
+}

go-chaos/internal/chaos-experiments/camunda-cloud/development/msg-correlation/experiment.json

@@ -0,0 +1,56 @@
+{
+    "version": "0.1.0",
+    "title": "Zeebe message correlation experiment",
+    "description": "Zeebe message correlation should work even if the leader was restarted on which the message was published.",
+    "contributions": {
+        "reliability": "high",
+        "availability": "high"
+    },
+    "steady-state-hypothesis": {
+        "title": "Zeebe is alive",
+        "probes": [
+            {
+                "name": "All pods should be ready",
+                "type": "probe",
+                "tolerance": 0,
+                "provider": {
+                    "type": "process",
+                    "path": "verify-readiness.sh",
+                    "timeout": 900
+                }
+            }
+        ]
+    },
+    "method": [
+        {
+            "type": "action",
+            "name": "Publish message to partition one",
+            "tolerance": 0,
+            "provider": {
+                "type": "process",
+                "path": "publish-message.sh",
+                "timeout": 900
+            }
+        },
+        {
+            "type": "action",
+            "name": "Terminate leader of partition 1 non-gracefully",
+            "provider": {
+                "type": "process",
+                "path": "terminate-partition.sh",
+                "arguments": [ "Leader", "1" ]
+            }
+        },
+        {
+                "name": "Should be able to create a process and await the message correlation",
+                "type": "probe",
+                "tolerance": 0,
+                "provider": {
+                    "type": "process",
+                    "path": "await-message-correlation.sh",
+                    "timeout": 900
+                }
+        }
+    ],
+    "rollbacks": []
+}

go-chaos/internal/chaos-experiments/camunda-cloud/development/multiple-leader-restart/experiment.json

@@ -0,0 +1,110 @@
+{
+    "version": "0.1.0",
+    "title": "Zeebe Leader restart multiple times experiment",
+    "description": "Zeebe should be able to handle multiple leader changes in short period.",
+    "contributions": {
+        "reliability": "high",
+        "availability": "high"
+    },
+    "steady-state-hypothesis": {
+        "title": "Zeebe is alive",
+        "probes": [
+            {
+                "name": "All pods should be ready",
+                "type": "probe",
+                "tolerance": 0,
+                "provider": {
+                    "type": "process",
+                    "path": "verify-readiness.sh",
+                    "timeout": 900
+                }
+            },
+            {
+                "name": "Should be able to create process instances on partition one",
+                "type": "probe",
+                "tolerance": 0,
+                "provider": {
+                    "type": "process",
+                    "path": "verify-steady-state.sh",
+                    "arguments": "1",
+                    "timeout": 900
+                }
+            }
+        ]
+    },
+    "method": [
+        {
+            "type": "action",
+            "name": "Terminate leader of partition one",
+            "provider": {
+                "type": "process",
+                "path": "terminate-partition.sh",
+                "arguments": ["Leader", "1"],
+                "status": "0"
+            }
+        },
+        {
+            "name": "All pods should be ready",
+            "type": "probe",
+            "tolerance": 0,
+            "provider": {
+                "type": "process",
+                "path": "verify-readiness.sh",
+                "timeout": 900
+            }
+        },
+        {
+            "name": "Should be able to create process instances on partition one",
+            "type": "probe",
+            "tolerance": 0,
+            "provider": {
+                "type": "process",
+                "path": "verify-steady-state.sh",
+                "arguments": "1",
+                "timeout": 900
+            }
+        },
+        {
+            "type": "action",
+            "name": "Terminate leader of partition one",
+            "provider": {
+                "type": "process",
+                "path": "terminate-partition.sh",
+                "arguments": ["Leader", "1"],
+                "status": "0"
+            }
+        },
+        {
+            "name": "All pods should be ready",
+            "type": "probe",
+            "tolerance": 0,
+            "provider": {
+                "type": "process",
+                "path": "verify-readiness.sh",
+                "timeout": 900
+            }
+        },
+        {
+            "name": "Should be able to create process instances on partition one",
+            "type": "probe",
+            "tolerance": 0,
+            "provider": {
+                "type": "process",
+                "path": "verify-steady-state.sh",
+                "arguments": "1",
+                "timeout": 900
+            }
+        },
+        {
+            "type": "action",
+            "name": "Terminate leader of partition one",
+            "provider": {
+                "type": "process",
+                "path": "terminate-partition.sh",
+                "arguments": ["Leader", "1"],
+                "status": "0"
+            }
+        }
+    ],
+    "rollbacks": []
+}

go-chaos/internal/chaos-experiments/camunda-cloud/development/stress-cpu-on-broker/experiment.json

@@ -0,0 +1,46 @@
+{
+    "version": "0.1.0",
+    "title": "CPU stress on an Broker",
+    "description": "The cpu stress on an abritrary node should not cause any failures. We should be able to start and complete instances.",
+    "contributions": {
+        "reliability": "high",
+        "availability": "high"
+    },
+    "steady-state-hypothesis": {
+        "title": "Zeebe is alive",
+        "probes": [
+            {
+                "name": "All pods should be ready",
+                "type": "probe",
+                "tolerance": 0,
+                "provider": {
+                    "type": "process",
+                    "path": "verify-readiness.sh",
+                    "timeout": 900
+                }
+            },
+            {
+                "name": "Should be able to create process instances on partition 1",
+                "type": "probe",
+                "tolerance": 0,
+                "provider": {
+                    "type": "process",
+                    "path": "verify-steady-state.sh",
+                    "arguments": "1",
+                    "timeout": 900
+                }
+            }
+        ]
+    },
+    "method": [
+        {
+            "type": "action",
+            "name": "Stress CPU on Broker",
+            "provider": {
+                "type": "process",
+                "path": "stress-cpu.sh"
+            }
+        }
+    ],
+    "rollbacks": []
+}

go-chaos/internal/chaos-experiments/camunda-cloud/development/worker-restart/experiment.json

@@ -0,0 +1,51 @@
+{
+    "version": "0.1.0",
+    "title": "Zeebe Worker restart experiment",
+    "description": "Zeebe Workers should be able to reconnect after restart.",
+    "contributions": {
+        "reliability": "high",
+        "availability": "high"
+    },
+    "steady-state-hypothesis": {
+        "title": "Zeebe is alive",
+        "probes": [
+            {
+                "name": "All pods should be ready",
+                "type": "probe",
+                "tolerance": 0,
+                "provider": {
+                    "type": "process",
+                    "path": "verify-readiness.sh",
+                    "timeout": 900
+                }
+            },
+            {
+                "name": "Should be able to create a process and await the result",
+                "type": "probe",
+                "tolerance": 0,
+                "provider": {
+                    "type": "process",
+                    "path": "await-processes-with-result.sh",
+                    "arguments": "1",
+                    "timeout": 900
+                }
+            }
+        ]
+    },
+    "method": [
+        {
+            "type": "action",
+            "name": "Restart worker pod",
+            "tolerance": 0,
+            "provider": {
+                "type": "process",
+                "path": "terminate-workers.sh",
+                "timeout": 900
+            },
+            "pauses": {
+                "after": 5
+            }
+        }
+    ],
+    "rollbacks": []
+}