From 22f6cc46d7e8d9d96180f1658ba7b77189b94c03 Mon Sep 17 00:00:00 2001 From: Christopher Zell Date: Fri, 9 Dec 2022 13:56:31 +0100 Subject: [PATCH 1/8] fix: use flags pointer In order to be able to see value changes by cobra we need to pass pointers around --- go-chaos/cmd/backup.go | 10 +++++----- go-chaos/cmd/connect.go | 2 +- go-chaos/cmd/dataloss_sim.go | 2 +- go-chaos/cmd/deploy.go | 2 +- go-chaos/cmd/disconnect.go | 2 +- go-chaos/cmd/exporting.go | 2 +- go-chaos/cmd/publish.go | 2 +- go-chaos/cmd/restart.go | 2 +- go-chaos/cmd/root.go | 29 +++++++++++++++-------------- go-chaos/cmd/stress.go | 2 +- go-chaos/cmd/terminate.go | 2 +- go-chaos/cmd/topology.go | 2 +- go-chaos/cmd/verify.go | 2 +- go-chaos/cmd/zeebePods.go | 2 +- 14 files changed, 32 insertions(+), 31 deletions(-) diff --git a/go-chaos/cmd/backup.go b/go-chaos/cmd/backup.go index 153048227..f15244ae2 100644 --- a/go-chaos/cmd/backup.go +++ b/go-chaos/cmd/backup.go @@ -34,7 +34,7 @@ import ( "github.com/zeebe-io/zeebe-chaos/go-chaos/internal" ) -func AddBackupCommand(rootCmd *cobra.Command, flags Flags) { +func AddBackupCommand(rootCmd *cobra.Command, flags *Flags) { var backupCommand = &cobra.Command{ Use: "backup", @@ -182,7 +182,7 @@ func createBackupSecret(k8Client internal.K8Client, namespace string) (*core.Sec ) } -func takeBackup(flags Flags) error { +func takeBackup(flags *Flags) error { k8Client, err := createK8ClientWithFlags(flags) if err != nil { panic(err) @@ -211,7 +211,7 @@ func takeBackup(flags Flags) error { return err } -func waitForBackup(flags Flags) error { +func waitForBackup(flags *Flags) error { k8Client, err := createK8ClientWithFlags(flags) if err != nil { panic(err) @@ -240,7 +240,7 @@ func waitForBackup(flags Flags) error { } -func restoreFromBackup(flags Flags) error { +func restoreFromBackup(flags *Flags) error { k8Client, err := createK8ClientWithFlags(flags) if err != nil { panic(err) @@ -314,7 +314,7 @@ func restoreFromBackup(flags Flags) error { return nil } -func restoreEnvFromSfs(flags Flags, sfs *apps.StatefulSet) []core.EnvVar { +func restoreEnvFromSfs(flags *Flags, sfs *apps.StatefulSet) []core.EnvVar { zeebeEnv := sfs.Spec.Template.Spec.Containers[0].Env restoreEnv := make([]core.EnvVar, 0) for _, env := range zeebeEnv { diff --git a/go-chaos/cmd/connect.go b/go-chaos/cmd/connect.go index bd47b6991..d89bc0a71 100644 --- a/go-chaos/cmd/connect.go +++ b/go-chaos/cmd/connect.go @@ -19,7 +19,7 @@ import ( "github.com/zeebe-io/zeebe-chaos/go-chaos/backend" ) -func AddConnectCmd(rootCmd *cobra.Command, flags Flags) { +func AddConnectCmd(rootCmd *cobra.Command, flags *Flags) { var connect = &cobra.Command{ Use: "connect", Short: "Connect Zeebe nodes", diff --git a/go-chaos/cmd/dataloss_sim.go b/go-chaos/cmd/dataloss_sim.go index 22368eea9..b1c6a6812 100644 --- a/go-chaos/cmd/dataloss_sim.go +++ b/go-chaos/cmd/dataloss_sim.go @@ -21,7 +21,7 @@ import ( "github.com/zeebe-io/zeebe-chaos/go-chaos/internal" ) -func AddDatalossSimulationCmd(rootCmd *cobra.Command, flags Flags) { +func AddDatalossSimulationCmd(rootCmd *cobra.Command, flags *Flags) { var datalossCmd = &cobra.Command{ Use: "dataloss", diff --git a/go-chaos/cmd/deploy.go b/go-chaos/cmd/deploy.go index b0992b3a2..bfc531406 100644 --- a/go-chaos/cmd/deploy.go +++ b/go-chaos/cmd/deploy.go @@ -20,7 +20,7 @@ import ( "github.com/zeebe-io/zeebe-chaos/go-chaos/internal" ) -func AddDeployCmd(rootCmd *cobra.Command, flags Flags) { +func AddDeployCmd(rootCmd *cobra.Command, flags *Flags) { var deployCmd = &cobra.Command{ Use: "deploy", diff --git a/go-chaos/cmd/disconnect.go b/go-chaos/cmd/disconnect.go index a704992e6..efb066b47 100644 --- a/go-chaos/cmd/disconnect.go +++ b/go-chaos/cmd/disconnect.go @@ -25,7 +25,7 @@ func ensureNoError(err error) { } } -func AddDisconnectCommand(rootCmd *cobra.Command, flags Flags) { +func AddDisconnectCommand(rootCmd *cobra.Command, flags *Flags) { var disconnect = &cobra.Command{ Use: "disconnect", diff --git a/go-chaos/cmd/exporting.go b/go-chaos/cmd/exporting.go index 7cd5413b6..5bbd7e827 100644 --- a/go-chaos/cmd/exporting.go +++ b/go-chaos/cmd/exporting.go @@ -22,7 +22,7 @@ import ( "github.com/zeebe-io/zeebe-chaos/go-chaos/internal" ) -func AddExportingCmds(rootCmd *cobra.Command, flags Flags) { +func AddExportingCmds(rootCmd *cobra.Command, flags *Flags) { var exportingCommand = &cobra.Command{ Use: "exporting", Short: "Controls Zeebe Exporting", diff --git a/go-chaos/cmd/publish.go b/go-chaos/cmd/publish.go index 98fb5a80a..45ebb1dd1 100644 --- a/go-chaos/cmd/publish.go +++ b/go-chaos/cmd/publish.go @@ -22,7 +22,7 @@ import ( "github.com/zeebe-io/zeebe-chaos/go-chaos/internal" ) -func AddPublishCmd(rootCmd *cobra.Command, flags Flags) { +func AddPublishCmd(rootCmd *cobra.Command, flags *Flags) { var publishCmd = &cobra.Command{ Use: "publish", diff --git a/go-chaos/cmd/restart.go b/go-chaos/cmd/restart.go index ab1cd125f..8adb484df 100644 --- a/go-chaos/cmd/restart.go +++ b/go-chaos/cmd/restart.go @@ -19,7 +19,7 @@ import ( "github.com/zeebe-io/zeebe-chaos/go-chaos/internal" ) -func AddRestartCmd(rootCmd *cobra.Command, flags Flags) { +func AddRestartCmd(rootCmd *cobra.Command, flags *Flags) { var restartCmd = &cobra.Command{ Use: "restart", diff --git a/go-chaos/cmd/root.go b/go-chaos/cmd/root.go index e1078f2e5..12024509f 100644 --- a/go-chaos/cmd/root.go +++ b/go-chaos/cmd/root.go @@ -86,6 +86,7 @@ func NewCmd() *cobra.Command { Long: `A chaos experimenting toolkit for Zeebe. Perfect to inject some chaos into your brokers and gateways.`, PersistentPreRun: func(cmd *cobra.Command, args []string) { + internal.LogInfo("%v", flags) internal.Verbosity = Verbose internal.JsonLogging = JsonLogging if JsonLogging { @@ -99,26 +100,26 @@ func NewCmd() *cobra.Command { rootCmd.PersistentFlags().StringVar(&flags.kubeConfigPath, "kubeconfig", "", "path the the kube config that will be used") rootCmd.PersistentFlags().StringVarP(&flags.namespace, "namespace", "n", "", "connect to the given namespace") - AddBackupCommand(rootCmd, flags) - AddBrokersCommand(rootCmd, flags) - AddConnectCmd(rootCmd, flags) - AddDatalossSimulationCmd(rootCmd, flags) - AddDeployCmd(rootCmd, flags) - AddDisconnectCommand(rootCmd, flags) - AddExportingCmds(rootCmd, flags) - AddPublishCmd(rootCmd, flags) - AddRestartCmd(rootCmd, flags) - AddStressCmd(rootCmd, flags) - AddTerminateCommand(rootCmd, flags) - AddTopologyCmd(rootCmd, flags) - AddVerifyCommands(rootCmd, flags) + AddBackupCommand(rootCmd, &flags) + AddBrokersCommand(rootCmd, &flags) + AddConnectCmd(rootCmd, &flags) + AddDatalossSimulationCmd(rootCmd, &flags) + AddDeployCmd(rootCmd, &flags) + AddDisconnectCommand(rootCmd, &flags) + AddExportingCmds(rootCmd, &flags) + AddPublishCmd(rootCmd, &flags) + AddRestartCmd(rootCmd, &flags) + AddStressCmd(rootCmd, &flags) + AddTerminateCommand(rootCmd, &flags) + AddTopologyCmd(rootCmd, &flags) + AddVerifyCommands(rootCmd, &flags) AddVersionCmd(rootCmd) AddWorkerCmd(rootCmd) return rootCmd } -func createK8ClientWithFlags(flags Flags) (internal.K8Client, error) { +func createK8ClientWithFlags(flags *Flags) (internal.K8Client, error) { return internal.CreateK8Client(flags.kubeConfigPath, flags.namespace) } diff --git a/go-chaos/cmd/stress.go b/go-chaos/cmd/stress.go index 092335ecf..d22b74573 100644 --- a/go-chaos/cmd/stress.go +++ b/go-chaos/cmd/stress.go @@ -24,7 +24,7 @@ import ( v1 "k8s.io/api/core/v1" ) -func AddStressCmd(rootCmd *cobra.Command, flags Flags) { +func AddStressCmd(rootCmd *cobra.Command, flags *Flags) { stress := &cobra.Command{ Use: "stress", Short: "Put stress on a Zeebe node", diff --git a/go-chaos/cmd/terminate.go b/go-chaos/cmd/terminate.go index 2e0484f39..52f0fc366 100644 --- a/go-chaos/cmd/terminate.go +++ b/go-chaos/cmd/terminate.go @@ -22,7 +22,7 @@ import ( "github.com/zeebe-io/zeebe-chaos/go-chaos/internal" ) -func AddTerminateCommand(rootCmd *cobra.Command, flags Flags) { +func AddTerminateCommand(rootCmd *cobra.Command, flags *Flags) { var terminateCmd = &cobra.Command{ Use: "terminate", diff --git a/go-chaos/cmd/topology.go b/go-chaos/cmd/topology.go index 66f6a1d4b..c9950e658 100644 --- a/go-chaos/cmd/topology.go +++ b/go-chaos/cmd/topology.go @@ -26,7 +26,7 @@ import ( "github.com/zeebe-io/zeebe-chaos/go-chaos/internal" ) -func AddTopologyCmd(rootCmd *cobra.Command, flags Flags) { +func AddTopologyCmd(rootCmd *cobra.Command, flags *Flags) { var topologyCmd = &cobra.Command{ Use: "topology", diff --git a/go-chaos/cmd/verify.go b/go-chaos/cmd/verify.go index 53fcfc554..e8baeb15b 100644 --- a/go-chaos/cmd/verify.go +++ b/go-chaos/cmd/verify.go @@ -21,7 +21,7 @@ import ( "github.com/zeebe-io/zeebe-chaos/go-chaos/internal" ) -func AddVerifyCommands(rootCmd *cobra.Command, flags Flags) { +func AddVerifyCommands(rootCmd *cobra.Command, flags *Flags) { var verifyCmd = &cobra.Command{ Use: "verify", diff --git a/go-chaos/cmd/zeebePods.go b/go-chaos/cmd/zeebePods.go index 75643e79e..90df3bd82 100644 --- a/go-chaos/cmd/zeebePods.go +++ b/go-chaos/cmd/zeebePods.go @@ -19,7 +19,7 @@ import ( "github.com/zeebe-io/zeebe-chaos/go-chaos/internal" ) -func AddBrokersCommand(rootCmd *cobra.Command, flags Flags) { +func AddBrokersCommand(rootCmd *cobra.Command, flags *Flags) { var getZeebeBrokersCmd = &cobra.Command{ Use: "brokers", Short: "Print the name of the Zeebe broker pods", From 2b77f8a3f5b87843ac63a2fa1a8c345e8aa6e594 Mon Sep 17 00:00:00 2001 From: Christopher Zell Date: Fri, 9 Dec 2022 14:53:04 +0100 Subject: [PATCH 2/8] refactor: make flags logging verbose --- go-chaos/cmd/root.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go-chaos/cmd/root.go b/go-chaos/cmd/root.go index 12024509f..c4b47e87e 100644 --- a/go-chaos/cmd/root.go +++ b/go-chaos/cmd/root.go @@ -86,8 +86,8 @@ func NewCmd() *cobra.Command { Long: `A chaos experimenting toolkit for Zeebe. Perfect to inject some chaos into your brokers and gateways.`, PersistentPreRun: func(cmd *cobra.Command, args []string) { - internal.LogInfo("%v", flags) internal.Verbosity = Verbose + internal.LogVerbose("Flags: %v", flags) internal.JsonLogging = JsonLogging if JsonLogging { internal.JsonLogger = log.With().Logger() From 7c89b23ca7dc8cc4607b5bf498af1a9a997be4fe Mon Sep 17 00:00:00 2001 From: Christopher Zell Date: Fri, 9 Dec 2022 14:53:52 +0100 Subject: [PATCH 3/8] fix: use random gateway Instead of always trying the same gateway, which might be broken try to use different ones. Log where the port forward goes to. --- go-chaos/internal/pods.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/go-chaos/internal/pods.go b/go-chaos/internal/pods.go index b9cf658cd..1edc49b80 100644 --- a/go-chaos/internal/pods.go +++ b/go-chaos/internal/pods.go @@ -27,6 +27,7 @@ import ( v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/rand" "k8s.io/client-go/kubernetes/scheme" "k8s.io/client-go/tools/portforward" "k8s.io/client-go/tools/remotecommand" @@ -281,11 +282,13 @@ func (c K8Client) createPortForwarder(localPort int, remotePort int, portForward // createPortForwardUrl constructs the Url to which is used to create the port forwarding func (c K8Client) createPortForwardUrl(names []string) *url.URL { + gatewayName := names[rand.Intn(len(names))] + LogVerbose("Port forward to %s", gatewayName) restClient := c.Clientset.CoreV1().RESTClient() portForwardCreateURL := restClient.Post(). Resource("pods"). Namespace(c.GetCurrentNamespace()). - Name(names[0]). + Name(gatewayName). SubResource("portforward"). URL() return portForwardCreateURL From 040a9e97e6909726ccaa2664943f26ed5a6a2fa0 Mon Sep 17 00:00:00 2001 From: Christopher Zell Date: Mon, 12 Dec 2022 11:09:31 +0100 Subject: [PATCH 4/8] fix: add pause after stress Previously we encountered several issues due to putting stress on broker, since this causes leader changes and then the deployment will not work. We should wait for a while and then try to verify the steady state --- .../production-s/stress-cpu-on-broker/experiment.json | 3 +++ 1 file changed, 3 insertions(+) diff --git a/go-chaos/internal/chaos-experiments/camunda-cloud/production-s/stress-cpu-on-broker/experiment.json b/go-chaos/internal/chaos-experiments/camunda-cloud/production-s/stress-cpu-on-broker/experiment.json index bc2c1cf0f..138ea10f8 100644 --- a/go-chaos/internal/chaos-experiments/camunda-cloud/production-s/stress-cpu-on-broker/experiment.json +++ b/go-chaos/internal/chaos-experiments/camunda-cloud/production-s/stress-cpu-on-broker/experiment.json @@ -52,6 +52,9 @@ "type": "process", "path": "zbchaos", "arguments": ["stress", "broker", "--cpu"] + }, + "pauses": { + "after": 30 } } ], From cd95ca28d5d3af11db2793948614aaa6049b0f57 Mon Sep 17 00:00:00 2001 From: Christopher Zell Date: Tue, 13 Dec 2022 05:51:26 +0100 Subject: [PATCH 5/8] fix: remove multiple leader restart The execution of this experiment is currently instable and doesn't work with the gateway, not resolving the ip addr everytime --- .../multiple-leader-restart/experiment.json | 121 ------------------ 1 file changed, 121 deletions(-) delete mode 100644 go-chaos/internal/chaos-experiments/camunda-cloud/production-s/multiple-leader-restart/experiment.json diff --git a/go-chaos/internal/chaos-experiments/camunda-cloud/production-s/multiple-leader-restart/experiment.json b/go-chaos/internal/chaos-experiments/camunda-cloud/production-s/multiple-leader-restart/experiment.json deleted file mode 100644 index 4b28ad038..000000000 --- a/go-chaos/internal/chaos-experiments/camunda-cloud/production-s/multiple-leader-restart/experiment.json +++ /dev/null @@ -1,121 +0,0 @@ -{ - "version": "0.1.0", - "title": "Zeebe Leader restart multiple times experiment", - "description": "Zeebe should be able to handle multiple leader changes in short period.", - "contributions": { - "reliability": "high", - "availability": "high" - }, - "steady-state-hypothesis": { - "title": "Zeebe is alive", - "probes": [ - { - "name": "All pods should be ready", - "type": "probe", - "tolerance": 0, - "provider": { - "type": "process", - "path": "zbchaos", - "arguments": ["verify", "readiness"], - "timeout": 900 - } - }, - { - "name": "Can deploy process model", - "type": "probe", - "tolerance": 0, - "provider": { - "type": "process", - "path": "zbchaos", - "arguments": ["deploy", "process"], - "timeout": 900 - } - }, - { - "name": "Should be able to create process instances on partition 1", - "type": "probe", - "tolerance": 0, - "provider": { - "type": "process", - "path": "zbchaos", - "arguments": ["verify", "instance-creation", "--partitionId", "1"], - "timeout": 900 - } - } - ] - }, - "method": [ - { - "type": "action", - "name": "Terminate leader of partition 1 non-gracefully", - "provider": { - "type": "process", - "path": "zbchaos", - "arguments": ["terminate", "broker", "--role", "LEADER", "--partitionId", "1"] - } - }, - { - "name": "All pods should be ready", - "type": "probe", - "tolerance": 0, - "provider": { - "type": "process", - "path": "zbchaos", - "arguments": ["verify", "readiness"], - "timeout": 900 - } - }, - { - "name": "Should be able to create process instances on partition 1", - "type": "probe", - "tolerance": 0, - "provider": { - "type": "process", - "path": "zbchaos", - "arguments": ["verify", "instance-creation", "--partitionId", "1"], - "timeout": 900 - } - }, - { - "type": "action", - "name": "Terminate leader of partition 1 non-gracefully", - "provider": { - "type": "process", - "path": "zbchaos", - "arguments": ["terminate", "broker", "--role", "LEADER", "--partitionId", "1"] - } - }, - { - "name": "All pods should be ready", - "type": "probe", - "tolerance": 0, - "provider": { - "type": "process", - "path": "zbchaos", - "arguments": ["verify", "readiness"], - "timeout": 900 - } - }, - { - "name": "Should be able to create process instances on partition 1", - "type": "probe", - "tolerance": 0, - "provider": { - "type": "process", - "path": "zbchaos", - "arguments": ["verify", "instance-creation", "--partitionId", "1"], - "timeout": 900 - } - }, - { - "type": "action", - "name": "Terminate leader of partition 1 non-gracefully", - "provider": { - "type": "process", - "path": "zbchaos", - "arguments": ["terminate", "broker", "--role", "LEADER", "--partitionId", "1"] - } - } - ], - "rollbacks": [] -} From 9f393ed01eb0b0f619abf6687962349fddbd3180 Mon Sep 17 00:00:00 2001 From: Christopher Zell Date: Tue, 13 Dec 2022 16:40:21 +0100 Subject: [PATCH 6/8] fix: increase timeout --- go-chaos/integration/integration_test.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/go-chaos/integration/integration_test.go b/go-chaos/integration/integration_test.go index 23190d179..5ef8785c4 100644 --- a/go-chaos/integration/integration_test.go +++ b/go-chaos/integration/integration_test.go @@ -73,7 +73,9 @@ func Test_ShouldBeAbleToRunExperiments(t *testing.T) { go cmd.OpenWorkers(zeebeClient) go func() { internal.LogInfo("Create ChaosToolkit instance") - response, err := commandStep3.WithResult().Send(ctx) + deadline, cancelFunc := context.WithDeadline(ctx, time.UnixMilli(time.Now().UnixMilli()+int64(60*time.Minute))) + defer cancelFunc() + response, err := commandStep3.WithResult().Send(deadline) require.NoError(t, err) internal.LogInfo("Instance %d [definition %d ] completed", response.ProcessInstanceKey, response.ProcessDefinitionKey) close(done) From 8390f41258bce30be082cac8713ad03f218c5665 Mon Sep 17 00:00:00 2001 From: Christopher Zell Date: Thu, 15 Dec 2022 08:40:52 +0100 Subject: [PATCH 7/8] Revert "fix: remove multiple leader restart" This reverts commit cd95ca28d5d3af11db2793948614aaa6049b0f57. --- .../multiple-leader-restart/experiment.json | 121 ++++++++++++++++++ 1 file changed, 121 insertions(+) create mode 100644 go-chaos/internal/chaos-experiments/camunda-cloud/production-s/multiple-leader-restart/experiment.json diff --git a/go-chaos/internal/chaos-experiments/camunda-cloud/production-s/multiple-leader-restart/experiment.json b/go-chaos/internal/chaos-experiments/camunda-cloud/production-s/multiple-leader-restart/experiment.json new file mode 100644 index 000000000..4b28ad038 --- /dev/null +++ b/go-chaos/internal/chaos-experiments/camunda-cloud/production-s/multiple-leader-restart/experiment.json @@ -0,0 +1,121 @@ +{ + "version": "0.1.0", + "title": "Zeebe Leader restart multiple times experiment", + "description": "Zeebe should be able to handle multiple leader changes in short period.", + "contributions": { + "reliability": "high", + "availability": "high" + }, + "steady-state-hypothesis": { + "title": "Zeebe is alive", + "probes": [ + { + "name": "All pods should be ready", + "type": "probe", + "tolerance": 0, + "provider": { + "type": "process", + "path": "zbchaos", + "arguments": ["verify", "readiness"], + "timeout": 900 + } + }, + { + "name": "Can deploy process model", + "type": "probe", + "tolerance": 0, + "provider": { + "type": "process", + "path": "zbchaos", + "arguments": ["deploy", "process"], + "timeout": 900 + } + }, + { + "name": "Should be able to create process instances on partition 1", + "type": "probe", + "tolerance": 0, + "provider": { + "type": "process", + "path": "zbchaos", + "arguments": ["verify", "instance-creation", "--partitionId", "1"], + "timeout": 900 + } + } + ] + }, + "method": [ + { + "type": "action", + "name": "Terminate leader of partition 1 non-gracefully", + "provider": { + "type": "process", + "path": "zbchaos", + "arguments": ["terminate", "broker", "--role", "LEADER", "--partitionId", "1"] + } + }, + { + "name": "All pods should be ready", + "type": "probe", + "tolerance": 0, + "provider": { + "type": "process", + "path": "zbchaos", + "arguments": ["verify", "readiness"], + "timeout": 900 + } + }, + { + "name": "Should be able to create process instances on partition 1", + "type": "probe", + "tolerance": 0, + "provider": { + "type": "process", + "path": "zbchaos", + "arguments": ["verify", "instance-creation", "--partitionId", "1"], + "timeout": 900 + } + }, + { + "type": "action", + "name": "Terminate leader of partition 1 non-gracefully", + "provider": { + "type": "process", + "path": "zbchaos", + "arguments": ["terminate", "broker", "--role", "LEADER", "--partitionId", "1"] + } + }, + { + "name": "All pods should be ready", + "type": "probe", + "tolerance": 0, + "provider": { + "type": "process", + "path": "zbchaos", + "arguments": ["verify", "readiness"], + "timeout": 900 + } + }, + { + "name": "Should be able to create process instances on partition 1", + "type": "probe", + "tolerance": 0, + "provider": { + "type": "process", + "path": "zbchaos", + "arguments": ["verify", "instance-creation", "--partitionId", "1"], + "timeout": 900 + } + }, + { + "type": "action", + "name": "Terminate leader of partition 1 non-gracefully", + "provider": { + "type": "process", + "path": "zbchaos", + "arguments": ["terminate", "broker", "--role", "LEADER", "--partitionId", "1"] + } + } + ], + "rollbacks": [] +} From 180f41f513432433a8851491819319a1c6bfba95 Mon Sep 17 00:00:00 2001 From: Christopher Zell Date: Thu, 15 Dec 2022 15:44:57 +0100 Subject: [PATCH 8/8] fix: adjust experiments to not always target one broker --- .../leader-restart/experiment.json | 8 +++---- .../leader-terminate/experiment.json | 8 +++---- .../multiple-leader-restart/experiment.json | 24 +++++++++---------- .../stress-cpu-on-broker/experiment.json | 2 +- 4 files changed, 21 insertions(+), 21 deletions(-) diff --git a/go-chaos/internal/chaos-experiments/camunda-cloud/production-s/leader-restart/experiment.json b/go-chaos/internal/chaos-experiments/camunda-cloud/production-s/leader-restart/experiment.json index 806963bf9..8510ef2a2 100644 --- a/go-chaos/internal/chaos-experiments/camunda-cloud/production-s/leader-restart/experiment.json +++ b/go-chaos/internal/chaos-experiments/camunda-cloud/production-s/leader-restart/experiment.json @@ -32,13 +32,13 @@ } }, { - "name": "Should be able to create process instances on partition 1", + "name": "Should be able to create process instances on partition three", "type": "probe", "tolerance": 0, "provider": { "type": "process", "path": "zbchaos", - "arguments": ["verify", "instance-creation", "--partitionId", "1"], + "arguments": ["verify", "instance-creation", "--partitionId", "3"], "timeout": 900 } } @@ -47,11 +47,11 @@ "method": [ { "type": "action", - "name": "Restart leader of partition 1", + "name": "Restart leader of partition three", "provider": { "type": "process", "path": "zbchaos", - "arguments": ["restart", "broker", "--role", "LEADER", "--partitionId", "1"] + "arguments": ["restart", "broker", "--role", "LEADER", "--partitionId", "3"] } } ], diff --git a/go-chaos/internal/chaos-experiments/camunda-cloud/production-s/leader-terminate/experiment.json b/go-chaos/internal/chaos-experiments/camunda-cloud/production-s/leader-terminate/experiment.json index dd22d948a..57beb229b 100644 --- a/go-chaos/internal/chaos-experiments/camunda-cloud/production-s/leader-terminate/experiment.json +++ b/go-chaos/internal/chaos-experiments/camunda-cloud/production-s/leader-terminate/experiment.json @@ -32,13 +32,13 @@ } }, { - "name": "Should be able to create process instances on partition 1", + "name": "Should be able to create process instances on partition two", "type": "probe", "tolerance": 0, "provider": { "type": "process", "path": "zbchaos", - "arguments": ["verify", "instance-creation", "--partitionId", "1"], + "arguments": ["verify", "instance-creation", "--partitionId", "2"], "timeout": 900 } } @@ -47,11 +47,11 @@ "method": [ { "type": "action", - "name": "Terminate leader of partition 1 non-gracefully", + "name": "Terminate leader of partition two non-gracefully", "provider": { "type": "process", "path": "zbchaos", - "arguments": ["terminate", "broker", "--role", "LEADER", "--partitionId", "1"] + "arguments": ["terminate", "broker", "--role", "LEADER", "--partitionId", "2"] } } ], diff --git a/go-chaos/internal/chaos-experiments/camunda-cloud/production-s/multiple-leader-restart/experiment.json b/go-chaos/internal/chaos-experiments/camunda-cloud/production-s/multiple-leader-restart/experiment.json index 4b28ad038..3bfa027ca 100644 --- a/go-chaos/internal/chaos-experiments/camunda-cloud/production-s/multiple-leader-restart/experiment.json +++ b/go-chaos/internal/chaos-experiments/camunda-cloud/production-s/multiple-leader-restart/experiment.json @@ -32,13 +32,13 @@ } }, { - "name": "Should be able to create process instances on partition 1", + "name": "Should be able to create process instances on partition two", "type": "probe", "tolerance": 0, "provider": { "type": "process", "path": "zbchaos", - "arguments": ["verify", "instance-creation", "--partitionId", "1"], + "arguments": ["verify", "instance-creation", "--partitionId", "2"], "timeout": 900 } } @@ -47,11 +47,11 @@ "method": [ { "type": "action", - "name": "Terminate leader of partition 1 non-gracefully", + "name": "Terminate leader of partition two non-gracefully", "provider": { "type": "process", "path": "zbchaos", - "arguments": ["terminate", "broker", "--role", "LEADER", "--partitionId", "1"] + "arguments": ["terminate", "broker", "--role", "LEADER", "--partitionId", "2"] } }, { @@ -66,23 +66,23 @@ } }, { - "name": "Should be able to create process instances on partition 1", + "name": "Should be able to create process instances on partition two", "type": "probe", "tolerance": 0, "provider": { "type": "process", "path": "zbchaos", - "arguments": ["verify", "instance-creation", "--partitionId", "1"], + "arguments": ["verify", "instance-creation", "--partitionId", "2"], "timeout": 900 } }, { "type": "action", - "name": "Terminate leader of partition 1 non-gracefully", + "name": "Terminate leader of partition two non-gracefully", "provider": { "type": "process", "path": "zbchaos", - "arguments": ["terminate", "broker", "--role", "LEADER", "--partitionId", "1"] + "arguments": ["terminate", "broker", "--role", "LEADER", "--partitionId", "2"] } }, { @@ -97,23 +97,23 @@ } }, { - "name": "Should be able to create process instances on partition 1", + "name": "Should be able to create process instances on partition two", "type": "probe", "tolerance": 0, "provider": { "type": "process", "path": "zbchaos", - "arguments": ["verify", "instance-creation", "--partitionId", "1"], + "arguments": ["verify", "instance-creation", "--partitionId", "2"], "timeout": 900 } }, { "type": "action", - "name": "Terminate leader of partition 1 non-gracefully", + "name": "Terminate leader of partition two non-gracefully", "provider": { "type": "process", "path": "zbchaos", - "arguments": ["terminate", "broker", "--role", "LEADER", "--partitionId", "1"] + "arguments": ["terminate", "broker", "--role", "LEADER", "--partitionId", "2"] } } ], diff --git a/go-chaos/internal/chaos-experiments/camunda-cloud/production-s/stress-cpu-on-broker/experiment.json b/go-chaos/internal/chaos-experiments/camunda-cloud/production-s/stress-cpu-on-broker/experiment.json index 138ea10f8..b7ade71b2 100644 --- a/go-chaos/internal/chaos-experiments/camunda-cloud/production-s/stress-cpu-on-broker/experiment.json +++ b/go-chaos/internal/chaos-experiments/camunda-cloud/production-s/stress-cpu-on-broker/experiment.json @@ -51,7 +51,7 @@ "provider": { "type": "process", "path": "zbchaos", - "arguments": ["stress", "broker", "--cpu"] + "arguments": ["stress", "broker", "--cpu", "--role=LEADER", "--partitionId=3"] }, "pauses": { "after": 30