From 22dd18240018afdc678d207f019cb3450445b851 Mon Sep 17 00:00:00 2001 From: Christopher Zell Date: Tue, 22 Nov 2022 16:27:42 +0100 Subject: [PATCH 1/7] refactor: extract restartBroker Extract method in order to reuse in restart and terminate broker command --- go-chaos/cmd/restart.go | 30 ++++------------------------ go-chaos/cmd/terminate.go | 41 +++++++++++++++++++++------------------ go-chaos/internal/pods.go | 5 +++++ 3 files changed, 31 insertions(+), 45 deletions(-) diff --git a/go-chaos/cmd/restart.go b/go-chaos/cmd/restart.go index 7da0fd89f..d22f47613 100644 --- a/go-chaos/cmd/restart.go +++ b/go-chaos/cmd/restart.go @@ -27,6 +27,8 @@ func init() { restartCmd.AddCommand(restartBrokerCmd) restartBrokerCmd.Flags().StringVar(&role, "role", "LEADER", "Specify the partition role [LEADER, FOLLOWER, INACTIVE]") restartBrokerCmd.Flags().IntVar(&partitionId, "partitionId", 1, "Specify the id of the partition") + restartBrokerCmd.Flags().IntVar(&nodeId, "nodeId", -1, "Specify the nodeId of the Broker") + restartBrokerCmd.MarkFlagsMutuallyExclusive("partitionId", "nodeId") restartCmd.AddCommand(restartGatewayCmd) restartCmd.AddCommand(restartWorkerCmd) @@ -44,32 +46,8 @@ var restartBrokerCmd = &cobra.Command{ Short: "Restarts a Zeebe broker", Long: `Restarts a Zeebe broker with a certain role and given partition.`, Run: func(cmd *cobra.Command, args []string) { - k8Client, err := internal.CreateK8Client() - if err != nil { - panic(err) - } - - port := 26500 - closeFn := k8Client.MustGatewayPortForward(port, port) - defer closeFn() - - zbClient, err := internal.CreateZeebeClient(port) - if err != nil { - panic(err) - } - defer zbClient.Close() - broker, err := internal.GetBrokerPodNameForPartitionAndRole(k8Client, zbClient, partitionId, role) - if err != nil { - panic(err) - } - - err = k8Client.RestartPod(broker) - if err != nil { - panic(err) - } - - fmt.Printf("Restarted %s", broker) - fmt.Println() + brokerPod := restartBroker(nodeId, partitionId, role, nil) + fmt.Printf("Restarted %s\n", brokerPod) }, } diff --git a/go-chaos/cmd/terminate.go b/go-chaos/cmd/terminate.go index 4c107309a..adfeac162 100644 --- a/go-chaos/cmd/terminate.go +++ b/go-chaos/cmd/terminate.go @@ -52,29 +52,32 @@ var terminateBrokerCmd = &cobra.Command{ Short: "Terminates a Zeebe broker", Long: `Terminates a Zeebe broker with a certain role and given partition.`, Run: func(cmd *cobra.Command, args []string) { - k8Client, err := internal.CreateK8Client() - if err != nil { - panic(err) - } + gracePeriodSec := int64(0) + brokerName := restartBroker(nodeId, partitionId, role, &gracePeriodSec) + fmt.Printf("Terminated %s\n", brokerName) + }, +} - port := 26500 - closeFn := k8Client.MustGatewayPortForward(port, port) - defer closeFn() +// Restart a broker pod. Pod is identified either by nodeId or by partitionId and role. +// GracePeriod (in second) can be negative, which would mean use default. +// Returns the broker which has been restarted +func restartBroker(nodeId int, partitionId int, role string, gracePeriod *int64) string { + k8Client, err := internal.CreateK8Client() + ensureNoError(err) - zbClient, err := internal.CreateZeebeClient(port) - if err != nil { - panic(err.Error()) - } - defer zbClient.Close() + port := 26500 + closeFn := k8Client.MustGatewayPortForward(port, port) + defer closeFn() - brokerPod := getBrokerPod(k8Client, zbClient, nodeId, partitionId, role) - err = k8Client.TerminatePod(brokerPod.Name) - if err != nil { - panic(err.Error()) - } + zbClient, err := internal.CreateZeebeClient(port) + ensureNoError(err) + defer zbClient.Close() - fmt.Printf("Terminated %s\n", brokerPod.Name) - }, + brokerPod := getBrokerPod(k8Client, zbClient, nodeId, partitionId, role) + err = k8Client.RestartPodWithGracePeriod(brokerPod.Name, gracePeriod) + ensureNoError(err) + + return brokerPod.Name } var terminateGatewayCmd = &cobra.Command{ diff --git a/go-chaos/internal/pods.go b/go-chaos/internal/pods.go index 09f9616a3..44d37029b 100644 --- a/go-chaos/internal/pods.go +++ b/go-chaos/internal/pods.go @@ -115,6 +115,11 @@ func (c K8Client) TerminatePod(podName string) error { return c.Clientset.CoreV1().Pods(c.GetCurrentNamespace()).Delete(context.TODO(), podName, options) } +func (c K8Client) RestartPodWithGracePeriod(podName string, gracePeriodSec *int64) error { + options := metav1.DeleteOptions{GracePeriodSeconds: gracePeriodSec} + return c.Clientset.CoreV1().Pods(c.GetCurrentNamespace()).Delete(context.TODO(), podName, options) +} + func (c K8Client) AwaitReadiness() error { retries := 0 maxRetries := 300 // 5 * 60s From 785a0ed0793fe3bd074ed85d6a3d6ebfab65a7c8 Mon Sep 17 00:00:00 2001 From: Christopher Zell Date: Tue, 22 Nov 2022 16:34:03 +0100 Subject: [PATCH 2/7] refactor: extract restart gateway Extract method to restart gateway and reuse in restart and terminate command --- go-chaos/cmd/restart.go | 21 +-------------------- go-chaos/cmd/terminate.go | 38 ++++++++++++++++++++------------------ 2 files changed, 21 insertions(+), 38 deletions(-) diff --git a/go-chaos/cmd/restart.go b/go-chaos/cmd/restart.go index d22f47613..1b220f393 100644 --- a/go-chaos/cmd/restart.go +++ b/go-chaos/cmd/restart.go @@ -56,26 +56,7 @@ var restartGatewayCmd = &cobra.Command{ Short: "Restarts a Zeebe gateway", Long: `Restarts a Zeebe gateway.`, Run: func(cmd *cobra.Command, args []string) { - k8Client, err := internal.CreateK8Client() - if err != nil { - panic(err) - } - - gatewayPodNames, err := k8Client.GetGatewayPodNames() - if err != nil { - panic(err) - } - - if len(gatewayPodNames) <= 0 { - panic(errors.New(fmt.Sprintf("Expected to find Zeebe gateway in namespace %s, but none found.", k8Client.GetCurrentNamespace()))) - } - - gatewayPod := gatewayPodNames[0] - err = k8Client.RestartPod(gatewayPod) - if err != nil { - panic(err) - } - + gatewayPod := restartGateway(nil) fmt.Printf("Restarted %s\n", gatewayPod) }, } diff --git a/go-chaos/cmd/terminate.go b/go-chaos/cmd/terminate.go index adfeac162..4c6916f2b 100644 --- a/go-chaos/cmd/terminate.go +++ b/go-chaos/cmd/terminate.go @@ -85,28 +85,30 @@ var terminateGatewayCmd = &cobra.Command{ Short: "Terminates a Zeebe gateway", Long: `Terminates a Zeebe gateway.`, Run: func(cmd *cobra.Command, args []string) { - k8Client, err := internal.CreateK8Client() - if err != nil { - panic(err) - } + gracePeriodSec := int64(0) + gatewayPod := restartGateway(&gracePeriodSec) + fmt.Printf("Terminated %s\n", gatewayPod) + }, +} - gatewayPodNames, err := k8Client.GetGatewayPodNames() - if err != nil { - panic(err) - } +// Restart a gateway pod. The pod is the first from a list of existing pods. +// GracePeriod (in second) can be negative, which would mean use default. +// Returns the gateway which has been restarted +func restartGateway(gracePeriod *int64) string { + k8Client, err := internal.CreateK8Client() + ensureNoError(err) - if len(gatewayPodNames) <= 0 { - panic(errors.New(fmt.Sprintf("Expected to find Zeebe gateway in namespace %s, but none found.", k8Client.GetCurrentNamespace()))) - } + gatewayPodNames, err := k8Client.GetGatewayPodNames() + ensureNoError(err) - gatewayPod := gatewayPodNames[0] - err = k8Client.TerminatePod(gatewayPod) - if err != nil { - panic(err) - } + if len(gatewayPodNames) <= 0 { + panic(errors.New(fmt.Sprintf("Expected to find Zeebe gateway in namespace %s, but none found.", k8Client.GetCurrentNamespace()))) + } - fmt.Printf("Terminated %s\n", gatewayPod) - }, + gatewayPod := gatewayPodNames[0] + err = k8Client.RestartPodWithGracePeriod(gatewayPod, gracePeriod) + ensureNoError(err) + return gatewayPod } var terminateWorkerCmd = &cobra.Command{ From 7b156ff8b61292ef55d91aa7495085dd52d7865f Mon Sep 17 00:00:00 2001 From: Christopher Zell Date: Tue, 22 Nov 2022 16:39:41 +0100 Subject: [PATCH 3/7] refactor: extract worker restart Extract method to restart workers, can be used in restart and terminate command --- go-chaos/cmd/restart.go | 26 +--------------------- go-chaos/cmd/terminate.go | 46 +++++++++++++++++++++++---------------- 2 files changed, 28 insertions(+), 44 deletions(-) diff --git a/go-chaos/cmd/restart.go b/go-chaos/cmd/restart.go index 1b220f393..112420d82 100644 --- a/go-chaos/cmd/restart.go +++ b/go-chaos/cmd/restart.go @@ -15,11 +15,9 @@ package cmd import ( - "errors" "fmt" "github.com/spf13/cobra" - "github.com/zeebe-io/zeebe-chaos/go-chaos/internal" ) func init() { @@ -66,28 +64,6 @@ var restartWorkerCmd = &cobra.Command{ Short: "Restart a Zeebe worker", Long: `Restart a Zeebe worker.`, Run: func(cmd *cobra.Command, args []string) { - k8Client, err := internal.CreateK8Client() - ensureNoError(err) - - workerPods, err := k8Client.GetWorkerPods() - ensureNoError(err) - - if workerPods == nil || len(workerPods.Items) <= 0 { - panic(errors.New(fmt.Sprintf("Expected to find workers in namespace %s, but none found.", k8Client.GetCurrentNamespace()))) - } - - if all { - for _, worker := range workerPods.Items { - err = k8Client.RestartPod(worker.Name) - ensureNoError(err) - fmt.Printf("Restart %s\n", worker.Name) - } - } else { - workerPod := workerPods.Items[0] - err = k8Client.RestartPod(workerPod.Name) - ensureNoError(err) - - fmt.Printf("Restart %s\n", workerPod.Name) - } + restartWorker(all, "Restarted", nil) }, } diff --git a/go-chaos/cmd/terminate.go b/go-chaos/cmd/terminate.go index 4c6916f2b..07cb12323 100644 --- a/go-chaos/cmd/terminate.go +++ b/go-chaos/cmd/terminate.go @@ -116,28 +116,36 @@ var terminateWorkerCmd = &cobra.Command{ Short: "Terminates a Zeebe worker", Long: `Terminates a Zeebe worker.`, Run: func(cmd *cobra.Command, args []string) { - k8Client, err := internal.CreateK8Client() - ensureNoError(err) + gracePeriodSec := int64(0) + restartWorker(all, "Terminated", &gracePeriodSec) + }, +} - workerPods, err := k8Client.GetWorkerPods() - ensureNoError(err) +// Restart a worker pod. The pod is the first from a list of existing pods, if all is not specified. +// GracePeriod (in second) can be negative, which would mean use default. +// The actionName specifies whether it was restarted or terminated to log the right thing. +func restartWorker(all bool, actionName string, gracePeriod *int64) { + k8Client, err := internal.CreateK8Client() + ensureNoError(err) - if workerPods == nil || len(workerPods.Items) <= 0 { - panic(errors.New(fmt.Sprintf("Expected to find workers in namespace %s, but none found.", k8Client.GetCurrentNamespace()))) - } + workerPods, err := k8Client.GetWorkerPods() + ensureNoError(err) - if all { - for _, worker := range workerPods.Items { - err = k8Client.TerminatePod(worker.Name) - ensureNoError(err) - fmt.Printf("Terminated %s\n", worker.Name) - } - } else { - workerPod := workerPods.Items[0] - err = k8Client.TerminatePod(workerPod.Name) - ensureNoError(err) + if workerPods == nil || len(workerPods.Items) <= 0 { + panic(errors.New(fmt.Sprintf("Expected to find workers in namespace %s, but none found.", k8Client.GetCurrentNamespace()))) + } - fmt.Printf("Terminated %s\n", workerPod.Name) + if all { + for _, worker := range workerPods.Items { + err = k8Client.RestartPodWithGracePeriod(worker.Name, gracePeriod) + ensureNoError(err) + fmt.Printf("%s %s\n", actionName, worker.Name) } - }, + } else { + workerPod := workerPods.Items[0] + err = k8Client.RestartPodWithGracePeriod(workerPod.Name, gracePeriod) + ensureNoError(err) + + fmt.Printf("%s %s\n", actionName, workerPod.Name) + } } From 69b80e918b6455dd826b08a5122d2c6409be1ff1 Mon Sep 17 00:00:00 2001 From: Christopher Zell Date: Tue, 22 Nov 2022 16:41:58 +0100 Subject: [PATCH 4/7] refactor: reorder functions --- go-chaos/cmd/terminate.go | 42 +++++++++++++++++++-------------------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/go-chaos/cmd/terminate.go b/go-chaos/cmd/terminate.go index 07cb12323..418701ad7 100644 --- a/go-chaos/cmd/terminate.go +++ b/go-chaos/cmd/terminate.go @@ -58,6 +58,27 @@ var terminateBrokerCmd = &cobra.Command{ }, } +var terminateGatewayCmd = &cobra.Command{ + Use: "gateway", + Short: "Terminates a Zeebe gateway", + Long: `Terminates a Zeebe gateway.`, + Run: func(cmd *cobra.Command, args []string) { + gracePeriodSec := int64(0) + gatewayPod := restartGateway(&gracePeriodSec) + fmt.Printf("Terminated %s\n", gatewayPod) + }, +} + +var terminateWorkerCmd = &cobra.Command{ + Use: "worker", + Short: "Terminates a Zeebe worker", + Long: `Terminates a Zeebe worker.`, + Run: func(cmd *cobra.Command, args []string) { + gracePeriodSec := int64(0) + restartWorker(all, "Terminated", &gracePeriodSec) + }, +} + // Restart a broker pod. Pod is identified either by nodeId or by partitionId and role. // GracePeriod (in second) can be negative, which would mean use default. // Returns the broker which has been restarted @@ -80,17 +101,6 @@ func restartBroker(nodeId int, partitionId int, role string, gracePeriod *int64) return brokerPod.Name } -var terminateGatewayCmd = &cobra.Command{ - Use: "gateway", - Short: "Terminates a Zeebe gateway", - Long: `Terminates a Zeebe gateway.`, - Run: func(cmd *cobra.Command, args []string) { - gracePeriodSec := int64(0) - gatewayPod := restartGateway(&gracePeriodSec) - fmt.Printf("Terminated %s\n", gatewayPod) - }, -} - // Restart a gateway pod. The pod is the first from a list of existing pods. // GracePeriod (in second) can be negative, which would mean use default. // Returns the gateway which has been restarted @@ -111,16 +121,6 @@ func restartGateway(gracePeriod *int64) string { return gatewayPod } -var terminateWorkerCmd = &cobra.Command{ - Use: "worker", - Short: "Terminates a Zeebe worker", - Long: `Terminates a Zeebe worker.`, - Run: func(cmd *cobra.Command, args []string) { - gracePeriodSec := int64(0) - restartWorker(all, "Terminated", &gracePeriodSec) - }, -} - // Restart a worker pod. The pod is the first from a list of existing pods, if all is not specified. // GracePeriod (in second) can be negative, which would mean use default. // The actionName specifies whether it was restarted or terminated to log the right thing. From 9ec6a965ec4993c3dc4e2a4879f3cb3695ab1ea8 Mon Sep 17 00:00:00 2001 From: Christopher Zell Date: Wed, 23 Nov 2022 10:02:15 +0100 Subject: [PATCH 5/7] docs: update docs --- go-chaos/cmd/terminate.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go-chaos/cmd/terminate.go b/go-chaos/cmd/terminate.go index 418701ad7..60b1e9d32 100644 --- a/go-chaos/cmd/terminate.go +++ b/go-chaos/cmd/terminate.go @@ -122,7 +122,7 @@ func restartGateway(gracePeriod *int64) string { } // Restart a worker pod. The pod is the first from a list of existing pods, if all is not specified. -// GracePeriod (in second) can be negative, which would mean use default. +// GracePeriod (in second) can be nil, which would mean using K8 default. // The actionName specifies whether it was restarted or terminated to log the right thing. func restartWorker(all bool, actionName string, gracePeriod *int64) { k8Client, err := internal.CreateK8Client() From 3115df74c9bcdfd841d19be27b39937465ed089e Mon Sep 17 00:00:00 2001 From: Christopher Zell Date: Wed, 23 Nov 2022 10:02:26 +0100 Subject: [PATCH 6/7] docs: update docs --- go-chaos/cmd/terminate.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go-chaos/cmd/terminate.go b/go-chaos/cmd/terminate.go index 60b1e9d32..c43564ee2 100644 --- a/go-chaos/cmd/terminate.go +++ b/go-chaos/cmd/terminate.go @@ -102,7 +102,7 @@ func restartBroker(nodeId int, partitionId int, role string, gracePeriod *int64) } // Restart a gateway pod. The pod is the first from a list of existing pods. -// GracePeriod (in second) can be negative, which would mean use default. +// GracePeriod (in second) can be nil, which would mean using K8 default. // Returns the gateway which has been restarted func restartGateway(gracePeriod *int64) string { k8Client, err := internal.CreateK8Client() From a500dfe166e4399e1e02022aa7a3a80ce5bd3c89 Mon Sep 17 00:00:00 2001 From: Christopher Zell Date: Wed, 23 Nov 2022 10:02:34 +0100 Subject: [PATCH 7/7] docs: update docs --- go-chaos/cmd/terminate.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go-chaos/cmd/terminate.go b/go-chaos/cmd/terminate.go index c43564ee2..2eb230064 100644 --- a/go-chaos/cmd/terminate.go +++ b/go-chaos/cmd/terminate.go @@ -80,7 +80,7 @@ var terminateWorkerCmd = &cobra.Command{ } // Restart a broker pod. Pod is identified either by nodeId or by partitionId and role. -// GracePeriod (in second) can be negative, which would mean use default. +// GracePeriod (in second) can be nil, which would mean using K8 default. // Returns the broker which has been restarted func restartBroker(nodeId int, partitionId int, role string, gracePeriod *int64) string { k8Client, err := internal.CreateK8Client()