From 7f291953e3006cfa676e01b2661b9677ef77be1c Mon Sep 17 00:00:00 2001
From: "Johannes M. Scheuermann" <jscheuermann@apple.com>
Date: Wed, 24 Jul 2024 15:45:46 +0200
Subject: [PATCH 1/3] Initial docs for coordinator recovery and e2e test

---
 api/v1beta2/foundationdb_process_address.go   |   2 +-
 api/v1beta2/foundationdbcluster_types.go      |   7 +-
 docs/cluster_spec.md                          |   2 +-
 docs/manual/operations.md                     |  49 ++++
 e2e/fixtures/factory.go                       |  78 +++++-
 e2e/fixtures/fdb_cluster.go                   |  15 +-
 e2e/fixtures/status.go                        |   2 +
 e2e/test_operator/operator_test.go            |   3 +
 e2e/test_operator_ha/operator_ha_test.go      | 257 ++++++++++++++----
 .../operator_upgrades_test.go                 |   3 +
 .../operator_upgrades_variations_test.go      |   3 +-
 go.mod                                        |   9 +
 go.sum                                        |  25 ++
 13 files changed, 380 insertions(+), 75 deletions(-)

diff --git a/api/v1beta2/foundationdb_process_address.go b/api/v1beta2/foundationdb_process_address.go
index 4d52bc6b1..c4979dde9 100644
--- a/api/v1beta2/foundationdb_process_address.go
+++ b/api/v1beta2/foundationdb_process_address.go
@@ -316,7 +316,7 @@ func GetFullAddressList(address string, primaryOnly bool, processNumber int, req
 		pAddr := NewProcessAddress(nil, address, GetProcessPort(processNumber, true), map[string]bool{"tls": true})
 		addrs = append(addrs, pAddr)
 
-		if requireTLS && primaryOnly {
+		if primaryOnly {
 			return addrs
 		}
 	}
diff --git a/api/v1beta2/foundationdbcluster_types.go b/api/v1beta2/foundationdbcluster_types.go
index dfa34b7f0..97f2ca38b 100644
--- a/api/v1beta2/foundationdbcluster_types.go
+++ b/api/v1beta2/foundationdbcluster_types.go
@@ -96,10 +96,9 @@ type FoundationDBClusterSpec struct {
 	// infer the process counts based on the database configuration.
 	ProcessCounts ProcessCounts `json:"processCounts,omitempty"`
 
-	// SeedConnectionString provides a connection string for the initial
-	// reconciliation.
-	//
-	// After the initial reconciliation, this will not be used.
+	// SeedConnectionString provides an additional connection string. This connection string will be used in addition
+	// to the connection string defined under cluster.Status.ConnectionString to connect to the cluster. This setting
+	// can be used to create a multi-region cluster or to recover a cluster if it is out of sync.
 	SeedConnectionString string `json:"seedConnectionString,omitempty"`
 
 	// PartialConnectionString provides a way to specify part of the
diff --git a/docs/cluster_spec.md b/docs/cluster_spec.md
index 7a6ec36e1..1996c545a 100644
--- a/docs/cluster_spec.md
+++ b/docs/cluster_spec.md
@@ -235,7 +235,7 @@ FoundationDBClusterSpec defines the desired state of a cluster.
 | databaseConfiguration | DatabaseConfiguration defines the database configuration. | [DatabaseConfiguration](#databaseconfiguration) | false |
 | processes | Processes defines process-level settings. | map[[ProcessClass](#processclass)][ProcessSettings](#processsettings) | false |
 | processCounts | ProcessCounts defines the number of processes to configure for each process class. You can generally omit this, to allow the operator to infer the process counts based on the database configuration. | [ProcessCounts](#processcounts) | false |
-| seedConnectionString | SeedConnectionString provides a connection string for the initial reconciliation.  After the initial reconciliation, this will not be used. | string | false |
+| seedConnectionString | SeedConnectionString provides an additional connection string. This connection string will be used in addition to the connection string defined under cluster.Status.ConnectionString to connect to the cluster. This setting can be used to create a multi-region cluster or to recover a cluster if it is out of sync. | string | false |
 | partialConnectionString | PartialConnectionString provides a way to specify part of the connection string (e.g. the database name and coordinator generation) without specifying the entire string. This does not allow for setting the coordinator IPs. If `SeedConnectionString` is set, `PartialConnectionString` will have no effect. They cannot be used together. | [ConnectionString](#connectionstring) | false |
 | faultDomain | FaultDomain defines the rules for what fault domain to replicate across. | [FoundationDBClusterFaultDomain](#foundationdbclusterfaultdomain) | false |
 | processGroupsToRemove | ProcessGroupsToRemove defines the process groups that we should remove from the cluster. This list contains the process group IDs. | [][ProcessGroupID](#processgroupid) | false |
diff --git a/docs/manual/operations.md b/docs/manual/operations.md
index 8d8a95de6..1d4b37ac6 100644
--- a/docs/manual/operations.md
+++ b/docs/manual/operations.md
@@ -212,6 +212,55 @@ There are a few risks and limitations to the current implementation:
 The current risks are limited to releasing the maintenance mode earlier than it should be.
 In this case data-movement will be triggered for the down processes after 60 seconds, the data-movement shouldn't cause any operational issues.
 
+## Recover Lost Quorum Of Coordinators
+
+If the coordinators Pods are still running but they got new IP addresses, read [Coordinators Getting New IPs](./debugging.md#coordinators-getting-new-ips).
+In case you lost the quorum of coordinators and you are not able to restore the coordinator state, you have two options:
+
+1. Recover the cluster from the latest backup.
+1. Recover the coordinator state with the risk of data loss.
+
+This section will describe the procedure for case 2.
+
+**NOTE** The assumption here is that at least one coordinator is still available with it's coordinator state.
+**NOTE** This action can cause data loss. Perform those actions with care.
+
+- Set all the `FoundationDBCluster` resources for this FDB cluster to `spec.Skip = true` to make sure the operator is not changing the manual changed state. 
+- Fetch the last connection string from the `FoundationDBCluster` status, e.g. `kubectl get fdb ${cluster} -o jsonpath='{ .status.connectionString }'`. 
+- Copy the coordinator state from one of the running coordinators to your local machine:
+
+```bash
+kubectl cp ${coordinator-pod}:/var/fdb/data/1/coordination-0.fdq ./coordination-0.fdq 
+kubectl cp ${coordinator-pod}:/var/fdb/data/1/coordination-1.fdq ./coordination-1.fdq
+```
+ 
+- Select a number of Pods you want to use as new coordinators and copy the files to those pods:
+
+```bash
+kubectl cp ./coordination-0.fdq ${new-coordinator-pod}:/var/fdb/data/1/coordination-0.fdq 
+kubectl cp ./coordination-1.fdq ${new-coordinator-pod}:/var/fdb/data/1/coordination-1.fdq 
+```
+
+- Update the `ConfigMap` to contain the new connection string, the new connection string must contain the still existing coordinators and the new coordinators. The old entries must be removed.
+- Wait ~1 min until the `ConfigMap` is synced to all Pods, you can check the `/var/dynamic-conf/fdb.cluster` inside a Pod if you are unsure.
+- Now all Pods must be restarted and the previous local cluster file must be deleted to make sure the fdbserver is picking the connection string from the seed cluster file (`/var/dynamic-conf/fdb.cluster`).
+
+```bash
+for pod in $(kubectl get po -l foundationdb.org/fdb-cluster-name=${cluster} -o name --no-headers);
+do
+    echo $pod
+    kubectl $pod -- bash -c 'pkill fdbserver && rm -f /var/fdb/data/fdb.cluster && pkill fdbserver'
+done
+```
+
+If the cluster is a multi-region cluster, perform this step for all running regions.
+
+- Now you can exec into a container and use `fdbcli` to connect to the cluster.
+- If you use a multi-region cluster you have to issue `force_recovery_with_data_loss`.
+- Update the `FoundationDBCluster` `seedConnectionString` under `spec.seedConnectionString` with the new connection string.
+- Now you can set `spec.Skip = false` to let the operator take over again.
+- Depending on the state of the multi-region cluster, you probably want to change the desired database configuration to drop ha.
+
 ## Next
 
 You can continue on to the [next section](scaling.md) or go back to the [table of contents](index.md).
diff --git a/e2e/fixtures/factory.go b/e2e/fixtures/factory.go
index 5ee074093..78311b2ac 100644
--- a/e2e/fixtures/factory.go
+++ b/e2e/fixtures/factory.go
@@ -24,6 +24,7 @@ import (
 	"bytes"
 	ctx "context"
 	"fmt"
+	"golang.org/x/net/context"
 	"io"
 	"log"
 	"math/rand"
@@ -441,16 +442,18 @@ type ClusterOption func(*Factory, *fdbv1beta2.FoundationDBCluster)
 
 // ExecuteCmdOnPod runs a command on the provided Pod. The command will be executed inside a bash -c ”.
 func (factory *Factory) ExecuteCmdOnPod(
+	ctx context.Context,
 	pod *corev1.Pod,
 	container string,
 	command string,
 	printOutput bool,
 ) (string, string, error) {
-	return factory.ExecuteCmd(pod.Namespace, pod.Name, container, command, printOutput)
+	return factory.ExecuteCmd(ctx, pod.Namespace, pod.Name, container, command, printOutput)
 }
 
 // ExecuteCmd executes command in the default container of a Pod with shell, returns stdout and stderr.
 func (factory *Factory) ExecuteCmd(
+	ctx context.Context,
 	namespace string,
 	name string,
 	container string,
@@ -464,7 +467,7 @@ func (factory *Factory) ExecuteCmd(
 	}
 	var stdout bytes.Buffer
 	var stderr bytes.Buffer
-	err := factory.ExecuteCommandRaw(namespace, name, container, cmd, nil, &stdout, &stderr, false)
+	err := factory.ExecuteCommandRaw(ctx, namespace, name, container, cmd, nil, &stdout, &stderr, false)
 	sout := stdout.String()
 	serr := stderr.String()
 	// TODO: Stream these to our own stdout as we run.
@@ -487,6 +490,7 @@ func (factory *Factory) ExecuteCmd(
 
 // ExecuteCommandRaw will run the command without putting it into a shell.
 func (factory *Factory) ExecuteCommandRaw(
+	ctx context.Context,
 	namespace string,
 	name string,
 	container string,
@@ -511,15 +515,73 @@ func (factory *Factory) ExecuteCommandRaw(
 		option,
 		scheme.ParameterCodec,
 	)
-	exec, err := remotecommand.NewSPDYExecutor(factory.getConfig(), "POST", req.URL())
+	spdyExec, err := remotecommand.NewSPDYExecutor(factory.getConfig(), "POST", req.URL())
 	if err != nil {
 		return err
 	}
-	return exec.Stream(remotecommand.StreamOptions{
-		Stdin:  stdin,
-		Stdout: stdout,
-		Stderr: stderr,
-	})
+	return spdyExec.StreamWithContext(ctx,
+		remotecommand.StreamOptions{
+			Stdin:  stdin,
+			Stdout: stdout,
+			Stderr: stderr,
+		})
+}
+
+// DownloadFile will download the file from the provided Pod/container into w.
+func (factory *Factory) DownloadFile(ctx context.Context, target *corev1.Pod, container string, src string, w io.Writer) error {
+	errOut := bytes.NewBuffer([]byte{})
+	reader, writer := io.Pipe()
+	defer func() {
+		log.Println("Done downloading file")
+		_ = writer.Close()
+	}()
+
+	// Copy the content from stdout of the container to the new file.
+	go func() {
+		defer func() {
+			_ = writer.Close()
+		}()
+		_, err := io.Copy(w, reader)
+		if err != nil {
+			log.Println("DownloadFile copy, err:", err)
+		}
+	}()
+
+	err := factory.ExecuteCommandRaw(ctx, target.Namespace, target.Name, container, []string{"/bin/cp", src, "/dev/stdout"}, nil, writer, errOut, false)
+	if err != nil {
+		log.Println(errOut.String())
+	}
+
+	return err
+}
+
+// UploadFile uploads a file from src into the Pod/container dst.
+func (factory *Factory) UploadFile(ctx context.Context, target *corev1.Pod, container string, src io.Reader, dst string) error {
+	out := bytes.NewBuffer([]byte{})
+	errOut := bytes.NewBuffer([]byte{})
+	reader, writer := io.Pipe()
+	defer func() {
+		log.Println("Done uploading file")
+		_ = reader.Close()
+	}()
+
+	// Read the file provided via src and pipe it to the reader.
+	go func(r io.Reader, writer *io.PipeWriter) {
+		defer func() {
+			_ = writer.Close()
+		}()
+		_, err := io.Copy(writer, r)
+		if err != nil {
+			log.Println("UploadFile copy, err:", err)
+		}
+	}(src, writer)
+
+	err := factory.ExecuteCommandRaw(ctx, target.Namespace, target.Name, container, []string{"tee", "-a", dst}, reader, out, errOut, false)
+	if err != nil {
+		log.Println(errOut.String())
+	}
+
+	return err
 }
 
 // GetLogsFromPod returns the logs for the provided Pod and container
diff --git a/e2e/fixtures/fdb_cluster.go b/e2e/fixtures/fdb_cluster.go
index 87ed11b91..9cb6a3ed9 100644
--- a/e2e/fixtures/fdb_cluster.go
+++ b/e2e/fixtures/fdb_cluster.go
@@ -23,6 +23,7 @@ package fixtures
 import (
 	ctx "context"
 	"fmt"
+	"golang.org/x/net/context"
 	"log"
 	"math"
 	"strconv"
@@ -79,7 +80,7 @@ func (fdbCluster *FdbCluster) ExecuteCmdOnPod(
 	command string,
 	printOutput bool,
 ) (string, string, error) {
-	return fdbCluster.factory.ExecuteCmd(pod.Namespace, pod.Name, container, command, printOutput)
+	return fdbCluster.factory.ExecuteCmd(context.Background(), pod.Namespace, pod.Name, container, command, printOutput)
 }
 
 func (factory *Factory) createFdbClusterObject(
@@ -1480,3 +1481,15 @@ func (fdbCluster *FdbCluster) GetListOfUIDsFromVolumeClaims(processClass fdbv1be
 
 	return uids
 }
+
+// UpdateConnectionString will update the connection string in the ConfigMap and the SeedConnectionString of the custer.
+func (fdbCluster *FdbCluster) UpdateConnectionString(connectionString string) {
+	fdbCluster.cluster.Spec.SeedConnectionString = connectionString
+	fdbCluster.UpdateClusterSpec()
+
+	cm := &corev1.ConfigMap{}
+	gomega.Expect(fdbCluster.factory.controllerRuntimeClient.Get(ctx.Background(), client.ObjectKey{Namespace: fdbCluster.Namespace(), Name: fdbCluster.Name() + "-config"}, cm)).NotTo(gomega.HaveOccurred())
+	gomega.Expect(cm.Data).To(gomega.HaveKey(fdbv1beta2.ClusterFileKey))
+	cm.Data[fdbv1beta2.ClusterFileKey] = connectionString
+	gomega.Expect(fdbCluster.factory.controllerRuntimeClient.Update(ctx.Background(), cm)).NotTo(gomega.HaveOccurred())
+}
diff --git a/e2e/fixtures/status.go b/e2e/fixtures/status.go
index f2d868029..8f7b6c557 100644
--- a/e2e/fixtures/status.go
+++ b/e2e/fixtures/status.go
@@ -22,6 +22,7 @@ package fixtures
 
 import (
 	"bytes"
+	"context"
 	"encoding/json"
 	"fmt"
 	"log"
@@ -133,6 +134,7 @@ func (fdbCluster *FdbCluster) RunFdbCliCommandInOperatorWithoutRetry(
 	// is returned) we will try the version for the spec. This should reduce some test flakiness.
 	for _, fdbCliPath := range fdbCliPaths {
 		stdout, stderr, err = fdbCluster.factory.ExecuteCmd(
+			context.Background(),
 			pod.Namespace,
 			pod.Name,
 			"manager",
diff --git a/e2e/test_operator/operator_test.go b/e2e/test_operator/operator_test.go
index 9a166e3fa..feafb54b3 100644
--- a/e2e/test_operator/operator_test.go
+++ b/e2e/test_operator/operator_test.go
@@ -865,6 +865,7 @@ var _ = Describe("Operator", Label("e2e", "pr"), func() {
 				// File creation should fail due to I/O error
 				Eventually(func() error {
 					_, _, err := factory.ExecuteCmdOnPod(
+						context.Background(),
 						&podWithIOError,
 						fdbv1beta2.MainContainerName,
 						"touch /var/fdb/data/test",
@@ -2378,6 +2379,7 @@ var _ = Describe("Operator", Label("e2e", "pr"), func() {
 			var kubernetesServiceHost string
 			Eventually(func(g Gomega) error {
 				std, _, err := factory.ExecuteCmdOnPod(
+					context.Background(),
 					&selectedPod,
 					fdbv1beta2.MainContainerName,
 					"printenv KUBERNETES_SERVICE_HOST",
@@ -2394,6 +2396,7 @@ var _ = Describe("Operator", Label("e2e", "pr"), func() {
 			// Make sure that the partition takes effect.
 			Eventually(func() error {
 				_, _, err := factory.ExecuteCmdOnPod(
+					context.Background(),
 					&selectedPod,
 					fdbv1beta2.MainContainerName,
 					fmt.Sprintf("nc -vz -w 2 %s 443", kubernetesServiceHost),
diff --git a/e2e/test_operator_ha/operator_ha_test.go b/e2e/test_operator_ha/operator_ha_test.go
index 177fb9a14..b4a2a1012 100644
--- a/e2e/test_operator_ha/operator_ha_test.go
+++ b/e2e/test_operator_ha/operator_ha_test.go
@@ -30,9 +30,11 @@ This cluster will be used for all tests.
 */
 
 import (
-	"encoding/json"
+	"context"
 	"fmt"
 	"log"
+	"os"
+	"path"
 	"strconv"
 	"strings"
 	"time"
@@ -50,9 +52,11 @@ import (
 )
 
 var (
-	factory     *fixtures.Factory
-	fdbCluster  *fixtures.HaFdbCluster
-	testOptions *fixtures.FactoryOptions
+	factory        *fixtures.Factory
+	fdbCluster     *fixtures.HaFdbCluster
+	testOptions    *fixtures.FactoryOptions
+	clusterConfig  *fixtures.ClusterConfig
+	clusterOptions []fixtures.ClusterOption
 )
 
 func init() {
@@ -61,10 +65,9 @@ func init() {
 
 var _ = BeforeSuite(func() {
 	factory = fixtures.CreateFactory(testOptions)
-	fdbCluster = factory.CreateFdbHaCluster(
-		fixtures.DefaultClusterConfigWithHaMode(fixtures.HaFourZoneSingleSat, false),
-		factory.GetClusterOptions()...,
-	)
+	clusterOptions = factory.GetClusterOptions()
+	clusterConfig = fixtures.DefaultClusterConfigWithHaMode(fixtures.HaFourZoneSingleSat, false)
+	fdbCluster = factory.CreateFdbHaCluster(clusterConfig, clusterOptions...)
 
 	// Load some data into the cluster.
 	factory.CreateDataLoaderIfAbsent(fdbCluster.GetPrimary())
@@ -218,24 +221,15 @@ var _ = Describe("Operator HA tests", Label("e2e", "pr"), func() {
 		})
 	})
 
-	// Right now this test case doesn't work as there is no way to recover when the majority of coordinators are failing.
-	PWhen("all Pods in the primary and primary satellite failing", func() {
+	When("all Pods in the primary and primary satellite are down", func() {
 		BeforeEach(func() {
+			// This tests is a destructive test where the cluster will stop working for some period.
+			availabilityCheck = false
 			primary := fdbCluster.GetPrimary()
-			primary.SetCrashLoopContainers([]fdbv1beta2.CrashLoopContainerObject{
-				{
-					ContainerName: fdbv1beta2.MainContainerName,
-					Targets:       []fdbv1beta2.ProcessGroupID{"*"},
-				},
-			}, false)
+			primary.SetSkipReconciliation(true)
 
 			primarySatellite := fdbCluster.GetPrimarySatellite()
-			primarySatellite.SetCrashLoopContainers([]fdbv1beta2.CrashLoopContainerObject{
-				{
-					ContainerName: fdbv1beta2.MainContainerName,
-					Targets:       []fdbv1beta2.ProcessGroupID{"*"},
-				},
-			}, false)
+			primarySatellite.SetSkipReconciliation(true)
 
 			primaryPods := primary.GetPods()
 			for _, pod := range primaryPods.Items {
@@ -247,63 +241,208 @@ var _ = Describe("Operator HA tests", Label("e2e", "pr"), func() {
 				factory.DeletePod(&pod)
 			}
 
-			remote := fdbCluster.GetRemote()
-			Eventually(func(g Gomega) bool {
-				out, _, err := remote.RunFdbCliCommandInOperatorWithoutRetry("status json", false, 30)
-				log.Println("Fetched status", out, err)
-				g.Expect(err).NotTo(HaveOccurred())
-
-				if strings.HasPrefix(out, "\r\nWARNING") {
-					out = strings.TrimPrefix(
-						out,
-						"\r\nWARNING: Long delay (Ctrl-C to interrupt)\r\n",
-					)
-				}
+			// Wait a short amount of time to let the cluster see that the primary and primary satellite is down.
+			time.Sleep(5 * time.Second)
+		})
 
-				status := &fdbv1beta2.FoundationDBStatus{}
-				err = json.Unmarshal([]byte(out), status)
-				g.Expect(err).NotTo(HaveOccurred())
-				return status.Client.DatabaseStatus.Available
-			}).WithTimeout(5 * time.Minute).WithPolling(1 * time.Second).MustPassRepeatedly(4).Should(BeFalse())
+		AfterEach(func() {
+			// Delete the broken cluster.
+			fdbCluster.Delete()
+			// Recreate the cluster to make sure  the next tests can proceed
+			fdbCluster = factory.CreateFdbHaCluster(clusterConfig, clusterOptions...)
+			// Load some data into the cluster.
+			factory.CreateDataLoaderIfAbsent(fdbCluster.GetPrimary())
 		})
 
-		It("should change the coordinators", func() {
-			newCoordinators := make([]fdbv1beta2.ProcessAddress, 0, 5)
+		It("should recover the coordinators", func() {
+			// Set all the `FoundationDBCluster` resources for this FDB cluster to `spec.Skip = true` to make sure the operator is not changing the manual changed state.
 			remote := fdbCluster.GetRemote()
-			remotePods := remote.GetLogPods()
+			remote.SetSkipReconciliation(true)
+			remoteSatellite := fdbCluster.GetRemoteSatellite()
+			remoteSatellite.SetSkipReconciliation(true)
 
+			// Fetch the last connection string from the `FoundationDBCluster` status, e.g. `kubectl get fdb ${cluster} -o jsonpath='{ .status.connectionString }'`.
+			lastConnectionString := remote.GetCluster().Status.ConnectionString
+			lastConnectionStringParts := strings.Split(lastConnectionString, "@")
+			addresses := strings.Split(lastConnectionStringParts[1], ",")
+			// Since this is a multi-region cluster, we expect 9 coordinators.
+			Expect(addresses).To(HaveLen(9))
+
+			log.Println("lastConnectionString", lastConnectionString)
+
+			var useTLS bool
+			coordinators := map[string]fdbv1beta2.ProcessAddress{}
+			for _, addr := range addresses {
+				parsed, err := fdbv1beta2.ParseProcessAddress(addr)
+				Expect(err).NotTo(HaveOccurred())
+				log.Println("found coordinator", parsed.String())
+				coordinators[parsed.MachineAddress()] = parsed
+				// If the tls flag is present we assume that the coordinators should make use of TLS.
+				_, useTLS = parsed.Flags["tls"]
+			}
+
+			log.Println("coordinators", coordinators, "useTLS", useTLS)
+			runningCoordinators := map[string]fdbv1beta2.None{}
+			var runningCoordinator *corev1.Pod
+			newCoordinators := make([]fdbv1beta2.ProcessAddress, 0, 5)
+			remotePods := remote.GetPods()
+			candidates := make([]corev1.Pod, 0, len(remotePods.Items))
 			for _, pod := range remotePods.Items {
-				if len(newCoordinators) >= 3 {
-					break
+				addr, err := fdbv1beta2.ParseProcessAddress(pod.Status.PodIP)
+				Expect(err).NotTo(HaveOccurred())
+				if coordinatorAddr, ok := coordinators[addr.MachineAddress()]; ok {
+					log.Println("Found coordinator for remote", pod.Name, "address", coordinatorAddr.String())
+					runningCoordinators[addr.MachineAddress()] = fdbv1beta2.None{}
+					newCoordinators = append(newCoordinators, coordinatorAddr)
+					if runningCoordinator == nil {
+						loopPod := pod
+						runningCoordinator = &loopPod
+					}
+					continue
+				}
+
+				if !fixtures.GetProcessClass(pod).IsTransaction() {
+					continue
 				}
-				addr, err := fdbv1beta2.ParseProcessAddress(fmt.Sprintf("%s:4500:tls", pod.Status.PodIP))
+
+				candidates = append(candidates, pod)
+			}
+
+			remoteSatellitePods := remoteSatellite.GetPods()
+			for _, pod := range remoteSatellitePods.Items {
+				addr, err := fdbv1beta2.ParseProcessAddress(pod.Status.PodIP)
 				Expect(err).NotTo(HaveOccurred())
+				if coordinatorAddr, ok := coordinators[addr.MachineAddress()]; ok {
+					log.Println("Found coordinator for remote satellite", pod.Name, "address", addr.MachineAddress())
+					runningCoordinators[addr.MachineAddress()] = fdbv1beta2.None{}
+					newCoordinators = append(newCoordinators, coordinatorAddr)
+					if runningCoordinator == nil {
+						loopPod := pod
+						runningCoordinator = &loopPod
+					}
+					continue
+				}
+
+				if !fixtures.GetProcessClass(pod).IsTransaction() {
+					continue
+				}
+
+				candidates = append(candidates, pod)
+			}
+
+			// Pick 5 new coordinators.
+			needsUpload := make([]corev1.Pod, 0, 5)
+			idx := 0
+			for len(newCoordinators) < 5 {
+				fmt.Println("Current coordinators:", len(newCoordinators))
+				candidate := candidates[idx]
+				addr, err := fdbv1beta2.ParseProcessAddress(candidate.Status.PodIP)
+				Expect(err).NotTo(HaveOccurred())
+				fmt.Println("Adding pod as new coordinators:", candidate.Name)
+				if useTLS {
+					addr.Port = 4500
+					addr.Flags = map[string]bool{"tls": true}
+				} else {
+					addr.Port = 4501
+				}
 				newCoordinators = append(newCoordinators, addr)
+				needsUpload = append(needsUpload, candidate)
+				idx++
 			}
 
-			remoteSatellite := fdbCluster.GetRemoteSatellite()
-			remoteSatellitePods := remoteSatellite.GetLogPods()
+			// Copy the coordinator state from one of the running coordinators to your local machine:
+			coordinatorFiles := []string{"coordination-0.fdq", "coordination-1.fdq"}
+			tmpCoordinatorFiles := make([]string, 2)
+			tmpDir := GinkgoT().TempDir()
+			for idx, coordinatorFile := range coordinatorFiles {
+				tmpCoordinatorFiles[idx] = path.Join(tmpDir, coordinatorFile)
+			}
 
-			for _, pod := range remoteSatellitePods.Items {
-				if len(newCoordinators) >= 5 {
+			log.Println("tmpCoordinatorFiles", tmpCoordinatorFiles)
+			stdout, stderr, err := factory.ExecuteCmdOnPod(context.Background(), runningCoordinator, fdbv1beta2.MainContainerName, "find /var/fdb/data/ -type f -name 'coordination-0.fdq'", true)
+			Expect(err).NotTo(HaveOccurred())
+			Expect(stderr).To(BeEmpty())
+
+			dataDir := path.Dir(strings.TrimSpace(stdout))
+			log.Println("find result:", stdout, ",dataDir", dataDir)
+			for idx, coordinatorFile := range coordinatorFiles {
+				tmpCoordinatorFile, err := os.OpenFile(tmpCoordinatorFiles[idx], os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0600)
+				Expect(err).NotTo(HaveOccurred())
+
+				log.Println("Download files, target:", tmpCoordinatorFiles[idx], "source", path.Join(dataDir, coordinatorFile), "pod", runningCoordinator.Name, "namespace", runningCoordinator.Namespace)
+				err = factory.DownloadFile(context.Background(), runningCoordinator, fdbv1beta2.MainContainerName, path.Join(dataDir, coordinatorFile), tmpCoordinatorFile)
+				Expect(err).NotTo(HaveOccurred())
+				Expect(tmpCoordinatorFile.Close()).NotTo(HaveOccurred())
+
+				fileInfo, err := os.Stat(tmpCoordinatorFiles[idx])
+				Expect(err).NotTo(HaveOccurred())
+				Expect(fileInfo.Size()).To(BeNumerically(">", 0))
+			}
+
+			for _, target := range needsUpload {
+				for idx, coordinatorFile := range coordinatorFiles {
+					tmpCoordinatorFile, err := os.OpenFile(tmpCoordinatorFiles[idx], os.O_RDONLY, 0600)
+					Expect(err).NotTo(HaveOccurred())
+
+					log.Println("Upload files, source:", tmpCoordinatorFile.Name(), "target", path.Join(dataDir, coordinatorFile), "pod", target.Name, "namespace", target.Namespace)
+					err = factory.UploadFile(context.Background(), &target, fdbv1beta2.MainContainerName, tmpCoordinatorFile, path.Join(dataDir, coordinatorFile))
+					Expect(err).NotTo(HaveOccurred())
+					Expect(tmpCoordinatorFile.Close()).NotTo(HaveOccurred())
+				}
+			}
+
+			// Update the `ConfigMap` to contain the new connection string, the new connection string must contain the still existing coordinators and the new coordinators. The old entries must be removed.
+			var newConnectionString strings.Builder
+			newConnectionString.WriteString(lastConnectionStringParts[0])
+			newConnectionString.WriteString("@")
+			for idx, coordinator := range newCoordinators {
+				newConnectionString.WriteString(coordinator.String())
+				if idx == len(newCoordinators)-1 {
 					break
 				}
-				addr, err := fdbv1beta2.ParseProcessAddress(fmt.Sprintf("%s:4500:tls", pod.Status.PodIP))
+
+				newConnectionString.WriteString(",")
+			}
+
+			newCS := newConnectionString.String()
+			log.Println("new connection string:", newCS)
+			for _, cluster := range fdbCluster.GetAllClusters() {
+				cluster.UpdateConnectionString(newCS)
+			}
+
+			// Wait ~1 min until the `ConfigMap` is synced to all Pods, you can check the `/var/dynamic-conf/fdb.cluster` inside a Pod if you are unsure.
+			time.Sleep(2 * time.Minute)
+
+			log.Println("Kill fdbserver processes")
+
+			debugOutput := true
+			// Now all Pods must be restarted and the previous local cluster file must be deleted to make sure the fdbserver is picking the connection string from the seed cluster file (`/var/dynamic-conf/fdb.cluster`).
+			for _, pod := range remote.GetPods().Items {
+				_, _, err := factory.ExecuteCmd(context.Background(), pod.Namespace, pod.Name, fdbv1beta2.MainContainerName, "pkill fdbserver && rm -f /var/fdb/data/fdb.cluster && pkill fdbserver || true", debugOutput)
 				Expect(err).NotTo(HaveOccurred())
-				newCoordinators = append(newCoordinators, addr)
 			}
 
-			coordinatorCmd := fmt.Sprintf(
-				"coordinators %s",
-				fdbv1beta2.ProcessAddressesString(newCoordinators, " "),
-			)
-			log.Println("Command:", coordinatorCmd)
+			for _, pod := range remoteSatellite.GetPods().Items {
+				_, _, err := factory.ExecuteCmd(context.Background(), pod.Namespace, pod.Name, fdbv1beta2.MainContainerName, "pkill fdbserver && rm -f /var/fdb/data/fdb.cluster && pkill fdbserver || true", debugOutput)
+				Expect(err).NotTo(HaveOccurred())
+			}
 
-			stdout, stderr, err := remote.RunFdbCliCommandInOperatorWithoutRetry(coordinatorCmd, true, 40)
-			log.Println("stdout:\n", stdout, "\nstderr:\n", stderr)
+			log.Println("force recovery")
+			// Now you can exec into a container and use `fdbcli` to connect to the cluster.
+			// If you use a multi-region cluster you have to issue `force_recovery_with_data_loss`
+			_, _, err = remote.RunFdbCliCommandInOperatorWithoutRetry(fmt.Sprintf("force_recovery_with_data_loss %s", remote.GetCluster().Spec.DataCenter), true, 40)
 			Expect(err).NotTo(HaveOccurred())
 
-			// TODO (johscheuer): Update the SeedConnectionString for all clusters.
+			// Now you can set `spec.Skip = false` to let the operator take over again.
+			remote.SetSkipReconciliation(false)
+			remoteSatellite.SetSkipReconciliation(false)
+
+			// Ensure the cluster is available again.
+			Eventually(func() bool {
+				return remote.GetStatus().Client.DatabaseStatus.Available
+			}).WithTimeout(2 * time.Minute).WithPolling(1 * time.Second).Should(BeTrue())
+
+			// TODO (johscheuer): Add additional testing for different cases.
 		})
 	})
 })
diff --git a/e2e/test_operator_upgrades/operator_upgrades_test.go b/e2e/test_operator_upgrades/operator_upgrades_test.go
index e18d8b1e0..e333d94a7 100644
--- a/e2e/test_operator_upgrades/operator_upgrades_test.go
+++ b/e2e/test_operator_upgrades/operator_upgrades_test.go
@@ -28,6 +28,7 @@ Since FoundationDB is version incompatible for major and minor versions and the
 */
 
 import (
+	"context"
 	"fmt"
 	"log"
 	"strings"
@@ -670,6 +671,7 @@ var _ = Describe("Operator Upgrades", Label("e2e", "pr"), func() {
 			var kubernetesServiceHost string
 			Eventually(func(g Gomega) error {
 				std, _, err := factory.ExecuteCmdOnPod(
+					context.Background(),
 					&selectedPod,
 					fdbv1beta2.MainContainerName,
 					"printenv KUBERNETES_SERVICE_HOST",
@@ -686,6 +688,7 @@ var _ = Describe("Operator Upgrades", Label("e2e", "pr"), func() {
 			// Make sure that the partition takes effect.
 			Eventually(func() error {
 				_, _, err := factory.ExecuteCmdOnPod(
+					context.Background(),
 					&selectedPod,
 					fdbv1beta2.MainContainerName,
 					fmt.Sprintf("nc -vz -w 2 %s 443", kubernetesServiceHost),
diff --git a/e2e/test_operator_upgrades_variations/operator_upgrades_variations_test.go b/e2e/test_operator_upgrades_variations/operator_upgrades_variations_test.go
index fb2cac2d0..99620003e 100644
--- a/e2e/test_operator_upgrades_variations/operator_upgrades_variations_test.go
+++ b/e2e/test_operator_upgrades_variations/operator_upgrades_variations_test.go
@@ -27,6 +27,7 @@ Each test will create a new FoundationDB cluster which will be upgraded.
 */
 
 import (
+	"context"
 	"fmt"
 	"log"
 	"time"
@@ -117,7 +118,7 @@ func performUpgrade(config testConfig, preUpgradeFunction func(cluster *fixtures
 			// If the Pod is missing check if the fdbserver processes are running and check the logs of the fdb-kubernetes-monitor.
 			if missingTime != nil && time.Since(time.Unix(*missingTime, 0)) > 60*time.Second {
 				log.Println("Missing process for:", processGroup.ProcessGroupID)
-				stdout, stderr, err := factory.ExecuteCmd(cluster.Namespace, processGroup.GetPodName(cluster), fdbv1beta2.MainContainerName, "ps aufx", true)
+				stdout, stderr, err := factory.ExecuteCmd(context.Background(), cluster.Namespace, processGroup.GetPodName(cluster), fdbv1beta2.MainContainerName, "ps aufx", true)
 				log.Println("stdout:", stdout, "stderr", stderr, "err", err)
 
 				pod, err := factory.GetPod(cluster.Namespace, processGroup.GetPodName(cluster))
diff --git a/go.mod b/go.mod
index 0464a4442..8ccfbb325 100644
--- a/go.mod
+++ b/go.mod
@@ -33,15 +33,21 @@ require (
 require golang.org/x/sync v0.3.0
 
 require (
+	github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 // indirect
+	github.com/MakeNowJust/heredoc v1.0.0 // indirect
 	github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/cespare/xxhash/v2 v2.2.0 // indirect
+	github.com/chai2010/gettext-go v1.0.2 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/docker/go-units v0.5.0 // indirect
 	github.com/emicklei/go-restful/v3 v3.9.0 // indirect
 	github.com/evanphx/json-patch v4.12.0+incompatible // indirect
 	github.com/evanphx/json-patch/v5 v5.6.0 // indirect
+	github.com/exponent-io/jsonpath v0.0.0-20151013193312-d6023ce2651d // indirect
+	github.com/fatih/camelcase v1.0.0 // indirect
 	github.com/fsnotify/fsnotify v1.6.0 // indirect
+	github.com/fvbommel/sortorder v1.0.1 // indirect
 	github.com/go-errors/errors v1.0.1 // indirect
 	github.com/go-logr/zapr v1.2.4 // indirect
 	github.com/go-openapi/jsonpointer v0.19.5 // indirect
@@ -70,8 +76,10 @@ require (
 	github.com/mattn/go-colorable v0.1.13 // indirect
 	github.com/mattn/go-isatty v0.0.20 // indirect
 	github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
+	github.com/mitchellh/go-wordwrap v1.0.0 // indirect
 	github.com/mitchellh/mapstructure v1.5.0 // indirect
 	github.com/moby/spdystream v0.2.0 // indirect
+	github.com/moby/term v0.0.0-20220808134915-39b0c02b01ae // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.2 // indirect
 	github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00 // indirect
@@ -82,6 +90,7 @@ require (
 	github.com/prometheus/client_model v0.3.0 // indirect
 	github.com/prometheus/procfs v0.10.1 // indirect
 	github.com/robfig/cron/v3 v3.0.1 // indirect
+	github.com/russross/blackfriday/v2 v2.1.0 // indirect
 	github.com/spf13/afero v1.9.3 // indirect
 	github.com/spf13/cast v1.5.0 // indirect
 	github.com/spf13/jwalterweatherman v1.1.0 // indirect
diff --git a/go.sum b/go.sum
index b78b7bd8f..9b45d4d96 100644
--- a/go.sum
+++ b/go.sum
@@ -36,8 +36,12 @@ cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RX
 cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0=
 cloud.google.com/go/storage v1.14.0/go.mod h1:GrKmX003DSIwi9o29oFT7YDnHYwZoctc3fOKtUw0Xmo=
 dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
+github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 h1:UQHMgLO+TxOElx5B5HZ4hJQsoJ/PvUvKRhJHDQXO8P8=
+github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
 github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
+github.com/MakeNowJust/heredoc v1.0.0 h1:cXCdzVdstXyiTqTvfqk9SDHpKNjxuom+DOlyEeQ4pzQ=
+github.com/MakeNowJust/heredoc v1.0.0/go.mod h1:mG5amYoWBHf8vpLOuehzbGGw0EHxpZZ6lCpQ4fNJ8LE=
 github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 h1:s6gZFSlWYmbqAuRjVTiNNhvNRfY2Wxp9nhfyel4rklc=
 github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137/go.mod h1:OMCwj8VM1Kc9e19TLln2VL61YJF0x1XFtfdL4JdbSyE=
 github.com/apple/foundationdb/bindings/go v0.0.0-20231020161252-ed27c828ca16 h1:+FSWyyQT1jj2LHHRsIWtTqWcJRmKSJP3IYjOb0NMutU=
@@ -56,6 +60,8 @@ github.com/bxcodec/faker v2.0.1+incompatible/go.mod h1:BNzfpVdTwnFJ6GtfYTcQu6l6r
 github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
 github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44=
 github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
+github.com/chai2010/gettext-go v1.0.2 h1:1Lwwip6Q2QGsAdl/ZKPCwTe9fe0CjlUbqj5bFNSjIRk=
+github.com/chai2010/gettext-go v1.0.2/go.mod h1:y+wnP2cHYaVj19NZhYKAwEMH2CI1gNHeQQ+5AjwawxA=
 github.com/chaos-mesh/chaos-mesh/api v0.0.0-20230613082117-03097981f627 h1:8rxLFVwOT1YX53H8RQA9LfMyMCSoQE/YZOB3Lzhqhzk=
 github.com/chaos-mesh/chaos-mesh/api v0.0.0-20230613082117-03097981f627/go.mod h1:5qllHIhMkPEWjIimDum42JtMj0P1Tn9x91XUceuPNjY=
 github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
@@ -67,6 +73,8 @@ github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnht
 github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
 github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
 github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
+github.com/creack/pty v1.1.11 h1:07n33Z8lZxZ2qwegKbObQohDhXDQxiMMz1NOUGYlesw=
+github.com/creack/pty v1.1.11/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@@ -88,12 +96,18 @@ github.com/evanphx/json-patch v4.12.0+incompatible h1:4onqiflcdA9EOZ4RxV643DvftH
 github.com/evanphx/json-patch v4.12.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk=
 github.com/evanphx/json-patch/v5 v5.6.0 h1:b91NhWfaz02IuVxO9faSllyAtNXHMPkC5J8sJCLunww=
 github.com/evanphx/json-patch/v5 v5.6.0/go.mod h1:G79N1coSVB93tBe7j6PhzjmR3/2VvlbKOFpnXhI9Bw4=
+github.com/exponent-io/jsonpath v0.0.0-20151013193312-d6023ce2651d h1:105gxyaGwCFad8crR9dcMQWvV9Hvulu6hwUh4tWPJnM=
+github.com/exponent-io/jsonpath v0.0.0-20151013193312-d6023ce2651d/go.mod h1:ZZMPRZwes7CROmyNKgQzC3XPs6L/G2EJLHddWejkmf4=
+github.com/fatih/camelcase v1.0.0 h1:hxNvNX/xYBp0ovncs8WyWZrOrpBNub/JfaMvbURyft8=
+github.com/fatih/camelcase v1.0.0/go.mod h1:yN2Sb0lFhZJUdVvtELVWefmrXpuZESvPmqwoZc+/fpc=
 github.com/fatih/color v1.16.0 h1:zmkK9Ngbjj+K0yRhTVONQh1p/HknKYSlNT+vZCzyokM=
 github.com/fatih/color v1.16.0/go.mod h1:fL2Sau1YI5c0pdGEVCbKQbLXB6edEj1ZgiY4NijnWvE=
 github.com/frankban/quicktest v1.14.3 h1:FJKSZTDHjyhriyC81FLQ0LY93eSai0ZyR/ZIkd3ZUKE=
 github.com/frankban/quicktest v1.14.3/go.mod h1:mgiwOwqx65TmIk1wJ6Q7wvnVMocbUorkibMOrVTHZps=
 github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY=
 github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw=
+github.com/fvbommel/sortorder v1.0.1 h1:dSnXLt4mJYH25uDDGa3biZNQsozaUWDSWeKJ0qqFfzE=
+github.com/fvbommel/sortorder v1.0.1/go.mod h1:uk88iVf1ovNn1iLfgUVU2F9o5eO30ui720w+kxuqRs0=
 github.com/go-errors/errors v1.0.1 h1:LUHzmkK3GUKUrL/1gfBUxAHzcev3apQlezX/+O7ma6w=
 github.com/go-errors/errors v1.0.1/go.mod h1:f4zRHt4oKfwPJE5k8C9vpYG+aDHdBFUsgrm6/TyX73Q=
 github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
@@ -245,10 +259,14 @@ github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWE
 github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
 github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo=
 github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4=
+github.com/mitchellh/go-wordwrap v1.0.0 h1:6GlHJ/LTGMrIJbwgdqdl2eEH8o+Exx/0m8ir9Gns0u4=
+github.com/mitchellh/go-wordwrap v1.0.0/go.mod h1:ZXFpozHsX6DPmq2I0TCekCxypsnAUbP2oI0UX1GXzOo=
 github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY=
 github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
 github.com/moby/spdystream v0.2.0 h1:cjW1zVyyoiM0T7b6UoySUFqzXMoqRckQtXwGPiBhOM8=
 github.com/moby/spdystream v0.2.0/go.mod h1:f7i0iNDQJ059oMTcWxx8MA/zKFIuD/lY+0GqbN2Wy8c=
+github.com/moby/term v0.0.0-20220808134915-39b0c02b01ae h1:O4SWKdcHVCvYqyDV+9CJA1fcDN2L11Bule0iFy3YlAI=
+github.com/moby/term v0.0.0-20220808134915-39b0c02b01ae/go.mod h1:E2VnQOmVuvZB6UYnnDB0qG5Nq/1tD9acaOpo6xmt0Kw=
 github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
 github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
 github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
@@ -287,6 +305,7 @@ github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzG
 github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
 github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8=
 github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4=
+github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk=
 github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
 github.com/sergi/go-diff v1.1.0 h1:we8PVUC3FE2uYfodKH/nBHMSetSfHDR6scGdBi+erh0=
 github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM=
@@ -298,6 +317,7 @@ github.com/spf13/cobra v1.6.1 h1:o94oiPyS4KD1mPy2fmcYYHHfCxLqYjJOhGsCHFZtEzA=
 github.com/spf13/cobra v1.6.1/go.mod h1:IOw/AERYS7UzyrGinqmz6HLUo219MORXGxhbaJUqzrY=
 github.com/spf13/jwalterweatherman v1.1.0 h1:ue6voC5bR5F8YxI5S67j9i582FU4Qvo2bmqnqMYADFk=
 github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo=
+github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
 github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
 github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
 github.com/spf13/viper v1.15.0 h1:js3yy885G8xwJa6iOISGFwd+qlUo5AvyXb7CiihdtiU=
@@ -482,6 +502,7 @@ golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20210423185535-09eb48e85fd7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
@@ -514,6 +535,7 @@ golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBn
 golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
 golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
 golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
+golang.org/x/tools v0.0.0-20190624222133-a101b041ded4/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
 golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
 golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
@@ -677,6 +699,9 @@ gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C
 gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk=
+gotest.tools/v3 v3.0.3 h1:4AuOwCGf4lLR9u3YOe2awrHygurzhO/HeQ6laiA6Sx0=
+gotest.tools/v3 v3.0.3/go.mod h1:Z7Lb0S5l+klDB31fvDQX8ss/FlKDxtlFlw3Oa8Ymbl8=
 honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=

From 9779d9719245113b94e2ef9a27360b3af3d156c0 Mon Sep 17 00:00:00 2001
From: "Johannes M. Scheuermann" <jscheuermann@apple.com>
Date: Mon, 2 Sep 2024 16:02:43 +0200
Subject: [PATCH 2/3] Remove bad imports

---
 go.mod | 15 ++++-----------
 go.sum | 25 -------------------------
 2 files changed, 4 insertions(+), 36 deletions(-)

diff --git a/go.mod b/go.mod
index 8ccfbb325..ccb843b4c 100644
--- a/go.mod
+++ b/go.mod
@@ -30,24 +30,21 @@ require (
 	sigs.k8s.io/yaml v1.3.0
 )
 
-require golang.org/x/sync v0.3.0
+require (
+	golang.org/x/net v0.23.0
+	golang.org/x/sync v0.3.0
+)
 
 require (
-	github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 // indirect
-	github.com/MakeNowJust/heredoc v1.0.0 // indirect
 	github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/cespare/xxhash/v2 v2.2.0 // indirect
-	github.com/chai2010/gettext-go v1.0.2 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/docker/go-units v0.5.0 // indirect
 	github.com/emicklei/go-restful/v3 v3.9.0 // indirect
 	github.com/evanphx/json-patch v4.12.0+incompatible // indirect
 	github.com/evanphx/json-patch/v5 v5.6.0 // indirect
-	github.com/exponent-io/jsonpath v0.0.0-20151013193312-d6023ce2651d // indirect
-	github.com/fatih/camelcase v1.0.0 // indirect
 	github.com/fsnotify/fsnotify v1.6.0 // indirect
-	github.com/fvbommel/sortorder v1.0.1 // indirect
 	github.com/go-errors/errors v1.0.1 // indirect
 	github.com/go-logr/zapr v1.2.4 // indirect
 	github.com/go-openapi/jsonpointer v0.19.5 // indirect
@@ -76,10 +73,8 @@ require (
 	github.com/mattn/go-colorable v0.1.13 // indirect
 	github.com/mattn/go-isatty v0.0.20 // indirect
 	github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
-	github.com/mitchellh/go-wordwrap v1.0.0 // indirect
 	github.com/mitchellh/mapstructure v1.5.0 // indirect
 	github.com/moby/spdystream v0.2.0 // indirect
-	github.com/moby/term v0.0.0-20220808134915-39b0c02b01ae // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.2 // indirect
 	github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00 // indirect
@@ -90,7 +85,6 @@ require (
 	github.com/prometheus/client_model v0.3.0 // indirect
 	github.com/prometheus/procfs v0.10.1 // indirect
 	github.com/robfig/cron/v3 v3.0.1 // indirect
-	github.com/russross/blackfriday/v2 v2.1.0 // indirect
 	github.com/spf13/afero v1.9.3 // indirect
 	github.com/spf13/cast v1.5.0 // indirect
 	github.com/spf13/jwalterweatherman v1.1.0 // indirect
@@ -99,7 +93,6 @@ require (
 	go.starlark.net v0.0.0-20200306205701-8dd3e2ee1dd5 // indirect
 	go.uber.org/multierr v1.10.0 // indirect
 	go.uber.org/zap v1.25.0 // indirect
-	golang.org/x/net v0.23.0 // indirect
 	golang.org/x/oauth2 v0.5.0 // indirect
 	golang.org/x/sys v0.20.0 // indirect
 	golang.org/x/term v0.18.0 // indirect
diff --git a/go.sum b/go.sum
index 9b45d4d96..b78b7bd8f 100644
--- a/go.sum
+++ b/go.sum
@@ -36,12 +36,8 @@ cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RX
 cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0=
 cloud.google.com/go/storage v1.14.0/go.mod h1:GrKmX003DSIwi9o29oFT7YDnHYwZoctc3fOKtUw0Xmo=
 dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
-github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 h1:UQHMgLO+TxOElx5B5HZ4hJQsoJ/PvUvKRhJHDQXO8P8=
-github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
 github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
-github.com/MakeNowJust/heredoc v1.0.0 h1:cXCdzVdstXyiTqTvfqk9SDHpKNjxuom+DOlyEeQ4pzQ=
-github.com/MakeNowJust/heredoc v1.0.0/go.mod h1:mG5amYoWBHf8vpLOuehzbGGw0EHxpZZ6lCpQ4fNJ8LE=
 github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 h1:s6gZFSlWYmbqAuRjVTiNNhvNRfY2Wxp9nhfyel4rklc=
 github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137/go.mod h1:OMCwj8VM1Kc9e19TLln2VL61YJF0x1XFtfdL4JdbSyE=
 github.com/apple/foundationdb/bindings/go v0.0.0-20231020161252-ed27c828ca16 h1:+FSWyyQT1jj2LHHRsIWtTqWcJRmKSJP3IYjOb0NMutU=
@@ -60,8 +56,6 @@ github.com/bxcodec/faker v2.0.1+incompatible/go.mod h1:BNzfpVdTwnFJ6GtfYTcQu6l6r
 github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
 github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44=
 github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
-github.com/chai2010/gettext-go v1.0.2 h1:1Lwwip6Q2QGsAdl/ZKPCwTe9fe0CjlUbqj5bFNSjIRk=
-github.com/chai2010/gettext-go v1.0.2/go.mod h1:y+wnP2cHYaVj19NZhYKAwEMH2CI1gNHeQQ+5AjwawxA=
 github.com/chaos-mesh/chaos-mesh/api v0.0.0-20230613082117-03097981f627 h1:8rxLFVwOT1YX53H8RQA9LfMyMCSoQE/YZOB3Lzhqhzk=
 github.com/chaos-mesh/chaos-mesh/api v0.0.0-20230613082117-03097981f627/go.mod h1:5qllHIhMkPEWjIimDum42JtMj0P1Tn9x91XUceuPNjY=
 github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
@@ -73,8 +67,6 @@ github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnht
 github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
 github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
 github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
-github.com/creack/pty v1.1.11 h1:07n33Z8lZxZ2qwegKbObQohDhXDQxiMMz1NOUGYlesw=
-github.com/creack/pty v1.1.11/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@@ -96,18 +88,12 @@ github.com/evanphx/json-patch v4.12.0+incompatible h1:4onqiflcdA9EOZ4RxV643DvftH
 github.com/evanphx/json-patch v4.12.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk=
 github.com/evanphx/json-patch/v5 v5.6.0 h1:b91NhWfaz02IuVxO9faSllyAtNXHMPkC5J8sJCLunww=
 github.com/evanphx/json-patch/v5 v5.6.0/go.mod h1:G79N1coSVB93tBe7j6PhzjmR3/2VvlbKOFpnXhI9Bw4=
-github.com/exponent-io/jsonpath v0.0.0-20151013193312-d6023ce2651d h1:105gxyaGwCFad8crR9dcMQWvV9Hvulu6hwUh4tWPJnM=
-github.com/exponent-io/jsonpath v0.0.0-20151013193312-d6023ce2651d/go.mod h1:ZZMPRZwes7CROmyNKgQzC3XPs6L/G2EJLHddWejkmf4=
-github.com/fatih/camelcase v1.0.0 h1:hxNvNX/xYBp0ovncs8WyWZrOrpBNub/JfaMvbURyft8=
-github.com/fatih/camelcase v1.0.0/go.mod h1:yN2Sb0lFhZJUdVvtELVWefmrXpuZESvPmqwoZc+/fpc=
 github.com/fatih/color v1.16.0 h1:zmkK9Ngbjj+K0yRhTVONQh1p/HknKYSlNT+vZCzyokM=
 github.com/fatih/color v1.16.0/go.mod h1:fL2Sau1YI5c0pdGEVCbKQbLXB6edEj1ZgiY4NijnWvE=
 github.com/frankban/quicktest v1.14.3 h1:FJKSZTDHjyhriyC81FLQ0LY93eSai0ZyR/ZIkd3ZUKE=
 github.com/frankban/quicktest v1.14.3/go.mod h1:mgiwOwqx65TmIk1wJ6Q7wvnVMocbUorkibMOrVTHZps=
 github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY=
 github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw=
-github.com/fvbommel/sortorder v1.0.1 h1:dSnXLt4mJYH25uDDGa3biZNQsozaUWDSWeKJ0qqFfzE=
-github.com/fvbommel/sortorder v1.0.1/go.mod h1:uk88iVf1ovNn1iLfgUVU2F9o5eO30ui720w+kxuqRs0=
 github.com/go-errors/errors v1.0.1 h1:LUHzmkK3GUKUrL/1gfBUxAHzcev3apQlezX/+O7ma6w=
 github.com/go-errors/errors v1.0.1/go.mod h1:f4zRHt4oKfwPJE5k8C9vpYG+aDHdBFUsgrm6/TyX73Q=
 github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
@@ -259,14 +245,10 @@ github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWE
 github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
 github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo=
 github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4=
-github.com/mitchellh/go-wordwrap v1.0.0 h1:6GlHJ/LTGMrIJbwgdqdl2eEH8o+Exx/0m8ir9Gns0u4=
-github.com/mitchellh/go-wordwrap v1.0.0/go.mod h1:ZXFpozHsX6DPmq2I0TCekCxypsnAUbP2oI0UX1GXzOo=
 github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY=
 github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
 github.com/moby/spdystream v0.2.0 h1:cjW1zVyyoiM0T7b6UoySUFqzXMoqRckQtXwGPiBhOM8=
 github.com/moby/spdystream v0.2.0/go.mod h1:f7i0iNDQJ059oMTcWxx8MA/zKFIuD/lY+0GqbN2Wy8c=
-github.com/moby/term v0.0.0-20220808134915-39b0c02b01ae h1:O4SWKdcHVCvYqyDV+9CJA1fcDN2L11Bule0iFy3YlAI=
-github.com/moby/term v0.0.0-20220808134915-39b0c02b01ae/go.mod h1:E2VnQOmVuvZB6UYnnDB0qG5Nq/1tD9acaOpo6xmt0Kw=
 github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
 github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
 github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
@@ -305,7 +287,6 @@ github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzG
 github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
 github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8=
 github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4=
-github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk=
 github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
 github.com/sergi/go-diff v1.1.0 h1:we8PVUC3FE2uYfodKH/nBHMSetSfHDR6scGdBi+erh0=
 github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM=
@@ -317,7 +298,6 @@ github.com/spf13/cobra v1.6.1 h1:o94oiPyS4KD1mPy2fmcYYHHfCxLqYjJOhGsCHFZtEzA=
 github.com/spf13/cobra v1.6.1/go.mod h1:IOw/AERYS7UzyrGinqmz6HLUo219MORXGxhbaJUqzrY=
 github.com/spf13/jwalterweatherman v1.1.0 h1:ue6voC5bR5F8YxI5S67j9i582FU4Qvo2bmqnqMYADFk=
 github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo=
-github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
 github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
 github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
 github.com/spf13/viper v1.15.0 h1:js3yy885G8xwJa6iOISGFwd+qlUo5AvyXb7CiihdtiU=
@@ -502,7 +482,6 @@ golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20210423185535-09eb48e85fd7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
@@ -535,7 +514,6 @@ golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBn
 golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
 golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
 golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
-golang.org/x/tools v0.0.0-20190624222133-a101b041ded4/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
 golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
 golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
@@ -699,9 +677,6 @@ gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C
 gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
-gotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk=
-gotest.tools/v3 v3.0.3 h1:4AuOwCGf4lLR9u3YOe2awrHygurzhO/HeQ6laiA6Sx0=
-gotest.tools/v3 v3.0.3/go.mod h1:Z7Lb0S5l+klDB31fvDQX8ss/FlKDxtlFlw3Oa8Ymbl8=
 honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=

From 04979b2f9d6b5041eec6dd201e05ee12719c7e44 Mon Sep 17 00:00:00 2001
From: "Johannes M. Scheuermann" <jscheuermann@apple.com>
Date: Mon, 2 Sep 2024 16:03:51 +0200
Subject: [PATCH 3/3] Update docs typo

---
 docs/manual/operations.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/manual/operations.md b/docs/manual/operations.md
index 1d4b37ac6..336585add 100644
--- a/docs/manual/operations.md
+++ b/docs/manual/operations.md
@@ -222,7 +222,7 @@ In case you lost the quorum of coordinators and you are not able to restore the
 
 This section will describe the procedure for case 2.
 
-**NOTE** The assumption here is that at least one coordinator is still available with it's coordinator state.
+**NOTE** The assumption here is that at least one coordinator is still available with its coordinator state.
 **NOTE** This action can cause data loss. Perform those actions with care.
 
 - Set all the `FoundationDBCluster` resources for this FDB cluster to `spec.Skip = true` to make sure the operator is not changing the manual changed state.