Initial docs for coordinator recovery and e2e test

api/v1beta2/foundationdb_process_address.go

@@ -316,7 +316,7 @@ func GetFullAddressList(address string, primaryOnly bool, processNumber int, req
 		pAddr := NewProcessAddress(nil, address, GetProcessPort(processNumber, true), map[string]bool{"tls": true})
 		addrs = append(addrs, pAddr)
 
-		if requireTLS && primaryOnly {
+		if primaryOnly {
 			return addrs
 		}
 	}

api/v1beta2/foundationdbcluster_types.go

@@ -96,10 +96,9 @@ type FoundationDBClusterSpec struct {
 	// infer the process counts based on the database configuration.
 	ProcessCounts ProcessCounts `json:"processCounts,omitempty"`
 
-	// SeedConnectionString provides a connection string for the initial
-	// reconciliation.
-	//
-	// After the initial reconciliation, this will not be used.
+	// SeedConnectionString provides an additional connection string. This connection string will be used in addition
+	// to the connection string defined under cluster.Status.ConnectionString to connect to the cluster. This setting
+	// can be used to create a multi-region cluster or to recover a cluster if it is out of sync.
 	SeedConnectionString string `json:"seedConnectionString,omitempty"`
 
 	// PartialConnectionString provides a way to specify part of the

docs/cluster_spec.md

@@ -235,7 +235,7 @@ FoundationDBClusterSpec defines the desired state of a cluster.
 | databaseConfiguration | DatabaseConfiguration defines the database configuration. | [DatabaseConfiguration](#databaseconfiguration) | false |
 | processes | Processes defines process-level settings. | map[[ProcessClass](#processclass)][ProcessSettings](#processsettings) | false |
 | processCounts | ProcessCounts defines the number of processes to configure for each process class. You can generally omit this, to allow the operator to infer the process counts based on the database configuration. | [ProcessCounts](#processcounts) | false |
-| seedConnectionString | SeedConnectionString provides a connection string for the initial reconciliation.  After the initial reconciliation, this will not be used. | string | false |
+| seedConnectionString | SeedConnectionString provides an additional connection string. This connection string will be used in addition to the connection string defined under cluster.Status.ConnectionString to connect to the cluster. This setting can be used to create a multi-region cluster or to recover a cluster if it is out of sync. | string | false |
 | partialConnectionString | PartialConnectionString provides a way to specify part of the connection string (e.g. the database name and coordinator generation) without specifying the entire string. This does not allow for setting the coordinator IPs. If `SeedConnectionString` is set, `PartialConnectionString` will have no effect. They cannot be used together. | [ConnectionString](#connectionstring) | false |
 | faultDomain | FaultDomain defines the rules for what fault domain to replicate across. | [FoundationDBClusterFaultDomain](#foundationdbclusterfaultdomain) | false |
 | processGroupsToRemove | ProcessGroupsToRemove defines the process groups that we should remove from the cluster. This list contains the process group IDs. | [][ProcessGroupID](#processgroupid) | false |

docs/manual/operations.md

@@ -212,6 +212,55 @@ There are a few risks and limitations to the current implementation:
 The current risks are limited to releasing the maintenance mode earlier than it should be.
 In this case data-movement will be triggered for the down processes after 60 seconds, the data-movement shouldn't cause any operational issues.
 
+## Recover Lost Quorum Of Coordinators
+
+If the coordinators Pods are still running but they got new IP addresses, read [Coordinators Getting New IPs](./debugging.md#coordinators-getting-new-ips).
+In case you lost the quorum of coordinators and you are not able to restore the coordinator state, you have two options:
+
+1. Recover the cluster from the latest backup.
+1. Recover the coordinator state with the risk of data loss.
+
+This section will describe the procedure for case 2.
+
+**NOTE** The assumption here is that at least one coordinator is still available with its coordinator state.
+**NOTE** This action can cause data loss. Perform those actions with care.
+
+- Set all the `FoundationDBCluster` resources for this FDB cluster to `spec.Skip = true` to make sure the operator is not changing the manual changed state. 
+- Fetch the last connection string from the `FoundationDBCluster` status, e.g. `kubectl get fdb ${cluster} -o jsonpath='{ .status.connectionString }'`. 
+- Copy the coordinator state from one of the running coordinators to your local machine:
+
+```bash
+kubectl cp ${coordinator-pod}:/var/fdb/data/1/coordination-0.fdq ./coordination-0.fdq 
+kubectl cp ${coordinator-pod}:/var/fdb/data/1/coordination-1.fdq ./coordination-1.fdq
+```
+
+- Select a number of Pods you want to use as new coordinators and copy the files to those pods:
+
+```bash
+kubectl cp ./coordination-0.fdq ${new-coordinator-pod}:/var/fdb/data/1/coordination-0.fdq 
+kubectl cp ./coordination-1.fdq ${new-coordinator-pod}:/var/fdb/data/1/coordination-1.fdq 
+```
+
+- Update the `ConfigMap` to contain the new connection string, the new connection string must contain the still existing coordinators and the new coordinators. The old entries must be removed.
+- Wait ~1 min until the `ConfigMap` is synced to all Pods, you can check the `/var/dynamic-conf/fdb.cluster` inside a Pod if you are unsure.
+- Now all Pods must be restarted and the previous local cluster file must be deleted to make sure the fdbserver is picking the connection string from the seed cluster file (`/var/dynamic-conf/fdb.cluster`).
+
+```bash
+for pod in $(kubectl get po -l foundationdb.org/fdb-cluster-name=${cluster} -o name --no-headers);
+do
+    echo $pod
+    kubectl $pod -- bash -c 'pkill fdbserver && rm -f /var/fdb/data/fdb.cluster && pkill fdbserver'
+done
+```
+
+If the cluster is a multi-region cluster, perform this step for all running regions.
+
+- Now you can exec into a container and use `fdbcli` to connect to the cluster.
+- If you use a multi-region cluster you have to issue `force_recovery_with_data_loss`.
+- Update the `FoundationDBCluster` `seedConnectionString` under `spec.seedConnectionString` with the new connection string.
+- Now you can set `spec.Skip = false` to let the operator take over again.
+- Depending on the state of the multi-region cluster, you probably want to change the desired database configuration to drop ha.
+
 ## Next
 
 You can continue on to the [next section](scaling.md) or go back to the [table of contents](index.md).

e2e/fixtures/factory.go

@@ -24,6 +24,7 @@ import (
 	"bytes"
 	ctx "context"
 	"fmt"
+	"golang.org/x/net/context"
 	"io"
 	"log"
 	"math/rand"
@@ -441,16 +442,18 @@ type ClusterOption func(*Factory, *fdbv1beta2.FoundationDBCluster)
 
 // ExecuteCmdOnPod runs a command on the provided Pod. The command will be executed inside a bash -c ”.
 func (factory *Factory) ExecuteCmdOnPod(
+	ctx context.Context,
 	pod *corev1.Pod,
 	container string,
 	command string,
 	printOutput bool,
 ) (string, string, error) {
-	return factory.ExecuteCmd(pod.Namespace, pod.Name, container, command, printOutput)
+	return factory.ExecuteCmd(ctx, pod.Namespace, pod.Name, container, command, printOutput)
 }
 
 // ExecuteCmd executes command in the default container of a Pod with shell, returns stdout and stderr.
 func (factory *Factory) ExecuteCmd(
+	ctx context.Context,
 	namespace string,
 	name string,
 	container string,
@@ -464,7 +467,7 @@ func (factory *Factory) ExecuteCmd(
 	}
 	var stdout bytes.Buffer
 	var stderr bytes.Buffer
-	err := factory.ExecuteCommandRaw(namespace, name, container, cmd, nil, &stdout, &stderr, false)
+	err := factory.ExecuteCommandRaw(ctx, namespace, name, container, cmd, nil, &stdout, &stderr, false)
 	sout := stdout.String()
 	serr := stderr.String()
 	// TODO: Stream these to our own stdout as we run.
@@ -487,6 +490,7 @@ func (factory *Factory) ExecuteCmd(
 
 // ExecuteCommandRaw will run the command without putting it into a shell.
 func (factory *Factory) ExecuteCommandRaw(
+	ctx context.Context,
 	namespace string,
 	name string,
 	container string,
@@ -511,15 +515,73 @@ func (factory *Factory) ExecuteCommandRaw(
 		option,
 		scheme.ParameterCodec,
 	)
-	exec, err := remotecommand.NewSPDYExecutor(factory.getConfig(), "POST", req.URL())
+	spdyExec, err := remotecommand.NewSPDYExecutor(factory.getConfig(), "POST", req.URL())
 	if err != nil {
 		return err
 	}
-	return exec.Stream(remotecommand.StreamOptions{
-		Stdin:  stdin,
-		Stdout: stdout,
-		Stderr: stderr,
-	})
+	return spdyExec.StreamWithContext(ctx,
+		remotecommand.StreamOptions{
+			Stdin:  stdin,
+			Stdout: stdout,
+			Stderr: stderr,
+		})
+}
+
+// DownloadFile will download the file from the provided Pod/container into w.
+func (factory *Factory) DownloadFile(ctx context.Context, target *corev1.Pod, container string, src string, w io.Writer) error {
+	errOut := bytes.NewBuffer([]byte{})
+	reader, writer := io.Pipe()
+	defer func() {
+		log.Println("Done downloading file")
+		_ = writer.Close()
+	}()
+
+	// Copy the content from stdout of the container to the new file.
+	go func() {
+		defer func() {
+			_ = writer.Close()
+		}()
+		_, err := io.Copy(w, reader)
+		if err != nil {
+			log.Println("DownloadFile copy, err:", err)
+		}
+	}()
+
+	err := factory.ExecuteCommandRaw(ctx, target.Namespace, target.Name, container, []string{"/bin/cp", src, "/dev/stdout"}, nil, writer, errOut, false)
+	if err != nil {
+		log.Println(errOut.String())
+	}
+
+	return err
+}
+
+// UploadFile uploads a file from src into the Pod/container dst.
+func (factory *Factory) UploadFile(ctx context.Context, target *corev1.Pod, container string, src io.Reader, dst string) error {
+	out := bytes.NewBuffer([]byte{})
+	errOut := bytes.NewBuffer([]byte{})
+	reader, writer := io.Pipe()
+	defer func() {
+		log.Println("Done uploading file")
+		_ = reader.Close()
+	}()
+
+	// Read the file provided via src and pipe it to the reader.
+	go func(r io.Reader, writer *io.PipeWriter) {
+		defer func() {
+			_ = writer.Close()
+		}()
+		_, err := io.Copy(writer, r)
+		if err != nil {
+			log.Println("UploadFile copy, err:", err)
+		}
+	}(src, writer)
+
+	err := factory.ExecuteCommandRaw(ctx, target.Namespace, target.Name, container, []string{"tee", "-a", dst}, reader, out, errOut, false)
+	if err != nil {
+		log.Println(errOut.String())
+	}
+
+	return err
 }
 
 // GetLogsFromPod returns the logs for the provided Pod and container

e2e/fixtures/fdb_cluster.go

@@ -23,6 +23,7 @@ package fixtures
 import (
 	ctx "context"
 	"fmt"
+	"golang.org/x/net/context"
 	"log"
 	"math"
 	"strconv"
@@ -79,7 +80,7 @@ func (fdbCluster *FdbCluster) ExecuteCmdOnPod(
 	command string,
 	printOutput bool,
 ) (string, string, error) {
-	return fdbCluster.factory.ExecuteCmd(pod.Namespace, pod.Name, container, command, printOutput)
+	return fdbCluster.factory.ExecuteCmd(context.Background(), pod.Namespace, pod.Name, container, command, printOutput)
 }
 
 func (factory *Factory) createFdbClusterObject(
@@ -1480,3 +1481,15 @@ func (fdbCluster *FdbCluster) GetListOfUIDsFromVolumeClaims(processClass fdbv1be
 
 	return uids
 }
+
+// UpdateConnectionString will update the connection string in the ConfigMap and the SeedConnectionString of the custer.
+func (fdbCluster *FdbCluster) UpdateConnectionString(connectionString string) {
+	fdbCluster.cluster.Spec.SeedConnectionString = connectionString
+	fdbCluster.UpdateClusterSpec()
+
+	cm := &corev1.ConfigMap{}
+	gomega.Expect(fdbCluster.factory.controllerRuntimeClient.Get(ctx.Background(), client.ObjectKey{Namespace: fdbCluster.Namespace(), Name: fdbCluster.Name() + "-config"}, cm)).NotTo(gomega.HaveOccurred())
+	gomega.Expect(cm.Data).To(gomega.HaveKey(fdbv1beta2.ClusterFileKey))
+	cm.Data[fdbv1beta2.ClusterFileKey] = connectionString
+	gomega.Expect(fdbCluster.factory.controllerRuntimeClient.Update(ctx.Background(), cm)).NotTo(gomega.HaveOccurred())
+}

e2e/fixtures/status.go

@@ -22,6 +22,7 @@ package fixtures
 
 import (
 	"bytes"
+	"context"
 	"encoding/json"
 	"fmt"
 	"log"
@@ -133,6 +134,7 @@ func (fdbCluster *FdbCluster) RunFdbCliCommandInOperatorWithoutRetry(
 	// is returned) we will try the version for the spec. This should reduce some test flakiness.
 	for _, fdbCliPath := range fdbCliPaths {
 		stdout, stderr, err = fdbCluster.factory.ExecuteCmd(
+			context.Background(),
 			pod.Namespace,
 			pod.Name,
 			"manager",

e2e/test_operator/operator_test.go

@@ -865,6 +865,7 @@ var _ = Describe("Operator", Label("e2e", "pr"), func() {
 				// File creation should fail due to I/O error
 				Eventually(func() error {
 					_, _, err := factory.ExecuteCmdOnPod(
+						context.Background(),
 						&podWithIOError,
 						fdbv1beta2.MainContainerName,
 						"touch /var/fdb/data/test",
@@ -2378,6 +2379,7 @@ var _ = Describe("Operator", Label("e2e", "pr"), func() {
 			var kubernetesServiceHost string
 			Eventually(func(g Gomega) error {
 				std, _, err := factory.ExecuteCmdOnPod(
+					context.Background(),
 					&selectedPod,
 					fdbv1beta2.MainContainerName,
 					"printenv KUBERNETES_SERVICE_HOST",
@@ -2394,6 +2396,7 @@ var _ = Describe("Operator", Label("e2e", "pr"), func() {
 			// Make sure that the partition takes effect.
 			Eventually(func() error {
 				_, _, err := factory.ExecuteCmdOnPod(
+					context.Background(),
 					&selectedPod,
 					fdbv1beta2.MainContainerName,
 					fmt.Sprintf("nc -vz -w 2 %s 443", kubernetesServiceHost),