Fix for many subclusters sandboxed at the same time (#815)

This update addresses an issue that occurs when multiple subclusters are
added to a new sandbox during the same reconciliation iteration. The
problem arose after sandboxing the first subcluster; the second
subcluster failed to sandbox.

The failure was due to the need to provide both a host from the main
cluster and a host from the sandbox to the vcluster API. To obtain the
sandbox host, the system relies on the `vdb.status.sandbox` being
updated. Since this update wasn't happening, the process of finding the
sandbox host by checking pod facts didn't locate any sandboxed pods.

The solution is to update `vdb.status.sandboxes` as each subcluster is
sandboxed. This update allows subsequent sandboxes to find the correct
pod facts, ensuring the process completes successfully for all
subclusters.

---------

Co-authored-by: Roy Paulin <rnguetsopken@opentext.com>

pkg/controllers/vdb/sandboxsubcluster_reconciler.go

@@ -17,7 +17,6 @@ package vdb
 
 import (
 	"context"
-	"errors"
 	"fmt"
 	"reflect"
 
@@ -159,7 +158,6 @@ func (s *SandboxSubclusterReconciler) sandboxSubclusters(ctx context.Context) (c
 // executeSandboxCommand will call sandbox API in vclusterOps, create/update sandbox config maps,
 // and update sandbox status in vdb
 func (s *SandboxSubclusterReconciler) executeSandboxCommand(ctx context.Context, scSbMap map[string]string) (ctrl.Result, error) {
-	succeedSbScMap := make(map[string][]string)
 	seenSandboxes := make(map[string]any)
 
 	// We can simply loop over the scSbMap and sandbox each subcluster. However,
@@ -187,22 +185,24 @@ func (s *SandboxSubclusterReconciler) executeSandboxCommand(ctx context.Context,
 					return true, nil
 				})
 				if err != nil {
-					return ctrl.Result{}, errors.Join(err, s.updateSandboxStatus(ctx, succeedSbScMap))
+					return ctrl.Result{}, err
 				}
 			}
 			res, err := s.sandboxSubcluster(ctx, sc, sb)
 			if verrors.IsReconcileAborted(res, err) {
-				// when one subcluster failed to be sandboxed, update sandbox status and return error
-				return res, errors.Join(err, s.updateSandboxStatus(ctx, succeedSbScMap))
+				return res, err
 			}
-			succeedSbScMap[sb] = append(succeedSbScMap[sb], sc)
 			seenSandboxes[sb] = struct{}{}
+
+			// Always update status as we go. When sandboxing two subclusters in
+			// the same sandbox, the second subcluster depends on the status
+			// update from the first subcluster in order to properly sandbox.
+			err = s.addSandboxedSubclusterToStatus(ctx, sc, sb)
+			if err != nil {
+				return ctrl.Result{}, err
+			}
 		}
 	}
-	err := s.updateSandboxStatus(ctx, succeedSbScMap)
-	if err != nil {
-		return ctrl.Result{}, err
-	}
 
 	// create/update a sandbox config map
 	for sb := range seenSandboxes {
@@ -229,10 +229,12 @@ func (s *SandboxSubclusterReconciler) findInitiatorIPs(ctx context.Context, sand
 		if err := pfs.Collect(ctx, s.Vdb); err != nil {
 			return nil, ctrl.Result{}, err
 		}
-		// If this is the first pod in the sandbox, then only an API from the
+		// If this is the first pod in the sandbox, then only an ip from the
 		// main cluster is needed.
 		if len(pfs.Detail) == 0 {
-			return []string{s.InitiatorIPs[vapi.MainCluster]}, ctrl.Result{}, nil
+			ips := []string{s.InitiatorIPs[vapi.MainCluster]}
+			s.Log.Info("detected first subcluster added to a new sandbox", "initiatorIPs", ips)
+			return ips, ctrl.Result{}, nil
 		}
 		pf, found := pfs.findFirstPodSorted(func(v *PodFact) bool {
 			return v.upNode && v.isPrimary
@@ -243,7 +245,9 @@ func (s *SandboxSubclusterReconciler) findInitiatorIPs(ctx context.Context, sand
 		}
 		s.InitiatorIPs[sandbox] = pf.podIP
 	}
-	return []string{s.InitiatorIPs[vapi.MainCluster], s.InitiatorIPs[sandbox]}, ctrl.Result{}, nil
+	ips := []string{s.InitiatorIPs[vapi.MainCluster], s.InitiatorIPs[sandbox]}
+	s.Log.Info("found two initiator IPs", "main", ips[0], "sandbox", ips[1])
+	return ips, ctrl.Result{}, nil
 }
 
 // checkSandboxConfigMap will create or update a sandbox config map if needed
@@ -378,7 +382,8 @@ func (s *SandboxSubclusterReconciler) sandboxSubcluster(ctx context.Context, sub
 	return ctrl.Result{}, nil
 }
 
-// updateSandboxStatus will update sandbox status in vdb
+// updateSandboxStatus will update sandbox status in vdb. This is a bulk update
+// and can handle multiple subclusters at once.
 func (s *SandboxSubclusterReconciler) updateSandboxStatus(ctx context.Context, originalSbScMap map[string][]string) error {
 	updateStatus := func(vdbChg *vapi.VerticaDB) error {
 		// make a copy of originalSbScMap since we will modify the map, and
@@ -408,3 +413,23 @@ func (s *SandboxSubclusterReconciler) updateSandboxStatus(ctx context.Context, o
 
 	return vdbstatus.Update(ctx, s.Client, s.Vdb, updateStatus)
 }
+
+// addSandboxedSubclusterToStatus will add a single subcluster to the sandbox status.
+func (s *SandboxSubclusterReconciler) addSandboxedSubclusterToStatus(ctx context.Context, subcluster, sandbox string) error {
+	updateStatus := func(vdbChg *vapi.VerticaDB) error {
+		// for existing sandboxes, update their subclusters in sandbox status
+		for i := range vdbChg.Status.Sandboxes {
+			if vdbChg.Status.Sandboxes[i].Name == sandbox {
+				vdbChg.Status.Sandboxes[i].Subclusters = append(vdbChg.Status.Sandboxes[i].Subclusters, subcluster)
+				return nil
+			}
+		}
+
+		// If we get here, we didn't find the sandbox. So we are sandboxing the
+		// first subcluster in a sandbox.
+		newStatus := vapi.SandboxStatus{Name: sandbox, Subclusters: []string{subcluster}}
+		vdbChg.Status.Sandboxes = append(vdbChg.Status.Sandboxes, newStatus)
+		return nil
+	}
+	return vdbstatus.Update(ctx, s.Client, s.Vdb, updateStatus)
+}

tests/e2e-leg-9/sandbox-on-create/05-create-secrets.yaml

@@ -0,0 +1,19 @@
+# (c) Copyright [2021-2024] Open Text.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# You may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+apiVersion: kuttl.dev/v1beta1
+kind: TestStep
+commands:
+  - script: kustomize build ../../manifests/communal-creds/overlay | kubectl apply -f - --namespace $NAMESPACE
+  - script: kustomize build ../../manifests/priv-container-creds/overlay | kubectl apply -f - --namespace $NAMESPACE
+  - script: kustomize build ../../manifests/vertica-license/overlay | kubectl apply -f - --namespace $NAMESPACE

tests/e2e-leg-9/sandbox-on-create/10-assert.yaml

@@ -0,0 +1,21 @@
+# (c) Copyright [2021-2024] Open Text.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# You may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+apiVersion: v1
+kind: Pod
+metadata:
+  namespace: verticadb-operator
+  labels:
+    control-plane: verticadb-operator
+status:
+  phase: Running

tests/e2e-leg-9/sandbox-on-create/10-verify-operator.yaml

@@ -0,0 +1,12 @@
+# (c) Copyright [2021-2024] Open Text.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# You may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

tests/e2e-leg-9/sandbox-on-create/15-assert.yaml

@@ -0,0 +1,38 @@
+# (c) Copyright [2021-2024] Open Text.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# You may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: v-sandbox-on-create-pri1
+status:
+  currentReplicas: 3
+---
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: v-sandbox-on-create-sec1
+status:
+  currentReplicas: 1
+---
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: v-sandbox-on-create-sec2
+status:
+  currentReplicas: 1
+---
+apiVersion: vertica.com/v1
+kind: VerticaDB
+metadata:
+  name: v-sandbox-on-create

tests/e2e-leg-9/sandbox-on-create/15-setup-vdb.yaml

@@ -0,0 +1,17 @@
+# (c) Copyright [2021-2024] Open Text.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# You may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+apiVersion: kuttl.dev/v1beta1
+kind: TestStep
+commands:
+  - command: bash -c "kustomize build setup-vdb/overlay | kubectl -n $NAMESPACE apply -f - "

tests/e2e-leg-9/sandbox-on-create/20-assert.yaml

@@ -0,0 +1,67 @@
+# (c) Copyright [2021-2024] Open Text.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# You may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+apiVersion: kuttl.dev/v1beta1
+kind: TestAssert
+timeout: 900 # 15 minutes since this step can be quite long
+---
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: v-sandbox-on-create-pri1
+status:
+  currentReplicas: 3
+  readyReplicas: 3
+---
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: v-sandbox-on-create-sec1
+status:
+  currentReplicas: 1
+  readyReplicas: 1
+---
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: v-sandbox-on-create-sec2
+status:
+  currentReplicas: 1
+  readyReplicas: 1
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: v-sandbox-on-create-sand1
+data:
+  sandboxName: sand1
+  verticaDBName: v-sandbox-on-create
+immutable: true
+---
+apiVersion: vertica.com/v1
+kind: VerticaDB
+metadata:
+  name: v-sandbox-on-create
+status:
+  sandboxes:
+  - name: sand1
+    subclusters:
+    - sec1
+    - sec2
+  subclusters:
+    - addedToDBCount: 3
+      upNodeCount: 3
+    - addedToDBCount: 1
+      upNodeCount: 1
+    - addedToDBCount: 1
+      upNodeCount: 1

tests/e2e-leg-9/sandbox-on-create/20-wait-for-createdb.yaml

@@ -0,0 +1 @@
+# Intentionally empty to give this step a name in kuttl

tests/e2e-leg-9/sandbox-on-create/25-wait-for-steady-state.yaml

@@ -0,0 +1,17 @@
+# (c) Copyright [2021-2024] Open Text.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# You may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+apiVersion: kuttl.dev/v1beta1
+kind: TestStep
+commands:
+  - command: bash -c "../../../scripts/wait-for-verticadb-steady-state.sh -n verticadb-operator -t 360 $NAMESPACE"

tests/e2e-leg-9/sandbox-on-create/95-delete-cr.yaml

@@ -0,0 +1,18 @@
+# (c) Copyright [2021-2024] Open Text.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# You may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+apiVersion: kuttl.dev/v1beta1
+kind: TestStep
+delete:
+  - apiVersion: vertica.com/v1
+    kind: VerticaDB

tests/e2e-leg-9/sandbox-on-create/95-errors.yaml

@@ -0,0 +1,24 @@
+# (c) Copyright [2021-2024] Open Text.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# You may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+apiVersion: apps/v1
+kind: StatefulSet
+---
+apiVersion: v1
+kind: Service
+metadata:
+  labels:
+    app.kubernetes.io/managed-by: verticadb-operator
+---
+apiVersion: vertica.com/v1
+kind: VerticaDB

tests/e2e-leg-9/sandbox-on-create/96-assert.yaml

@@ -0,0 +1,19 @@
+# (c) Copyright [2021-2024] Open Text.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# You may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+apiVersion: v1
+kind: Pod
+metadata:
+  name: clean-communal
+status:
+  phase: Succeeded

tests/e2e-leg-9/sandbox-on-create/96-cleanup-storage.yaml

@@ -0,0 +1,17 @@
+# (c) Copyright [2021-2024] Open Text.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# You may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+apiVersion: kuttl.dev/v1beta1
+kind: TestStep
+commands:
+  - command: bash -c "kustomize build clean-communal/overlay | kubectl -n $NAMESPACE apply -f - "