Update sandbox label (#798)

A nondeterministic failure occurs when a label is updated based on the
sandbox spec. We need to ensure that the label is updated only after the
status is set by the sandbox subcluster

api/v1/helpers.go

@@ -816,10 +816,10 @@ func (v *VerticaDB) IsHTTPSTLSConfGenerationEnabled() (bool, error) {
 // GetSubclusterSandboxName returns the sandbox the given subcluster belongs to,
 // or an empty string if it does not belong to any
 func (v *VerticaDB) GetSubclusterSandboxName(scName string) string {
-	for i := range v.Spec.Sandboxes {
-		for j := range v.Spec.Sandboxes[i].Subclusters {
-			if scName == v.Spec.Sandboxes[i].Subclusters[j].Name {
-				return v.Spec.Sandboxes[i].Name
+	for i := range v.Status.Sandboxes {
+		for j := range v.Status.Sandboxes[i].Subclusters {
+			if scName == v.Status.Sandboxes[i].Subclusters[j] {
+				return v.Status.Sandboxes[i].Name
 			}
 		}
 	}

pkg/controllers/vdb/upgrade_test.go

@@ -79,15 +79,14 @@ var _ = Describe("upgrade", func() {
 		vdb.Spec.Sandboxes = []vapi.Sandbox{
 			{Name: sbName, Image: "new-img", Subclusters: []vapi.SubclusterName{{Name: "sc2"}}},
 		}
+		vdb.Status.Sandboxes = []vapi.SandboxStatus{
+			{Name: sbName, Subclusters: []string{"sc2"}},
+		}
 		test.CreatePods(ctx, k8sClient, vdb, test.AllPodsRunning)
 		defer test.DeletePods(ctx, k8sClient, vdb)
 		test.CreateVDB(ctx, k8sClient, vdb)
 		defer test.DeleteVDB(ctx, k8sClient, vdb)
 
-		vdb.Status.Sandboxes = []vapi.SandboxStatus{
-			{Name: sbName, Subclusters: []string{"sc2"}},
-		}
-
 		// upgrade not needed on main cluster
 		mgr := MakeUpgradeManager(vdbRec, logger, vdb, vapi.OnlineUpgradeInProgress,
 			func(vdb *vapi.VerticaDB) bool { return true })

pkg/controllers/vdb/verticadb_controller.go

@@ -261,6 +261,8 @@ func (r *VerticaDBReconciler) constructActors(log logr.Logger, vdb *vapi.Vertica
 		// Trigger sandbox upgrade when the image field for the sandbox
 		// is changed
 		MakeSandboxUpgradeReconciler(r, log, vdb),
+		// Add the label after update the sandbox subcluster status field
+		MakeObjReconciler(r, log, vdb, pfacts, ObjReconcileModeAll),
 		// Resize any PVs if the local data size changed in the vdb
 		MakeResizePVReconciler(r, log, vdb, prunner, pfacts),
 		// This must be the last reconciler. It makes sure that all dependent

pkg/iter/sc_finder_test.go

@@ -142,6 +142,9 @@ var _ = Describe("sc_finder", func() {
 		vdb.Spec.Sandboxes = []vapi.Sandbox{
 			{Name: sbName, Subclusters: []vapi.SubclusterName{{Name: scNames[0]}}},
 		}
+		vdb.Status.Sandboxes = []vapi.SandboxStatus{
+			{Name: sbName, Subclusters: []string{scNames[0]}},
+		}
 		test.CreatePods(ctx, k8sClient, vdb, test.AllPodsRunning)
 		vdbCopy := *vdb // Make a copy for cleanup since we will mutate vdb
 		defer test.DeletePods(ctx, k8sClient, &vdbCopy)
@@ -154,7 +157,6 @@ var _ = Describe("sc_finder", func() {
 		// Change the image in the vdb spec to prove that we fill in the image
 		// from the statefulset
 		vdb.Spec.Image = "should-not-report-this-image"
-
 		finder := MakeSubclusterFinder(k8sClient, vdb)
 		sts, err := finder.FindStatefulSets(ctx, FindExisting, vapi.MainCluster)
 		Expect(err).Should(Succeed())
@@ -190,6 +192,9 @@ var _ = Describe("sc_finder", func() {
 		vdb.Spec.Sandboxes = []vapi.Sandbox{
 			{Name: sbName, Subclusters: []vapi.SubclusterName{{Name: scNames[0]}}},
 		}
+		vdb.Status.Sandboxes = []vapi.SandboxStatus{
+			{Name: sbName, Subclusters: []string{scNames[0]}},
+		}
 		test.CreatePods(ctx, k8sClient, vdb, test.AllPodsRunning)
 		defer test.DeletePods(ctx, k8sClient, vdb)