Create sandbox config maps in sandbox reconciler (#790)

This will create/update a sandbox config map whenever we create a new
sandbox in sandbox reconciler. The config map is watched by sandbox
controller for restarting/upgrading the nodes in a sandbox. The config
map will be deleted if vdb is deleted or the sandbox is unsandboxed.

pkg/builder/builder.go

@@ -1290,6 +1290,30 @@ func BuildStsSpec(nm types.NamespacedName, vdb *vapi.VerticaDB, sc *vapi.Subclus
 	}
 }
 
+// BuildSandboxConfigMap builds a config map for sandbox controller
+func BuildSandboxConfigMap(nm types.NamespacedName, vdb *vapi.VerticaDB, sandbox string) *corev1.ConfigMap {
+	immutable := true
+	return &corev1.ConfigMap{
+		TypeMeta: metav1.TypeMeta{
+			Kind:       "ConfigMap",
+			APIVersion: "v1",
+		},
+		ObjectMeta: metav1.ObjectMeta{
+			Name:            nm.Name,
+			Namespace:       nm.Namespace,
+			Labels:          MakeLabelsForSandboxConfigMap(vdb),
+			Annotations:     MakeAnnotationsForSandboxConfigMap(vdb),
+			OwnerReferences: []metav1.OwnerReference{vdb.GenerateOwnerReference()},
+		},
+		// the data should be immutable since dbName and sandboxName are fixed
+		Immutable: &immutable,
+		Data: map[string]string{
+			vapi.VerticaDBNameKey: vdb.Spec.DBName,
+			vapi.SandboxNameKey:   sandbox,
+		},
+	}
+}
+
 // BuildScrutinizePod construct the spec for the scrutinize pod
 func BuildScrutinizePod(vscr *v1beta1.VerticaScrutinize, vdb *vapi.VerticaDB, args []string) *corev1.Pod {
 	return &corev1.Pod{

pkg/builder/labels_annotations.go

@@ -110,6 +110,13 @@ func MakeLabelsForSvcObject(vdb *vapi.VerticaDB, sc *vapi.Subcluster, svcType st
 	return labels
 }
 
+// MakeLabelsForSandboxConfigMap constructs the labels of the sandbox config map
+func MakeLabelsForSandboxConfigMap(vdb *vapi.VerticaDB) map[string]string {
+	labels := makeLabelsForObject(vdb, nil, false)
+	labels[vmeta.WatchedBySandboxLabel] = vmeta.WatchedBySandboxTrue
+	return labels
+}
+
 // MakeAnnotationsForObjects builds the list of annotations that are to be
 // included on new objects.
 func MakeAnnotationsForObject(vdb *vapi.VerticaDB) map[string]string {
@@ -135,6 +142,16 @@ func MakeAnnotationsForStsObject(vdb *vapi.VerticaDB, sc *vapi.Subcluster) map[s
 	return annotations
 }
 
+// MakeAnnotationsForSandboxConfigMap builds the list of annotations that are included
+// in the sandbox config map.
+func MakeAnnotationsForSandboxConfigMap(vdb *vapi.VerticaDB) map[string]string {
+	annotations := MakeAnnotationsForObject(vdb)
+	if ver, ok := vdb.Annotations[vmeta.VersionAnnotation]; ok {
+		annotations[vmeta.VersionAnnotation] = ver
+	}
+	return annotations
+}
+
 // MakeSvcSelectorLabels returns the labels that are used for selectors in service objects.
 func MakeBaseSvcSelectorLabels(vdb *vapi.VerticaDB) map[string]string {
 	// We intentionally don't use the common labels because that includes things

pkg/controllers/vdb/imageversion_reconciler.go

@@ -281,7 +281,7 @@ func (v *ImageVersionReconciler) updateVDBAnnotations(ctx context.Context, versi
 // updateConfigMapAnnotations updates the sandbox's configmap annotations with
 // the version
 func (v *ImageVersionReconciler) updateConfigMapAnnotations(ctx context.Context, versionAnn map[string]string) error {
-	nm := names.GenConfigMapName(v.Vdb, v.PFacts.SandboxName)
+	nm := names.GenSandboxConfigMapName(v.Vdb, v.PFacts.SandboxName)
 
 	chgs := vk8s.MetaChanges{
 		NewAnnotations: map[string]string{

pkg/controllers/vdb/imageversion_reconciler_test.go

@@ -75,10 +75,19 @@ vertica(v11.1.0) built by @re-docker2 from tag@releases/VER_10_1_RELEASE_BUILD_1
 			vmeta.VersionAnnotation:     "v23.4.0",
 		}
 		const sbName = "sb1"
-		vdb.Spec.Subclusters[0].Size = 1
+		vdb.Spec.Subclusters = []vapi.Subcluster{
+			{Name: "sc1", Size: 1, Type: vapi.SecondarySubcluster},
+			{Name: "default", Size: 1, Type: vapi.PrimarySubcluster},
+		}
+		vdb.Spec.Sandboxes = []vapi.Sandbox{
+			{Name: sbName, Subclusters: []vapi.SubclusterName{{Name: vdb.Spec.Subclusters[0].Name}}},
+		}
+		test.CreateVDB(ctx, k8sClient, vdb)
+		defer test.DeleteVDB(ctx, k8sClient, vdb)
 		vdb.Status.Sandboxes = []vapi.SandboxStatus{
 			{Name: sbName, Subclusters: []string{vdb.Spec.Subclusters[0].Name}},
 		}
+		Expect(k8sClient.Status().Update(ctx, vdb)).Should(Succeed())
 		test.CreatePods(ctx, k8sClient, vdb, test.AllPodsRunning)
 		defer test.DeletePods(ctx, k8sClient, vdb)
 		test.CreateConfigMap(ctx, k8sClient, vdb, "", sbName)
@@ -95,7 +104,7 @@ vertica(v11.1.0) built by @re-docker2 from tag@releases/VER_10_1_RELEASE_BUILD_1
 		Expect(r.Reconcile(ctx, &ctrl.Request{})).Should(Equal(ctrl.Result{}))
 
 		cm := &corev1.ConfigMap{}
-		nm := names.GenConfigMapName(vdb, sbName)
+		nm := names.GenSandboxConfigMapName(vdb, sbName)
 		Expect(k8sClient.Get(ctx, nm, cm)).Should(Succeed())
 		Expect(cm.ObjectMeta.Annotations).ShouldNot(BeNil())
 		Expect(cm.ObjectMeta.Annotations[vmeta.VersionAnnotation]).Should(Equal("v11.1.1-0"))

pkg/controllers/vdb/podfacts.go

@@ -392,7 +392,7 @@ func (p *PodFacts) collectPodByStsIndex(ctx context.Context, vdb *vapi.VerticaDB
 		// we get the sandbox name from the sts labels if the subcluster
 		// belongs to a sandbox. If the node is up, we will later retrieve
 		// the sandbox state from the catalog
-		pf.sandbox = p.SandboxName
+		pf.sandbox = sts.Labels[vmeta.SandboxNameLabel]
 		setSandboxNodeType(&pf)
 	}
 

pkg/controllers/vdb/sandbox_configmap.go

@@ -65,13 +65,13 @@ func (s *SandboxConfigMapManager) triggerSandboxController(ctx context.Context)
 	chgs := vk8s.MetaChanges{
 		NewAnnotations: anns,
 	}
-	nm := names.GenConfigMapName(s.vdb, s.sandbox)
+	nm := names.GenSandboxConfigMapName(s.vdb, s.sandbox)
 	return vk8s.MetaUpdate(ctx, s.vrec.GetClient(), nm, s.configMap, chgs)
 }
 
 // fetchConfigMap will fetch the sandbox configmap
 func (s *SandboxConfigMapManager) fetchConfigMap(ctx context.Context) error {
-	nm := names.GenConfigMapName(s.vdb, s.sandbox)
+	nm := names.GenSandboxConfigMapName(s.vdb, s.sandbox)
 	err := s.vrec.GetClient().Get(ctx, nm, s.configMap)
 	if err != nil {
 		return err

pkg/controllers/vdb/sandboxsubcluster_reconciler.go

@@ -18,15 +18,20 @@ package vdb
 import (
 	"context"
 	"errors"
+	"reflect"
 
 	"github.com/go-logr/logr"
 	vapi "github.com/vertica/vertica-kubernetes/api/v1"
+	"github.com/vertica/vertica-kubernetes/pkg/builder"
 	"github.com/vertica/vertica-kubernetes/pkg/controllers"
 	"github.com/vertica/vertica-kubernetes/pkg/events"
+	vmeta "github.com/vertica/vertica-kubernetes/pkg/meta"
+	"github.com/vertica/vertica-kubernetes/pkg/names"
 	"github.com/vertica/vertica-kubernetes/pkg/vadmin"
 	"github.com/vertica/vertica-kubernetes/pkg/vadmin/opts/sandboxsc"
 	"github.com/vertica/vertica-kubernetes/pkg/vdbstatus"
 	corev1 "k8s.io/api/core/v1"
+	kerrors "k8s.io/apimachinery/pkg/api/errors"
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 )
@@ -68,9 +73,51 @@ func (s *SandboxSubclusterReconciler) Reconcile(ctx context.Context, _ *ctrl.Req
 		return ctrl.Result{}, err
 	}
 
+	// reconcile sandbox status for the subclusters that are already sandboxed
+	if err := s.reconcileSandboxStatus(ctx); err != nil {
+		return ctrl.Result{}, err
+	}
+
+	// reconcile sandbox config maps for the existing sandboxes
+	if err := s.reconcileSandboxConfigMaps(ctx); err != nil {
+		return ctrl.Result{}, err
+	}
+
 	return s.sandboxSubclusters(ctx)
 }
 
+// reconcileSandboxStatus will update sandbox status for the subclusters that are already sandboxed
+func (s *SandboxSubclusterReconciler) reconcileSandboxStatus(ctx context.Context) error {
+	sbScMap := make(map[string][]string)
+	seenScs := make(map[string]any)
+	for _, v := range s.PFacts.Detail {
+		if _, ok := seenScs[v.subclusterName]; ok {
+			continue
+		}
+		if v.sandbox != vapi.MainCluster {
+			sbScMap[v.sandbox] = append(sbScMap[v.sandbox], v.subclusterName)
+		}
+		seenScs[v.subclusterName] = struct{}{}
+	}
+	if len(sbScMap) > 0 {
+		return s.updateSandboxStatus(ctx, sbScMap)
+	}
+
+	return nil
+}
+
+// reconcileSandboxConfigMaps will create/update sandbox config maps for the existing sandboxes
+func (s *SandboxSubclusterReconciler) reconcileSandboxConfigMaps(ctx context.Context) error {
+	for _, sb := range s.Vdb.Status.Sandboxes {
+		err := s.checkSandboxConfigMap(ctx, sb.Name)
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
 // sandboxSubclusters will add subclusters to their sandboxes defined in the vdb
 func (s *SandboxSubclusterReconciler) sandboxSubclusters(ctx context.Context) (ctrl.Result, error) {
 	// find qualified subclusters with their sandboxes
@@ -86,7 +133,7 @@ func (s *SandboxSubclusterReconciler) sandboxSubclusters(ctx context.Context) (c
 
 	// find an initiator to call vclusterOps
 	initiator, ok := s.PFacts.findFirstPodSorted(func(v *PodFact) bool {
-		return v.sandbox == "" && v.isPrimary && v.upNode
+		return v.sandbox == vapi.MainCluster && v.isPrimary && v.upNode
 	})
 	if ok {
 		s.InitiatorIP = initiator.podIP
@@ -104,21 +151,104 @@ func (s *SandboxSubclusterReconciler) sandboxSubclusters(ctx context.Context) (c
 	return ctrl.Result{}, nil
 }
 
-// executeSandboxCommand will call sandbox API in vclusterOps, then update sandbox status in vdb
+// executeSandboxCommand will call sandbox API in vclusterOps, create/update sandbox config maps,
+// and update sandbox status in vdb
 func (s *SandboxSubclusterReconciler) executeSandboxCommand(ctx context.Context, scSbMap map[string]string) error {
 	succeedSbScMap := make(map[string][]string)
+	seenSandboxes := make(map[string]any)
 	for sc, sb := range scSbMap {
 		err := s.sandboxSubcluster(ctx, sc, sb)
 		if err != nil {
 			// when one subcluster failed to be sandboxed, update sandbox status and return error
 			return errors.Join(err, s.updateSandboxStatus(ctx, succeedSbScMap))
-		} else {
-			succeedSbScMap[sb] = append(succeedSbScMap[sb], sc)
 		}
+		succeedSbScMap[sb] = append(succeedSbScMap[sb], sc)
+		// create/update a sandbox config map
+		if _, ok := seenSandboxes[sb]; !ok {
+			err = s.checkSandboxConfigMap(ctx, sb)
+			if err != nil {
+				// when creating/updating sandbox config map failed, update sandbox status and return error
+				return errors.Join(err, s.updateSandboxStatus(ctx, succeedSbScMap))
+			}
+		}
+		seenSandboxes[sb] = struct{}{}
 	}
 	return s.updateSandboxStatus(ctx, succeedSbScMap)
 }
 
+// checkSandboxConfigMap will create or update a sandbox config map if needed
+func (s *SandboxSubclusterReconciler) checkSandboxConfigMap(ctx context.Context, sandbox string) error {
+	nm := names.GenSandboxConfigMapName(s.Vdb, sandbox)
+	curCM := &corev1.ConfigMap{}
+	newCM := builder.BuildSandboxConfigMap(nm, s.Vdb, sandbox)
+	err := s.Client.Get(ctx, nm, curCM)
+	if err != nil && kerrors.IsNotFound(err) {
+		s.Log.Info("Creating sandbox config map", "Name", nm)
+		return s.Client.Create(ctx, newCM)
+	}
+	if s.updateSandboxConfigMapFields(curCM, newCM) {
+		s.Log.Info("Updating sandbox config map", "Name", nm)
+		return s.Client.Update(ctx, newCM)
+	}
+	s.Log.Info("Found an existing sandbox config map with correct content, skip updating it", "Name", nm)
+	return nil
+}
+
+// updateSandboxConfigMapFields checks if we need to update the content of a config map,
+// if so, we will update the content of that config map and return true
+func (s *SandboxSubclusterReconciler) updateSandboxConfigMapFields(curCM, newCM *corev1.ConfigMap) bool {
+	updated := false
+	// exclude sandbox controller trigger ID from the annotations because
+	// vdb controller will set this in current config map, and the new
+	// config map cannot get it
+	triggerID, hasTriggerID := curCM.Annotations[vmeta.SandboxControllerTriggerID]
+	if hasTriggerID {
+		delete(curCM.Annotations, vmeta.SandboxControllerTriggerID)
+	}
+	delete(newCM.Annotations, vmeta.SandboxControllerTriggerID)
+	// exclude version annotation because vdb controller can set a different
+	// vertica version annotation for a sandbox in current config map
+	version, hasVersion := curCM.Annotations[vmeta.VersionAnnotation]
+	if hasVersion {
+		delete(curCM.Annotations, vmeta.VersionAnnotation)
+	}
+	delete(newCM.Annotations, vmeta.VersionAnnotation)
+	if stringMapDiffer(curCM.ObjectMeta.Annotations, newCM.ObjectMeta.Annotations) {
+		updated = true
+		curCM.ObjectMeta.Annotations = newCM.ObjectMeta.Annotations
+	}
+	// add sandbox controller trigger ID back to the annotations
+	if hasTriggerID {
+		curCM.Annotations[vmeta.SandboxControllerTriggerID] = triggerID
+	}
+	// add vertica version back to the annotations
+	if hasVersion {
+		curCM.Annotations[vmeta.VersionAnnotation] = version
+	}
+	if stringMapDiffer(curCM.ObjectMeta.Labels, newCM.ObjectMeta.Labels) {
+		updated = true
+		curCM.ObjectMeta.Labels = newCM.ObjectMeta.Labels
+	}
+	if !reflect.DeepEqual(curCM.ObjectMeta.OwnerReferences, newCM.ObjectMeta.OwnerReferences) {
+		updated = true
+		curCM.ObjectMeta.OwnerReferences = newCM.ObjectMeta.OwnerReferences
+	}
+	if !reflect.DeepEqual(curCM.TypeMeta, newCM.TypeMeta) {
+		updated = true
+		curCM.TypeMeta = newCM.TypeMeta
+	}
+	if !*curCM.Immutable && stringMapDiffer(curCM.Data, newCM.Data) {
+		updated = true
+		curCM.Data = newCM.Data
+	}
+	if *curCM.Immutable != *newCM.Immutable {
+		updated = true
+		curCM.Immutable = newCM.Immutable
+	}
+
+	return updated
+}
+
 // fetchSubclustersWithSandboxes will return the qualified subclusters with their sandboxes
 func (s *SandboxSubclusterReconciler) fetchSubclustersWithSandboxes() (map[string]string, bool) {
 	vdbScSbMap := s.Vdb.GenSubclusterSandboxMap()