diff --git a/addons/agent_mirrorpeer_controller.go b/addons/agent_mirrorpeer_controller.go index e44af4cd..1bf0760d 100644 --- a/addons/agent_mirrorpeer_controller.go +++ b/addons/agent_mirrorpeer_controller.go @@ -115,14 +115,21 @@ func (r *MirrorPeerReconciler) Reconcile(ctx context.Context, req ctrl.Request) return ctrl.Result{}, nil } + hasStorageClientRef, err := utils.IsStorageClientType(ctx, r.SpokeClient, mirrorPeer, true) + + if err != nil { + logger.Error("Failed to check if storage client ref exists", "error", err) + return ctrl.Result{}, err + } + logger.Info("Creating S3 buckets") - err = r.createS3(ctx, mirrorPeer, scr.Namespace) + err = r.createS3(ctx, mirrorPeer, scr.Namespace, hasStorageClientRef) if err != nil { logger.Error("Failed to create ODR S3 resources", "error", err) return ctrl.Result{}, err } - if mirrorPeer.Spec.Type == multiclusterv1alpha1.Async && !utils.IsStorageClientType(mirrorPeer.Spec.Items) { + if mirrorPeer.Spec.Type == multiclusterv1alpha1.Async && !hasStorageClientRef { clusterFSIDs := make(map[string]string) logger.Info("Fetching clusterFSIDs") err = r.fetchClusterFSIDs(ctx, &mirrorPeer, clusterFSIDs) @@ -156,7 +163,7 @@ func (r *MirrorPeerReconciler) Reconcile(ctx context.Context, req ctrl.Request) } } - if mirrorPeer.Spec.Type == multiclusterv1alpha1.Async { + if mirrorPeer.Spec.Type == multiclusterv1alpha1.Async && hasStorageClientRef { if mirrorPeer.Status.Phase == multiclusterv1alpha1.ExchangedSecret { logger.Info("Cleaning up stale onboarding token", "Token", string(mirrorPeer.GetUID())) err = deleteStorageClusterPeerTokenSecret(ctx, r.HubClient, r.SpokeClusterName, string(mirrorPeer.GetUID())) @@ -306,16 +313,15 @@ func (r *MirrorPeerReconciler) labelRBDStorageClasses(ctx context.Context, stora return errs } -func (r *MirrorPeerReconciler) createS3(ctx context.Context, mirrorPeer multiclusterv1alpha1.MirrorPeer, scNamespace string) error { - logger := r.Logger.With("MirrorPeer", mirrorPeer.Name) +func (r *MirrorPeerReconciler) createS3(ctx context.Context, mirrorPeer multiclusterv1alpha1.MirrorPeer, scNamespace string, hasStorageClientRef bool) error { bucketCount := 1 - if utils.IsStorageClientType(mirrorPeer.Spec.Items) { + if hasStorageClientRef { bucketCount = 2 } for index := 0; index < bucketCount; index++ { bucketNamespace := utils.GetEnv("ODR_NAMESPACE", scNamespace) var bucketName string - if utils.IsStorageClientType(mirrorPeer.Spec.Items) { + if hasStorageClientRef { bucketName = utils.GenerateBucketName(mirrorPeer, mirrorPeer.Spec.Items[index].StorageClusterRef.Name) } else { bucketName = utils.GenerateBucketName(mirrorPeer) @@ -324,7 +330,7 @@ func (r *MirrorPeerReconciler) createS3(ctx context.Context, mirrorPeer multiclu if err != nil { return err } - logger.Info(fmt.Sprintf("ObjectBucketClaim %s was %s in namespace %s", bucketName, operationResult, bucketNamespace)) + r.Logger.Info(fmt.Sprintf("ObjectBucketClaim %s was %s in namespace %s", bucketName, operationResult, bucketNamespace)) } return nil diff --git a/addons/agent_mirrorpeer_controller_test.go b/addons/agent_mirrorpeer_controller_test.go index 643e927e..e2dc8989 100644 --- a/addons/agent_mirrorpeer_controller_test.go +++ b/addons/agent_mirrorpeer_controller_test.go @@ -21,6 +21,27 @@ import ( ) var ( + odfInfoConfigMap = corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: "odf-info", + Namespace: "test-namespace", // Use a generic namespace + UID: types.UID("268e6cdb-54fc-4f10-afab-67b106880be3"), + }, + Data: map[string]string{ + "test-namespace_test-storagecluster.config.yaml": ` +version: 4.17.0-95.stable +deploymentType: internal +clients: [] +storageCluster: + namespacedName: + namespace: test-namespace + name: test-storagecluster + storageProviderEndpoint: "" + cephClusterFSID: 986532da-8dba-4d35-a8d2-12f037712b39 +storageSystemName: ocs-storagecluster-storagesystem +`, + }, + } mpItems = []multiclusterv1alpha1.PeerRef{ { ClusterName: "cluster1", @@ -141,7 +162,7 @@ func TestMirrorPeerReconcile(t *testing.T) { rcm.Data = make(map[string]string) rcm.Data[RookCSIEnableKey] = "false" - fakeSpokeClient := fake.NewClientBuilder().WithScheme(scheme).WithObjects(&storageCluster, &rcm, &clusterPeerToken, &exchangedSecret1, &exchangedSecret2, rbdStorageClass, cephfsStorageClass).Build() + fakeSpokeClient := fake.NewClientBuilder().WithScheme(scheme).WithObjects(&storageCluster, &rcm, &clusterPeerToken, &exchangedSecret1, &exchangedSecret2, rbdStorageClass, cephfsStorageClass, &odfInfoConfigMap).Build() r := MirrorPeerReconciler{ HubClient: fakeHubClient, diff --git a/api/v1alpha1/mirrorpeer_types.go b/api/v1alpha1/mirrorpeer_types.go index f88d9b8c..ec11e75f 100644 --- a/api/v1alpha1/mirrorpeer_types.go +++ b/api/v1alpha1/mirrorpeer_types.go @@ -38,7 +38,7 @@ type StorageClusterRef struct { Name string `json:"name"` // +kubebuilder:validation:Optional - Namespace string `json:"namespace"` + Namespace string `json:"namespace,omitempty"` } // PeerRef holds a reference to a mirror peer diff --git a/controllers/drpolicy_controller.go b/controllers/drpolicy_controller.go index 251c8362..ffaca719 100644 --- a/controllers/drpolicy_controller.go +++ b/controllers/drpolicy_controller.go @@ -113,7 +113,14 @@ func (r *DRPolicyReconciler) Reconcile(ctx context.Context, req ctrl.Request) (c return ctrl.Result{}, err } - if utils.IsStorageClientType(mirrorPeer.Spec.Items) { + // Check if the MirrorPeer contains StorageClient reference + hasStorageClientRef, err := utils.IsStorageClientType(ctx, r.HubClient, *mirrorPeer, false) + if err != nil { + logger.Error("Failed to determine if MirrorPeer contains StorageClient reference", "error", err) + return ctrl.Result{}, err + } + + if hasStorageClientRef { logger.Info("MirrorPeer contains StorageClient reference. Skipping creation of VolumeReplicationClasses", "MirrorPeer", mirrorPeer.Name) return ctrl.Result{}, nil } diff --git a/controllers/drpolicy_controller_test.go b/controllers/drpolicy_controller_test.go index 1d21a398..389f9c55 100644 --- a/controllers/drpolicy_controller_test.go +++ b/controllers/drpolicy_controller_test.go @@ -3,11 +3,13 @@ package controllers import ( "context" "fmt" + "os" "testing" ramenv1alpha1 "github.com/ramendr/ramen/api/v1alpha1" multiclusterv1alpha1 "github.com/red-hat-storage/odf-multicluster-orchestrator/api/v1alpha1" "github.com/red-hat-storage/odf-multicluster-orchestrator/controllers/utils" + viewv1beta1 "github.com/stolostron/multicloud-operators-foundation/pkg/apis/view/v1beta1" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" @@ -100,6 +102,7 @@ func TestDRPolicyReconcile(t *testing.T) { func getFakeDRPolicyReconciler(drpolicy *ramenv1alpha1.DRPolicy, mp *multiclusterv1alpha1.MirrorPeer) DRPolicyReconciler { scheme := mgrScheme + os.Setenv("POD_NAMESPACE", "openshift-operators") ns1 := &corev1.Namespace{ ObjectMeta: metav1.ObjectMeta{ Name: cName1, @@ -139,8 +142,23 @@ func getFakeDRPolicyReconciler(drpolicy *ramenv1alpha1.DRPolicy, mp *multicluste }, Type: "multicluster.odf.openshift.io/secret-type", } + odfClientInfoConfigMap := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: "odf-client-info", + Namespace: os.Getenv("POD_NAMESPACE"), + OwnerReferences: []metav1.OwnerReference{ + { + APIVersion: viewv1beta1.GroupVersion.String(), + Kind: "ManagedClusterView", + Name: "mcv-1", + UID: "mcv-uid", + }, + }, + }, + Data: map[string]string{}, + } - fakeClient := fake.NewClientBuilder().WithScheme(scheme).WithObjects(drpolicy, mp, ns1, ns2, &hubSecret1, &hubSecret2).Build() + fakeClient := fake.NewClientBuilder().WithScheme(scheme).WithObjects(drpolicy, mp, ns1, ns2, &hubSecret1, &hubSecret2, odfClientInfoConfigMap).Build() r := DRPolicyReconciler{ HubClient: fakeClient, diff --git a/controllers/managedclusterview_controller.go b/controllers/managedclusterview_controller.go index b766026c..689a6a7d 100644 --- a/controllers/managedclusterview_controller.go +++ b/controllers/managedclusterview_controller.go @@ -78,7 +78,7 @@ func (r *ManagedClusterViewReconciler) SetupWithManager(mgr ctrl.Manager) error } func hasODFInfoInScope(mc *viewv1beta1.ManagedClusterView) bool { - if mc.Spec.Scope.Name == ODFInfoConfigMapName && mc.Spec.Scope.Resource == ConfigMapResourceType { + if mc.Spec.Scope.Name == utils.ODFInfoConfigMapName && mc.Spec.Scope.Resource == ConfigMapResourceType { return true } return false diff --git a/controllers/mirrorpeer_controller.go b/controllers/mirrorpeer_controller.go index 2dbb3876..f11559f6 100644 --- a/controllers/mirrorpeer_controller.go +++ b/controllers/mirrorpeer_controller.go @@ -59,7 +59,6 @@ type MirrorPeerReconciler struct { const ( mirrorPeerFinalizer = "hub.multicluster.odf.openshift.io" spokeClusterRoleBindingName = "spoke-clusterrole-bindings" - ClientConfigMapKeyTemplate = "%s/%s" ) //+kubebuilder:rbac:groups=multicluster.odf.openshift.io,resources=mirrorpeers,verbs=get;list;watch;create;update;patch;delete @@ -247,7 +246,14 @@ func (r *MirrorPeerReconciler) Reconcile(ctx context.Context, req ctrl.Request) return ctrl.Result{}, err } - if utils.IsStorageClientType(mirrorPeer.Spec.Items) { + // Check if the MirrorPeer contains StorageClient reference + hasStorageClientRef, err := utils.IsStorageClientType(ctx, r.Client, mirrorPeer, false) + if err != nil { + logger.Error("Failed to determine if MirrorPeer contains StorageClient reference", "error", err) + return ctrl.Result{}, err + } + + if hasStorageClientRef { result, err := createStorageClusterPeer(ctx, r.Client, logger, mirrorPeer) if err != nil { logger.Error("Failed to create StorageClusterPeer", "error", err) @@ -258,7 +264,7 @@ func (r *MirrorPeerReconciler) Reconcile(ctx context.Context, req ctrl.Request) } func getKey(clusterName, clientName string) string { - return fmt.Sprintf(ClientConfigMapKeyTemplate, clusterName, clientName) + return fmt.Sprintf("%s/%s", clusterName, clientName) } func createStorageClusterPeer(ctx context.Context, client client.Client, logger *slog.Logger, mirrorPeer multiclusterv1alpha1.MirrorPeer) (ctrl.Result, error) { @@ -298,7 +304,7 @@ func createStorageClusterPeer(ctx context.Context, client client.Client, logger // Provider B's onboarding token will be used for Provider A's StorageClusterPeer onboardingToken, err := fetchOnboardingTicket(ctx, client, oppositeClient, mirrorPeer) if err != nil { - return ctrl.Result{}, fmt.Errorf("failed to fetch onboarding token for provider %s. err %w", oppositeClient.ProviderInfo.ProviderManagedClusterName, err) + return ctrl.Result{}, fmt.Errorf("failed to fetch onboarding token for provider %s. %w", oppositeClient.ProviderInfo.ProviderManagedClusterName, err) } storageClusterPeer := ocsv1.StorageClusterPeer{ ObjectMeta: metav1.ObjectMeta{ @@ -374,7 +380,7 @@ func fetchClientInfoConfigMap(ctx context.Context, c client.Client) (*corev1.Con if currentNamespace == "" { return nil, fmt.Errorf("cannot detect the current namespace") } - clientInfoMap, err := utils.FetchConfigMap(ctx, c, ClientInfoConfigMapName, currentNamespace) + clientInfoMap, err := utils.FetchConfigMap(ctx, c, utils.ClientInfoConfigMapName, currentNamespace) if err != nil { return nil, err } @@ -409,7 +415,14 @@ func getClientInfoFromConfigMap(clientInfoMap map[string]string, key string) (Cl func getConfig(ctx context.Context, c client.Client, mp multiclusterv1alpha1.MirrorPeer) ([]ManagedClusterAddonConfig, error) { managedClusterAddonsConfig := make([]ManagedClusterAddonConfig, 0) - if utils.IsStorageClientType(mp.Spec.Items) { + + // Check if the MirrorPeer contains StorageClient reference + hasStorageClientRef, err := utils.IsStorageClientType(ctx, c, mp, false) + if err != nil { + return []ManagedClusterAddonConfig{}, err + } + + if hasStorageClientRef { clientInfoMap, err := fetchClientInfoConfigMap(ctx, c) if err != nil { return []ManagedClusterAddonConfig{}, err diff --git a/controllers/mirrorpeer_controller_test.go b/controllers/mirrorpeer_controller_test.go index 8f95a834..8bc59887 100644 --- a/controllers/mirrorpeer_controller_test.go +++ b/controllers/mirrorpeer_controller_test.go @@ -21,11 +21,13 @@ package controllers import ( "context" + "os" "testing" "github.com/red-hat-storage/odf-multicluster-orchestrator/addons/setup" multiclusterv1alpha1 "github.com/red-hat-storage/odf-multicluster-orchestrator/api/v1alpha1" "github.com/red-hat-storage/odf-multicluster-orchestrator/controllers/utils" + viewv1beta1 "github.com/stolostron/multicloud-operators-foundation/pkg/apis/view/v1beta1" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" @@ -91,7 +93,7 @@ func getFakeMirrorPeerReconciler(mirrorpeer multiclusterv1alpha1.MirrorPeer) Mir // Using a different scheme for test might cause issues like // missing scheme in manager scheme := mgrScheme - + os.Setenv("POD_NAMESPACE", "openshift-operators") managedcluster1 := clusterv1.ManagedCluster{ ObjectMeta: metav1.ObjectMeta{ Name: "cluster1", @@ -106,7 +108,23 @@ func getFakeMirrorPeerReconciler(mirrorpeer multiclusterv1alpha1.MirrorPeer) Mir Spec: clusterv1.ManagedClusterSpec{}, } - fakeClient := fake.NewClientBuilder().WithScheme(scheme).WithObjects(&mirrorpeer, &managedcluster1, &managedcluster2).Build() + var odfClientInfoConfigMap = &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: "odf-client-info", + Namespace: os.Getenv("POD_NAMESPACE"), + OwnerReferences: []metav1.OwnerReference{ + { + APIVersion: viewv1beta1.GroupVersion.String(), + Kind: "ManagedClusterView", + Name: "mcv-1", + UID: "mcv-uid", + }, + }, + }, + Data: map[string]string{}, + } + + fakeClient := fake.NewClientBuilder().WithScheme(scheme).WithObjects(&mirrorpeer, &managedcluster1, &managedcluster2, odfClientInfoConfigMap).Build() r := MirrorPeerReconciler{ Client: fakeClient, diff --git a/controllers/utils/configmap.go b/controllers/utils/configmap.go index 9bd9c4c2..2f6fc357 100644 --- a/controllers/utils/configmap.go +++ b/controllers/utils/configmap.go @@ -3,11 +3,20 @@ package utils import ( "context" "fmt" + "strings" corev1 "k8s.io/api/core/v1" + k8serrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/client" ) +const ( + ODFInfoConfigMapName = "odf-info" + ConfigMapResourceType = "ConfigMap" + ClientInfoConfigMapName = "odf-client-info" +) + // FetchConfigMap fetches a ConfigMap with a given name from a given namespace func FetchConfigMap(ctx context.Context, c client.Client, name, namespace string) (*corev1.ConfigMap, error) { configMap := &corev1.ConfigMap{} @@ -16,7 +25,22 @@ func FetchConfigMap(ctx context.Context, c client.Client, name, namespace string Namespace: namespace, }, configMap) if err != nil { + if k8serrors.IsNotFound(err) { + return nil, err + } return nil, fmt.Errorf("failed to fetch ConfigMap %s in namespace %s: %v", name, namespace, err) } return configMap, nil } + +// GetODFInfoConfigMap fetches the odf-info ConfigMap from the given namespace. This will only work on the managed cluster +func GetODFInfoConfigMap(ctx context.Context, c client.Client, namespace string) (*corev1.ConfigMap, error) { + return FetchConfigMap(ctx, c, ODFInfoConfigMapName, namespace) +} + +func SplitKeyForNamespacedName(key string) types.NamespacedName { + // key = openshift-storage_ocs-storagecluster.config.yaml + splitKey := strings.Split(key, ".") // [openshift-storage_ocs-storagecluster,config,yaml] + namespacedName := strings.Split(splitKey[0], "_") // [openshift-storage,ocs-storagecluster] + return types.NamespacedName{Namespace: namespacedName[0], Name: namespacedName[1]} +} diff --git a/controllers/utils/peer_ref.go b/controllers/utils/peer_ref.go index e3d7d366..4b9230f7 100644 --- a/controllers/utils/peer_ref.go +++ b/controllers/utils/peer_ref.go @@ -3,11 +3,28 @@ package utils import ( "context" "fmt" + "os" + ocsv1alpha1 "github.com/red-hat-storage/ocs-operator/api/v4/v1alpha1" multiclusterv1alpha1 "github.com/red-hat-storage/odf-multicluster-orchestrator/api/v1alpha1" + "gopkg.in/yaml.v2" + k8serrors "k8s.io/apimachinery/pkg/api/errors" "sigs.k8s.io/controller-runtime/pkg/client" ) +type PeerRefType string + +const ( + // PeerRefTypeStorageClient represents a storage client + PeerRefTypeStorageClient PeerRefType = "StorageClient" + + // PeerRefTypeStorageCluster represents a storage cluster + PeerRefTypeStorageCluster PeerRefType = "StorageCluster" + + // PeerRefTypeUnknown represents an unknown type + PeerRefTypeUnknown PeerRefType = "Unknown" +) + // DoesAnotherMirrorPeerPointToPeerRef checks if another mirrorpeer is pointing to the provided peer ref func DoesAnotherMirrorPeerPointToPeerRef(ctx context.Context, rc client.Client, peerRef *multiclusterv1alpha1.PeerRef) (bool, error) { mirrorPeers, err := FetchAllMirrorPeers(ctx, rc) @@ -34,6 +51,56 @@ func GetPeerRefForSpokeCluster(mp *multiclusterv1alpha1.MirrorPeer, spokeCluster return nil, fmt.Errorf("PeerRef for cluster %s under mirrorpeer %s not found", spokeClusterName, mp.Name) } -func IsStorageClientType(peerRefs []multiclusterv1alpha1.PeerRef) bool { - return peerRefs[0].StorageClusterRef.Namespace == "" && peerRefs[1].StorageClusterRef.Namespace == "" +func getPeerRefType(ctx context.Context, c client.Client, peerRef multiclusterv1alpha1.PeerRef, isManagedCluster bool) (PeerRefType, error) { + if isManagedCluster { + cm, err := GetODFInfoConfigMap(ctx, c, peerRef.StorageClusterRef.Namespace) + if err != nil { + return PeerRefTypeUnknown, fmt.Errorf("failed to get ODF Info ConfigMap for namespace %s: %w", peerRef.StorageClusterRef.Namespace, err) + } + var odfInfo ocsv1alpha1.OdfInfoData + for key, value := range cm.Data { + namespacedName := SplitKeyForNamespacedName(key) + if namespacedName.Name == peerRef.StorageClusterRef.Name { + err := yaml.Unmarshal([]byte(value), &odfInfo) + if err != nil { + return PeerRefTypeUnknown, fmt.Errorf("failed to unmarshal ODF info data for key %s: %w", key, err) + } + + for _, client := range odfInfo.Clients { + if client.Name == peerRef.ClusterName { + return PeerRefTypeStorageClient, nil + } + } + } + } + return PeerRefTypeStorageCluster, nil + } else { + operatorNamespace := os.Getenv("POD_NAMESPACE") + cm, err := FetchConfigMap(ctx, c, ClientInfoConfigMapName, operatorNamespace) + if k8serrors.IsNotFound(err) { + return PeerRefTypeStorageCluster, nil + } + if err != nil { + return PeerRefTypeUnknown, err + } + + if _, ok := cm.Data[peerRef.ClusterName]; ok { + return PeerRefTypeStorageClient, nil + } + return PeerRefTypeStorageCluster, nil + } +} + +// IsStorageClientType checks if peerRefs on MirrorPeer is of type StorageClient or StorageCluster +func IsStorageClientType(ctx context.Context, c client.Client, mirrorPeer multiclusterv1alpha1.MirrorPeer, isManagedCluster bool) (bool, error) { + for _, v := range mirrorPeer.Spec.Items { + peerRefType, err := getPeerRefType(ctx, c, v, isManagedCluster) + if err != nil { + return false, err + } + if peerRefType != PeerRefTypeStorageClient { + return false, nil + } + } + return true, nil } diff --git a/go.mod b/go.mod index abe5c159..aa7127fa 100644 --- a/go.mod +++ b/go.mod @@ -20,6 +20,7 @@ require ( go.uber.org/zap v1.27.0 go.uber.org/zap/exp v0.2.0 golang.org/x/sync v0.7.0 + gopkg.in/yaml.v2 v2.4.0 k8s.io/api v0.30.2 k8s.io/apiextensions-apiserver v0.29.2 k8s.io/apimachinery v0.30.2 @@ -108,7 +109,6 @@ require ( google.golang.org/protobuf v1.33.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect - gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect k8s.io/apiserver v0.29.2 // indirect k8s.io/component-base v0.29.2 // indirect