From 3986183267549583f27f7c822a583d4a54518b62 Mon Sep 17 00:00:00 2001 From: raaizik <132667934+raaizik@users.noreply.github.com> Date: Thu, 1 Aug 2024 15:57:48 +0300 Subject: [PATCH 1/2] Sends V*ReplicationClass to client Sends two V*RCs (one for image flattening) for RDR Signed-off-by: raaizik <132667934+raaizik@users.noreply.github.com> Co-Authored-By: Rewant Soni --- services/provider/server/server.go | 50 ++++++++++++++++ services/provider/server/server_test.go | 78 +++++++++++++++++++++++-- 2 files changed, 124 insertions(+), 4 deletions(-) diff --git a/services/provider/server/server.go b/services/provider/server/server.go index a41b934f59..697934e23a 100644 --- a/services/provider/server/server.go +++ b/services/provider/server/server.go @@ -745,6 +745,56 @@ func (s *OCSProviderServer) GetStorageClaimConfig(ctx context.Context, req *pb.S Data: mustMarshal(map[string]string{ "csi.storage.k8s.io/group-snapshotter-secret-name": provisionerSecretName, })}, + &pb.ExternalResource{ + Name: "ceph-rbd", + Kind: "VolumeReplicationClass", + Data: mustMarshal(map[string]string{ + "replication.storage.openshift.io/replication-secret-name": provisionerSecretName, + "mirroringMode": "snapshot", + }), + Annotations: map[string]string{ + "replication.storage.openshift.io/is-default-class": "true", + }, + }, + &pb.ExternalResource{ + Name: "ceph-rbd-flatten", + Kind: "VolumeReplicationClass", + Data: mustMarshal(map[string]string{ + "replication.storage.openshift.io/replication-secret-name": provisionerSecretName, + "mirroringMode": "snapshot", + }), + Labels: map[string]string{ + "flattenMode": "force", + }, + Annotations: map[string]string{ + "replication.storage.openshift.io/flatten-mode": "force", + }, + }, + &pb.ExternalResource{ + Name: "ceph-rbd", + Kind: "VolumeGroupReplicationClass", + Data: mustMarshal(map[string]string{ + "replication.storage.openshift.io/group-replication-secret-name": provisionerSecretName, + "mirroringMode": "snapshot", + }), + Annotations: map[string]string{ + "replication.storage.openshift.io/is-default-class": "true", + }, + }, + &pb.ExternalResource{ + Name: "ceph-rbd-flatten", + Kind: "VolumeGroupReplicationClass", + Data: mustMarshal(map[string]string{ + "replication.storage.openshift.io/group-replication-secret-name": provisionerSecretName, + "mirroringMode": "snapshot", + }), + Labels: map[string]string{ + "flattenMode": "force", + }, + Annotations: map[string]string{ + "replication.storage.openshift.io/flatten-mode": "force", + }, + }, &pb.ExternalResource{ Kind: "ClientProfile", Name: "ceph-rbd", diff --git a/services/provider/server/server_test.go b/services/provider/server/server_test.go index f103724990..96798b8f95 100644 --- a/services/provider/server/server_test.go +++ b/services/provider/server/server_test.go @@ -4,6 +4,7 @@ import ( "context" "encoding/json" "fmt" + "reflect" "strconv" "testing" @@ -29,9 +30,11 @@ import ( ) type externalResource struct { - Kind string `json:"kind"` - Data any `json:"data"` - Name string `json:"name"` + Kind string `json:"kind"` + Data any `json:"data"` + Name string `json:"name"` + Labels map[string]string `json:"labels"` + Annotations map[string]string `json:"annotations"` } var serverNamespace = "openshift-storage" @@ -350,7 +353,10 @@ func TestGetExternalResources(t *testing.T) { data, err := json.Marshal(mockResoruce.Data) assert.NoError(t, err) assert.Equal(t, string(extResource.Data), string(data)) + } + assert.True(t, reflect.DeepEqual(extResource.Labels, mockResoruce.Labels)) + assert.True(t, reflect.DeepEqual(extResource.Annotations, mockResoruce.Annotations)) assert.Equal(t, extResource.Kind, mockResoruce.Kind) assert.Equal(t, extResource.Name, mockResoruce.Name) } @@ -388,7 +394,8 @@ func TestGetExternalResources(t *testing.T) { assert.NoError(t, err) assert.Equal(t, string(extResource.Data), string(data)) } - + assert.True(t, reflect.DeepEqual(extResource.Labels, mockResoruce.Labels)) + assert.True(t, reflect.DeepEqual(extResource.Annotations, mockResoruce.Annotations)) assert.Equal(t, extResource.Kind, mockResoruce.Kind) assert.Equal(t, extResource.Name, mockResoruce.Name) } @@ -681,6 +688,57 @@ func TestOCSProviderServerGetStorageClaimConfig(t *testing.T) { "csi.storage.k8s.io/group-snapshotter-secret-name": "ceph-client-provisioner-8d40b6be71600457b5dec219d2ce2d4c", }, }, + "ceph-rbd-volumereplicationclass": { + Name: "ceph-rbd", + Kind: "VolumeReplicationClass", + Data: map[string]string{ + "replication.storage.openshift.io/replication-secret-name": "ceph-client-provisioner-8d40b6be71600457b5dec219d2ce2d4c", + "mirroringMode": "snapshot", + }, + + Annotations: map[string]string{ + "replication.storage.openshift.io/is-default-class": "true", + }, + }, + "ceph-rbd-flatten-volumereplicationclass": { + Name: "ceph-rbd-flatten", + Kind: "VolumeReplicationClass", + Data: map[string]string{ + "replication.storage.openshift.io/replication-secret-name": "ceph-client-provisioner-8d40b6be71600457b5dec219d2ce2d4c", + "mirroringMode": "snapshot", + }, + Labels: map[string]string{ + "flattenMode": "force", + }, + Annotations: map[string]string{ + "replication.storage.openshift.io/flatten-mode": "force", + }, + }, + "ceph-rbd-volumegroupreplicationclass": { + Name: "ceph-rbd", + Kind: "VolumeGroupReplicationClass", + Data: map[string]string{ + "replication.storage.openshift.io/group-replication-secret-name": "ceph-client-provisioner-8d40b6be71600457b5dec219d2ce2d4c", + "mirroringMode": "snapshot", + }, + Annotations: map[string]string{ + "replication.storage.openshift.io/is-default-class": "true", + }, + }, + "ceph-rbd-flatten-volumegroupreplicationclass": { + Name: "ceph-rbd-flatten", + Kind: "VolumeGroupReplicationClass", + Data: map[string]string{ + "replication.storage.openshift.io/group-replication-secret-name": "ceph-client-provisioner-8d40b6be71600457b5dec219d2ce2d4c", + "mirroringMode": "snapshot", + }, + Labels: map[string]string{ + "flattenMode": "force", + }, + Annotations: map[string]string{ + "replication.storage.openshift.io/flatten-mode": "force", + }, + }, "ceph-client-provisioner-8d40b6be71600457b5dec219d2ce2d4c": { Name: "ceph-client-provisioner-8d40b6be71600457b5dec219d2ce2d4c", Kind: "Secret", @@ -1065,6 +1123,10 @@ func TestOCSProviderServerGetStorageClaimConfig(t *testing.T) { name = fmt.Sprintf("%s-storageclass", name) } else if extResource.Kind == "VolumeGroupSnapshotClass" { name = fmt.Sprintf("%s-volumegroupsnapshotclass", name) + } else if extResource.Kind == "VolumeReplicationClass" { + name = fmt.Sprintf("%s-volumereplicationclass", name) + } else if extResource.Kind == "VolumeGroupReplicationClass" { + name = fmt.Sprintf("%s-volumegroupreplicationclass", name) } else if extResource.Kind == "ClientProfile" { name = fmt.Sprintf("%s-clientprofile", name) } @@ -1074,6 +1136,8 @@ func TestOCSProviderServerGetStorageClaimConfig(t *testing.T) { data, err := json.Marshal(mockResoruce.Data) assert.NoError(t, err) assert.Equal(t, string(extResource.Data), string(data)) + assert.True(t, reflect.DeepEqual(extResource.Labels, mockResoruce.Labels)) + assert.True(t, reflect.DeepEqual(extResource.Annotations, mockResoruce.Annotations)) assert.Equal(t, extResource.Kind, mockResoruce.Kind) assert.Equal(t, extResource.Name, mockResoruce.Name) } @@ -1096,6 +1160,10 @@ func TestOCSProviderServerGetStorageClaimConfig(t *testing.T) { name = fmt.Sprintf("%s-storageclass", name) } else if extResource.Kind == "VolumeGroupSnapshotClass" { name = fmt.Sprintf("%s-volumegroupsnapshotclass", name) + } else if extResource.Kind == "VolumeReplicationClass" { + name = fmt.Sprintf("%s-volumereplicationclass", name) + } else if extResource.Kind == "VolumeGroupReplicationClass" { + name = fmt.Sprintf("%s-volumegroupreplicationclass", name) } else if extResource.Kind == "ClientProfile" { name = fmt.Sprintf("%s-clientprofile", name) } @@ -1104,6 +1172,8 @@ func TestOCSProviderServerGetStorageClaimConfig(t *testing.T) { data, err := json.Marshal(mockResoruce.Data) assert.NoError(t, err) assert.Equal(t, string(extResource.Data), string(data)) + assert.True(t, reflect.DeepEqual(extResource.Labels, mockResoruce.Labels)) + assert.True(t, reflect.DeepEqual(extResource.Annotations, mockResoruce.Annotations)) assert.Equal(t, extResource.Kind, mockResoruce.Kind) assert.Equal(t, extResource.Name, mockResoruce.Name) } From 52095140b680ff38978d16fc68a307e581af0006 Mon Sep 17 00:00:00 2001 From: raaizik <132667934+raaizik@users.noreply.github.com> Date: Mon, 23 Sep 2024 12:48:25 +0300 Subject: [PATCH 2/2] getExternalResourceLabels Signed-off-by: raaizik <132667934+raaizik@users.noreply.github.com> --- services/provider/server/server.go | 82 ++++++++++++++++++++++--- services/provider/server/server_test.go | 30 ++++++--- 2 files changed, 93 insertions(+), 19 deletions(-) diff --git a/services/provider/server/server.go b/services/provider/server/server.go index 697934e23a..4f559b08df 100644 --- a/services/provider/server/server.go +++ b/services/provider/server/server.go @@ -3,6 +3,7 @@ package server import ( "context" "crypto" + "crypto/md5" "crypto/rsa" "crypto/sha256" "crypto/x509" @@ -668,10 +669,32 @@ func (s *OCSProviderServer) GetStorageClaimConfig(ctx context.Context, req *pb.S return nil, status.Error(codes.Internal, msg) } } + + // fetch storage cluster peer to indicate whether replication is enabled + scp := &ocsv1.StorageClusterPeerList{} + if err = s.client.List(ctx, scp, client.InNamespace(s.namespace)); err != nil { + return nil, status.Errorf(codes.Internal, "failed to get StorageClusterPeerList. %v", err) + } + replicationEnabled := len(scp.Items) > 1 + ID := "" + if replicationEnabled { + storageClaimName, err := json.Marshal(req.StorageClaimName) + if err != nil { + klog.Errorf("failed to marshal a name for a storage claim request based on %v. %v", s, err) + panic("failed to marshal storage class request name") + } + md5Sum := md5.Sum(storageClaimName) + ID = hex.EncodeToString(md5Sum[:16]) + } // todo storageID in provider is planned to be string(r.storageClaim.UID) + // SID for RamenDR + SID := req.StorageConsumerUUID + var extR []*pb.ExternalResource storageRequestHash := getStorageRequestHash(req.StorageConsumerUUID, req.StorageClaimName) + for _, cephRes := range storageRequest.Status.CephResources { + switch cephRes.Kind { case "CephClient": clientSecretName, clientUserType, err := s.getCephClientInformation(ctx, cephRes.Name) @@ -702,13 +725,19 @@ func (s *OCSProviderServer) GetStorageClaimConfig(ctx context.Context, req *pb.S }) case "CephBlockPoolRadosNamespace": - rns := &rookCephv1.CephBlockPoolRadosNamespace{} err = s.client.Get(ctx, types.NamespacedName{Name: cephRes.Name, Namespace: s.namespace}, rns) if err != nil { return nil, status.Errorf(codes.Internal, "failed to get %s CephBlockPoolRadosNamespace. %v", cephRes.Name, err) } - + // Fetch mirroring flag for the current ceph resource + cbp := &rookCephv1.CephBlockPool{} + cbp.Name = rns.Spec.BlockPoolName + cbp.Namespace = s.namespace + if err = s.client.Get(ctx, client.ObjectKeyFromObject(cbp), cbp); err != nil { + return nil, status.Errorf(codes.Internal, "failed to get %s CephBlockPool. %v", cephRes.Name, err) + } + mirroringEnabled := cbp.Spec.Mirroring.Enabled provisionerSecretName := storageClaimCephCsiSecretName("provisioner", storageRequestHash) nodeSecretName := storageClaimCephCsiSecretName("node", storageRequestHash) rbdStorageClassData := map[string]string{ @@ -752,6 +781,7 @@ func (s *OCSProviderServer) GetStorageClaimConfig(ctx context.Context, req *pb.S "replication.storage.openshift.io/replication-secret-name": provisionerSecretName, "mirroringMode": "snapshot", }), + Labels: getExternalResourceLabels("VolumeReplicationClass", replicationEnabled, mirroringEnabled, false, ID, SID), Annotations: map[string]string{ "replication.storage.openshift.io/is-default-class": "true", }, @@ -762,12 +792,11 @@ func (s *OCSProviderServer) GetStorageClaimConfig(ctx context.Context, req *pb.S Data: mustMarshal(map[string]string{ "replication.storage.openshift.io/replication-secret-name": provisionerSecretName, "mirroringMode": "snapshot", + "flattenMode": "force", }), - Labels: map[string]string{ - "flattenMode": "force", - }, + Labels: getExternalResourceLabels("VolumeReplicationClass", replicationEnabled, mirroringEnabled, true, ID, SID), Annotations: map[string]string{ - "replication.storage.openshift.io/flatten-mode": "force", + "replication.storage.openshift.io/is-default-class": "true", }, }, &pb.ExternalResource{ @@ -777,6 +806,7 @@ func (s *OCSProviderServer) GetStorageClaimConfig(ctx context.Context, req *pb.S "replication.storage.openshift.io/group-replication-secret-name": provisionerSecretName, "mirroringMode": "snapshot", }), + Labels: getExternalResourceLabels("VolumeGroupReplicationClass", replicationEnabled, mirroringEnabled, false, ID, SID), Annotations: map[string]string{ "replication.storage.openshift.io/is-default-class": "true", }, @@ -787,12 +817,11 @@ func (s *OCSProviderServer) GetStorageClaimConfig(ctx context.Context, req *pb.S Data: mustMarshal(map[string]string{ "replication.storage.openshift.io/group-replication-secret-name": provisionerSecretName, "mirroringMode": "snapshot", + "flattenMode": "force", }), - Labels: map[string]string{ - "flattenMode": "force", - }, + Labels: getExternalResourceLabels("VolumeGroupReplicationClass", replicationEnabled, mirroringEnabled, true, ID, SID), Annotations: map[string]string{ - "replication.storage.openshift.io/flatten-mode": "force", + "replication.storage.openshift.io/is-default-class": "true", }, }, &pb.ExternalResource{ @@ -882,6 +911,39 @@ func (s *OCSProviderServer) GetStorageClaimConfig(ctx context.Context, req *pb.S } +func getExternalResourceLabels(kind string, isReplicationEnabled bool, isMirroringEnabled bool, isFlattenMode bool, + replicationID string, storageID string) map[string]string { + labels := make(map[string]string) + switch kind { + case "VolumeReplicationClass", "VolumeGroupReplicationClass": + if isReplicationEnabled { + replicationID := replicationID + labels["ramendr.openshift.io/replicationid"] = replicationID + if isFlattenMode { + labels["replication.storage.openshift.io/flatten-mode"] = "force" + } + } + if isMirroringEnabled { + labels["ramendr.openshift.io/storageID"] = storageID + } + case "VolumeSnapshotClass", "VolumeGroupSnapshotClass": + if isMirroringEnabled { + labels["ramendr.openshift.io/storageID"] = storageID + } + case "StorageClass": + if isReplicationEnabled { + labels["ramendr.openshift.io/replicationid"] = replicationID + } + if isMirroringEnabled { + labels["ramendr.openshift.io/storageID"] = storageID + } + + default: + panic(fmt.Sprintf("unknown storage class kind %q", kind)) + } + return labels +} + // ReportStatus rpc call to check if a consumer can reach to the provider. func (s *OCSProviderServer) ReportStatus(ctx context.Context, req *pb.ReportStatusRequest) (*pb.ReportStatusResponse, error) { // Update the status in storageConsumer CR diff --git a/services/provider/server/server_test.go b/services/provider/server/server_test.go index 96798b8f95..97464bb6a7 100644 --- a/services/provider/server/server_test.go +++ b/services/provider/server/server_test.go @@ -695,7 +695,7 @@ func TestOCSProviderServerGetStorageClaimConfig(t *testing.T) { "replication.storage.openshift.io/replication-secret-name": "ceph-client-provisioner-8d40b6be71600457b5dec219d2ce2d4c", "mirroringMode": "snapshot", }, - + Labels: map[string]string{}, Annotations: map[string]string{ "replication.storage.openshift.io/is-default-class": "true", }, @@ -706,12 +706,11 @@ func TestOCSProviderServerGetStorageClaimConfig(t *testing.T) { Data: map[string]string{ "replication.storage.openshift.io/replication-secret-name": "ceph-client-provisioner-8d40b6be71600457b5dec219d2ce2d4c", "mirroringMode": "snapshot", + "flattenMode": "force", }, - Labels: map[string]string{ - "flattenMode": "force", - }, + Labels: map[string]string{}, Annotations: map[string]string{ - "replication.storage.openshift.io/flatten-mode": "force", + "replication.storage.openshift.io/is-default-class": "true", }, }, "ceph-rbd-volumegroupreplicationclass": { @@ -721,6 +720,7 @@ func TestOCSProviderServerGetStorageClaimConfig(t *testing.T) { "replication.storage.openshift.io/group-replication-secret-name": "ceph-client-provisioner-8d40b6be71600457b5dec219d2ce2d4c", "mirroringMode": "snapshot", }, + Labels: map[string]string{}, Annotations: map[string]string{ "replication.storage.openshift.io/is-default-class": "true", }, @@ -731,12 +731,11 @@ func TestOCSProviderServerGetStorageClaimConfig(t *testing.T) { Data: map[string]string{ "replication.storage.openshift.io/group-replication-secret-name": "ceph-client-provisioner-8d40b6be71600457b5dec219d2ce2d4c", "mirroringMode": "snapshot", + "flattenMode": "force", }, - Labels: map[string]string{ - "flattenMode": "force", - }, + Labels: map[string]string{}, Annotations: map[string]string{ - "replication.storage.openshift.io/flatten-mode": "force", + "replication.storage.openshift.io/is-default-class": "true", }, }, "ceph-client-provisioner-8d40b6be71600457b5dec219d2ce2d4c": { @@ -1105,6 +1104,18 @@ func TestOCSProviderServerGetStorageClaimConfig(t *testing.T) { } assert.NoError(t, client.Create(ctx, radosNamespace)) + cephBlockPool := &rookCephv1.CephBlockPool{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cephblockpool", + Namespace: server.namespace, + }, + Spec: rookCephv1.NamedBlockPoolSpec{ + PoolSpec: rookCephv1.PoolSpec{ + Mirroring: rookCephv1.MirroringSpec{Enabled: false}, + }}, + } + assert.NoError(t, client.Create(ctx, cephBlockPool)) + // get the storage class request config for block pool req := pb.StorageClaimConfigRequest{ StorageConsumerUUID: string(consumerResource.UID), @@ -1136,6 +1147,7 @@ func TestOCSProviderServerGetStorageClaimConfig(t *testing.T) { data, err := json.Marshal(mockResoruce.Data) assert.NoError(t, err) assert.Equal(t, string(extResource.Data), string(data)) + assert.Equal(t, extResource.Labels, mockResoruce.Labels) assert.True(t, reflect.DeepEqual(extResource.Labels, mockResoruce.Labels)) assert.True(t, reflect.DeepEqual(extResource.Annotations, mockResoruce.Annotations)) assert.Equal(t, extResource.Kind, mockResoruce.Kind)